saber-pt 0.5.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- saber_pt-0.5.4/LICENSE +21 -0
- saber_pt-0.5.4/PKG-INFO +340 -0
- saber_pt-0.5.4/README.md +311 -0
- saber_pt-0.5.4/pyproject.toml +58 -0
- saber_pt-0.5.4/saber/__init__.py +58 -0
- saber_pt-0.5.4/saber/_compat.py +31 -0
- saber_pt-0.5.4/saber/extraction.py +357 -0
- saber_pt-0.5.4/saber/label_translation.py +387 -0
- saber_pt-0.5.4/saber/matchers/a1_nouns.py +256 -0
- saber_pt-0.5.4/saber/matchers/a2_1_adjectives.py +349 -0
- saber_pt-0.5.4/saber/matchers/a3_1_verbs_tense_mood.py +2248 -0
- saber_pt-0.5.4/saber/matchers/a3_2_compound_verbs.py +1816 -0
- saber_pt-0.5.4/saber/matchers/a4_1_adverbs.py +46 -0
- saber_pt-0.5.4/saber/matchers/a5_1_personal_pronouns.py +299 -0
- saber_pt-0.5.4/saber/matchers/a5_2_demonstratives.py +235 -0
- saber_pt-0.5.4/saber/matchers/a5_3_possessives.py +172 -0
- saber_pt-0.5.4/saber/matchers/a5_4_indefinite_pronouns.py +96 -0
- saber_pt-0.5.4/saber/matchers/a5_5_interrogatives.py +110 -0
- saber_pt-0.5.4/saber/matchers/a5_6_relative_pronouns.py +159 -0
- saber_pt-0.5.4/saber/matchers/a6_1_articles.py +574 -0
- saber_pt-0.5.4/saber/matchers/a6_2_demonstratives.py +269 -0
- saber_pt-0.5.4/saber/matchers/a6_3_possessives.py +93 -0
- saber_pt-0.5.4/saber/matchers/a6_4_indefinite_determiners.py +70 -0
- saber_pt-0.5.4/saber/matchers/a6_6_relative_determiners.py +41 -0
- saber_pt-0.5.4/saber/matchers/a7_1_existentials.py +73 -0
- saber_pt-0.5.4/saber/matchers/a7_2_universals.py +153 -0
- saber_pt-0.5.4/saber/matchers/a7_3_numerals.py +169 -0
- saber_pt-0.5.4/saber/matchers/a7_4_interrogatives.py +26 -0
- saber_pt-0.5.4/saber/matchers/a7_5_relative_quantifiers.py +21 -0
- saber_pt-0.5.4/saber/matchers/b2_1_syntactic_functions.py +1740 -0
- saber_pt-0.5.4/saber/matchers/b2_2_order_of_constituents.py +231 -0
- saber_pt-0.5.4/saber/matchers/b2_3_agreement.py +808 -0
- saber_pt-0.5.4/saber/matchers/b3_2_interrogatives.py +348 -0
- saber_pt-0.5.4/saber/matchers/b3_3_exclamatives.py +281 -0
- saber_pt-0.5.4/saber/matchers/b3_4_imperatives.py +158 -0
- saber_pt-0.5.4/saber/matchers/b3_5_active_passive_sentences.py +359 -0
- saber_pt-0.5.4/saber/matchers/b4_1_sentence_polarity.py +27 -0
- saber_pt-0.5.4/saber/matchers/b5_1_coordination.py +525 -0
- saber_pt-0.5.4/saber/matchers/b5_2_subordination.py +2496 -0
- saber_pt-0.5.4/saber/models.py +61 -0
- saber_pt-0.5.4/saber/nlp.py +189 -0
- saber_pt-0.5.4/saber/process_and_display.py +19 -0
- saber_pt-0.5.4/saber/progress.py +103 -0
- saber_pt-0.5.4/saber/registry.py +360 -0
- saber_pt-0.5.4/saber/sources.py +273 -0
- saber_pt-0.5.4/saber/text_reconstruction.py +149 -0
- saber_pt-0.5.4/saber_pt.egg-info/PKG-INFO +340 -0
- saber_pt-0.5.4/saber_pt.egg-info/SOURCES.txt +51 -0
- saber_pt-0.5.4/saber_pt.egg-info/dependency_links.txt +1 -0
- saber_pt-0.5.4/saber_pt.egg-info/requires.txt +10 -0
- saber_pt-0.5.4/saber_pt.egg-info/top_level.txt +1 -0
- saber_pt-0.5.4/setup.cfg +4 -0
- saber_pt-0.5.4/tests/test_suite.py +250 -0
saber_pt-0.5.4/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Soroosh Akef
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
saber_pt-0.5.4/PKG-INFO
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: saber-pt
|
|
3
|
+
Version: 0.5.4
|
|
4
|
+
Summary: Identify European Portuguese grammatical structures and extract CEFR-linked features from text
|
|
5
|
+
Author-email: Soroosh Akef <s.akef@iwm-tuebingen.de>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/sorooshakef/SABER-Toolkit
|
|
8
|
+
Project-URL: Issues, https://github.com/sorooshakef/SABER-Toolkit/issues
|
|
9
|
+
Keywords: portuguese,cefr,nlp,grammatical structures,second language acquisition,readability
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Education
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Natural Language :: Portuguese
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
16
|
+
Requires-Python: <3.14,>=3.9
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: spacy==3.8.7
|
|
20
|
+
Requires-Dist: spacy_stanza==1.0.4
|
|
21
|
+
Requires-Dist: Unidecode==1.3.8
|
|
22
|
+
Requires-Dist: click>=8.0
|
|
23
|
+
Requires-Dist: torch==2.5.1
|
|
24
|
+
Requires-Dist: pandas>=1.5
|
|
25
|
+
Requires-Dist: tqdm>=4.64
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pyyaml>=6.0; extra == "dev"
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
|
|
30
|
+
# SABER
|
|
31
|
+
|
|
32
|
+
**SABER** (*Sistema de Análise e Busca de Estruturas Relevantes*) identifies
|
|
33
|
+
European Portuguese grammatical structures in text and reports the CEFR level
|
|
34
|
+
(A1–C2) at which each one is taught, following the
|
|
35
|
+
[**Referencial Camões**](https://www.instituto-camoes.pt/activity/centro-virtual/referencial-camoes-ple).
|
|
36
|
+
|
|
37
|
+
It gives you two things:
|
|
38
|
+
|
|
39
|
+
- **Spans** — every occurrence of every structure, with its text, its token and
|
|
40
|
+
character offsets, and its CEFR level. One row per occurrence.
|
|
41
|
+
- **Features** — per document, the raw frequency, the frequency per 100 tokens,
|
|
42
|
+
and the presence (0/1) of each structure. One row per document, ready to feed
|
|
43
|
+
into a model or a statistical analysis.
|
|
44
|
+
|
|
45
|
+
252 structures are covered, described in Portuguese and organised into 11 broad
|
|
46
|
+
groups (nouns, adjectives, verbs, adverbs, pronouns, determiners, quantifiers,
|
|
47
|
+
relations between constituents, sentence types, sentence polarity, relations
|
|
48
|
+
between sentences) and 31 finer categories. You can extract all of them, or
|
|
49
|
+
filter by group, category, CEFR level, or individual structure.
|
|
50
|
+
|
|
51
|
+
## Table of contents
|
|
52
|
+
|
|
53
|
+
- [Installation](#installation)
|
|
54
|
+
- [Quick start](#quick-start)
|
|
55
|
+
- [Input: strings, files, folders](#input-strings-files-folders)
|
|
56
|
+
- [Choosing which structures to extract](#choosing-which-structures-to-extract)
|
|
57
|
+
- [Output columns](#output-columns)
|
|
58
|
+
- [How it works](#how-it-works)
|
|
59
|
+
- [Performance](#performance)
|
|
60
|
+
- [Working on the matchers](#working-on-the-matchers)
|
|
61
|
+
- [Testing the matchers](#testing-the-matchers)
|
|
62
|
+
- [Licence and attribution](#licence-and-attribution)
|
|
63
|
+
|
|
64
|
+
## Installation
|
|
65
|
+
|
|
66
|
+
Python 3.9 or newer, but not above 3.13.
|
|
67
|
+
|
|
68
|
+
From a clone:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
git clone https://github.com/sorooshakef/SABER-Toolkit.git
|
|
72
|
+
cd SABER-Toolkit
|
|
73
|
+
python -m venv .venv
|
|
74
|
+
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
75
|
+
pip install -e .
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Then download the Portuguese models **once per machine** (~1 GB, mostly Stanza):
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
python -c "import saber; saber.download_models()"
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
> **Note on PyTorch.** `torch` is pinned to 2.5.1 because Stanza's model loading
|
|
85
|
+
> breaks with newer releases. If another package upgrades it, reinstall with
|
|
86
|
+
> `pip install torch==2.5.1`.
|
|
87
|
+
|
|
88
|
+
## Quick start
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
import saber
|
|
92
|
+
|
|
93
|
+
spans = saber.extract("Ele foi ao cinema no domingo com a minha irmã.", mode="spans")
|
|
94
|
+
print(spans[["structure", "level", "text", "description"]])
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
```
|
|
98
|
+
structure level text description
|
|
99
|
+
0 a3d1_12_A2 A2 foi Pretérito perfeito simples do indicativo - ver...
|
|
100
|
+
1 a6d1_6_A1 A1 ao cinema Artigo definido - concordância - em género e n...
|
|
101
|
+
2 a1d1_7_A2 A2 cinema Nomes masculinos terminados em "a"
|
|
102
|
+
3 a6d1_10_A2 A2 no Artigo definido - contração com prep...
|
|
103
|
+
4 a6d1_6_A1 A1 no domingo Artigo definido - concordância - em género e n...
|
|
104
|
+
5 a6d1_5_A1 A1 a minha Artigo definido - uso/valor - antes de determi...
|
|
105
|
+
6 a6d1_6_A1 A1 a minha irmã Artigo definido - concordância - em género e n...
|
|
106
|
+
7 a5d3_1_A1 A1 minha irmã Pronomes Possessivos - variação em pessoa, gén...
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
And the feature matrix for a folder of texts:
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
from pathlib import Path
|
|
113
|
+
|
|
114
|
+
features = saber.extract(Path("corpus/"), mode="features")
|
|
115
|
+
features.to_csv("features.csv", index=False)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
`extract` always returns a `pandas.DataFrame`. `saber.extract_spans(...)` and
|
|
119
|
+
`saber.extract_features(...)` are aliases for the two modes.
|
|
120
|
+
|
|
121
|
+
## Input: strings, files, folders
|
|
122
|
+
|
|
123
|
+
The first argument accepts text, a file, a folder, or a mix of them. The
|
|
124
|
+
distinction between "a string of text" and "a path" is made by **type**, so it
|
|
125
|
+
is never guessed:
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
saber.extract("Um texto qualquer.") # str -> raw text
|
|
129
|
+
saber.extract(Path("texto.txt")) # Path -> one file
|
|
130
|
+
saber.extract(Path("corpus/")) # Path -> every text file in the folder
|
|
131
|
+
saber.extract(["Primeiro texto.", "Segundo."]) # several texts
|
|
132
|
+
saber.extract([Path("a.txt"), Path("corpus/")]) # several paths
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
To read a plain string as a path, pass `source_type="path"`.
|
|
136
|
+
|
|
137
|
+
In a folder, every plain-text file is read: files with a `.txt` extension and
|
|
138
|
+
files with **no file-type extension at all**. A trailing number is not treated
|
|
139
|
+
as an extension, so corpus names like `A1.1` or `texto.2` are read too. Hidden
|
|
140
|
+
files (`.DS_Store`, dotfiles) and binary files are skipped. To pick up something
|
|
141
|
+
else — or to narrow the selection — pass an explicit glob:
|
|
142
|
+
|
|
143
|
+
```python
|
|
144
|
+
saber.extract(Path("corpus/"), pattern="*.txt") # only .txt
|
|
145
|
+
saber.extract(Path("corpus/"), pattern="*.tsv") # some other extension
|
|
146
|
+
saber.extract(Path("corpus/"), pattern="aluno_*") # a subset, by name
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Other folder options: `recursive=False`, `encoding="utf-8"` (files that are not
|
|
150
|
+
valid UTF-8 fall back to latin-1 with a warning).
|
|
151
|
+
|
|
152
|
+
Each document is named after its file, without the extension (`texto57.txt` and
|
|
153
|
+
`texto57` both become `texto57`, while `A1.1` keeps its number); inline strings
|
|
154
|
+
become `text`, or `text_1`, `text_2`, … when there are several.
|
|
155
|
+
|
|
156
|
+
## Choosing which structures to extract
|
|
157
|
+
|
|
158
|
+
Four independent filters. Values within one filter are OR-ed, and the filters
|
|
159
|
+
are AND-ed together. Matching ignores case and diacritics.
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
# A broad group: by key or by its Portuguese name
|
|
163
|
+
saber.extract(text, groups="a5")
|
|
164
|
+
saber.extract(text, groups="Pronomes") # the same 29 structures
|
|
165
|
+
|
|
166
|
+
# A finer category
|
|
167
|
+
saber.extract(text, categories="a5d2") # Pronomes > Demonstrativos
|
|
168
|
+
|
|
169
|
+
# A CEFR level, or several
|
|
170
|
+
saber.extract(text, levels="B1")
|
|
171
|
+
saber.extract(text, levels=["A1", "A2"])
|
|
172
|
+
|
|
173
|
+
# Individual structures, by id or by matcher name
|
|
174
|
+
saber.extract(text, structures=["a5d2_1_A2", "b5d2_6"])
|
|
175
|
+
|
|
176
|
+
# Combined: subjunctive-level pronoun structures only
|
|
177
|
+
saber.extract(text, groups="a5", levels="B2")
|
|
178
|
+
|
|
179
|
+
# Everything except a few structures
|
|
180
|
+
saber.extract(text, exclude=["a1d1_7_A2"])
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
To see what the valid values are:
|
|
184
|
+
|
|
185
|
+
```python
|
|
186
|
+
saber.list_structures() # all 252, as a DataFrame
|
|
187
|
+
saber.list_structures(groups="a7") # just the quantifiers
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
| structure | matcher | level | group | group_name | category | category_name | description |
|
|
191
|
+
|---|---|---|---|---|---|---|---|
|
|
192
|
+
| a5d2_1_A2 | a5d2_1 | A2 | a5 | Pronomes | a5d2 | Demonstrativos | Pronomes - Demonstrativos - contração com preposições |
|
|
193
|
+
|
|
194
|
+
An unknown filter value raises `ValueError` listing the valid options, and so
|
|
195
|
+
does a combination that selects nothing.
|
|
196
|
+
|
|
197
|
+
Structure ids encode the taxonomy, so you can also filter the output frame
|
|
198
|
+
directly: `a5d2_1_A2` → group `a5`, category `a5d2`, matcher `a5d2_1`, level
|
|
199
|
+
`A2`.
|
|
200
|
+
|
|
201
|
+
## Output columns
|
|
202
|
+
|
|
203
|
+
### `mode="spans"` — one row per occurrence
|
|
204
|
+
|
|
205
|
+
| Column | Meaning |
|
|
206
|
+
|---|---|
|
|
207
|
+
| `document`, `path` | Which text the match came from (`path` is empty for inline strings) |
|
|
208
|
+
| `structure` | Full structure id, e.g. `a5d2_1_A2` |
|
|
209
|
+
| `matcher` | Name of the matcher function, e.g. `a5d2_1` |
|
|
210
|
+
| `level` | CEFR level, `A1`–`C2` |
|
|
211
|
+
| `group`, `group_name` | Broad group key and Portuguese name |
|
|
212
|
+
| `category`, `category_name` | Category key and Portuguese name |
|
|
213
|
+
| `description` | Portuguese description of the structure |
|
|
214
|
+
| `text` | The matched substring, sliced out of the preprocessed text |
|
|
215
|
+
| `reconstructed_text` | The matcher's own rendering of the match |
|
|
216
|
+
| `token_start`, `token_end` | Token offsets, end-exclusive |
|
|
217
|
+
| `char_start`, `char_end` | Character offsets into the preprocessed text |
|
|
218
|
+
| `pipeline` | `stanza` or `spacy`, depending on which parse the matcher needs |
|
|
219
|
+
|
|
220
|
+
Rows are sorted by document and position. Structures overlap freely — a single
|
|
221
|
+
phrase commonly matches several — so occurrences are *not* mutually exclusive.
|
|
222
|
+
|
|
223
|
+
`text` is the authoritative one: it is cut from the text by character offset, so
|
|
224
|
+
`analysis.text[char_start:char_end] == text` always holds and contractions come
|
|
225
|
+
out as written (`ao`, `no`, `daquele`). `reconstructed_text` comes from the
|
|
226
|
+
matcher's own `reconstruct_text` helper and is normally identical, but it is
|
|
227
|
+
rebuilt from tokens rather than sliced, so keep to `text` when exactness matters.
|
|
228
|
+
|
|
229
|
+
To get offsets against your own untouched string, note that they refer to the
|
|
230
|
+
*preprocessed* text — see [How it works](#how-it-works). `saber.analyze()`
|
|
231
|
+
returns that text alongside the matches if you need it:
|
|
232
|
+
|
|
233
|
+
```python
|
|
234
|
+
analysis = saber.analyze("Ele foi ao cinema.", saber.all_structures())
|
|
235
|
+
analysis.text # the preprocessed text the offsets refer to
|
|
236
|
+
analysis.n_tokens # tokens excluding punctuation
|
|
237
|
+
analysis.matches # list of Match objects
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
### `mode="features"` — one row per document
|
|
241
|
+
|
|
242
|
+
| Column | Meaning |
|
|
243
|
+
|---|---|
|
|
244
|
+
| `document`, `path` | Which text the row describes |
|
|
245
|
+
| `n_tokens` | Tokens excluding whitespace and punctuation |
|
|
246
|
+
| `<structure>_count` | Raw number of occurrences |
|
|
247
|
+
| `<structure>_norm` | Occurrences per 100 tokens |
|
|
248
|
+
| `<structure>_present` | `1` if the structure occurs at all, else `0` |
|
|
249
|
+
|
|
250
|
+
Columns are emitted for **every** selected structure, including those that never
|
|
251
|
+
matched, so feature matrices line up across documents and across runs. Their
|
|
252
|
+
order follows the declaration order of `LABEL_TRANSLATIONS` in
|
|
253
|
+
`saber/label_translation.py`, which is curated taxonomically.
|
|
254
|
+
|
|
255
|
+
Selecting all 252 structures therefore gives `3 × 252 + 3 = 759` columns. Use
|
|
256
|
+
`features=` to keep only what you need:
|
|
257
|
+
|
|
258
|
+
```python
|
|
259
|
+
saber.extract(corpus, mode="features", features=["normalized", "presence"])
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
## How it works
|
|
263
|
+
|
|
264
|
+
1. **Preprocessing.** Dialogue dashes are stripped and whitespace collapsed. All
|
|
265
|
+
offsets refer to this preprocessed text, which `saber.analyze()` exposes as
|
|
266
|
+
`analysis.text`.
|
|
267
|
+
2. **Parsing.** The text is parsed twice: by
|
|
268
|
+
[spaCy-Stanza](https://github.com/explosion/spacy-stanza) (`tokenize`, `mwt`,
|
|
269
|
+
`pos`, `lemma`, `depparse`) for morphology and dependencies, and by spaCy's
|
|
270
|
+
`pt_core_news_md` for the handful of matchers that need named entities or
|
|
271
|
+
spaCy's own tokenisation. A matcher declares which parse it needs; you never
|
|
272
|
+
choose.
|
|
273
|
+
3. **Matching.** Each selected structure's matcher runs over the appropriate
|
|
274
|
+
parse using spaCy's `Matcher`, `PhraseMatcher` and `DependencyMatcher`, and
|
|
275
|
+
returns token spans.
|
|
276
|
+
4. **Offsets.** Token spans are converted to character offsets. Stanza expands
|
|
277
|
+
multi-word tokens (`no` → `em o`), which destroys spaCy's own character
|
|
278
|
+
offsets, so they are recovered from the underlying Stanza document — this is
|
|
279
|
+
why `text` reads `no` and not `em o`.
|
|
280
|
+
5. **Tabulating.** Occurrences become the spans table, or are counted and
|
|
281
|
+
normalised into the feature table.
|
|
282
|
+
|
|
283
|
+
A matcher that raises is reported as a warning and skipped, so one broken
|
|
284
|
+
pattern cannot abort a corpus run. Use `on_error="raise"` while debugging, or
|
|
285
|
+
`on_error="ignore"` to silence it.
|
|
286
|
+
|
|
287
|
+
## Performance
|
|
288
|
+
|
|
289
|
+
Expect **a few seconds per short text**, and add a few additional seconds of one-off model loading
|
|
290
|
+
to the first extraction in a process. Two things dominate: the Stanza parse, and
|
|
291
|
+
the fact that each matcher rebuilds its spaCy `Matcher` object on every call.
|
|
292
|
+
|
|
293
|
+
Practical consequences:
|
|
294
|
+
|
|
295
|
+
- Extract once for all the structures you might want, then filter the resulting
|
|
296
|
+
DataFrame — that is much cheaper than re-running with different filters.
|
|
297
|
+
- Restricting `groups`/`categories`/`levels` up front does cut the matching cost
|
|
298
|
+
roughly in proportion, but not the parsing cost.
|
|
299
|
+
- A progress bar over the documents appears whenever there is more than one
|
|
300
|
+
document and you are working in a terminal or a notebook. `progress=False`
|
|
301
|
+
turns it off, `progress=True` forces it on (useful when output is redirected).
|
|
302
|
+
- Progress is also logged per document, which is the better option for scripts
|
|
303
|
+
and log files. Turn it on with `logging.basicConfig(level=logging.INFO)`.
|
|
304
|
+
|
|
305
|
+
## Working on the matchers
|
|
306
|
+
|
|
307
|
+
A matcher function is named after its structure minus the CEFR suffix
|
|
308
|
+
(`a5d2_1` → `a5d2_1_A2`), takes a spaCy `Doc`, and returns a list of
|
|
309
|
+
`(label, text, token_start, token_end)` tuples. Setting
|
|
310
|
+
`<function>.REQUIRES_SPACY = True` after the definition routes it to the
|
|
311
|
+
`pt_core_news_md` parse instead of the Stanza one.
|
|
312
|
+
|
|
313
|
+
A structure is available only if it has **both** an entry in
|
|
314
|
+
`LABEL_TRANSLATIONS` and a matcher function, so commenting out a label in
|
|
315
|
+
`saber/label_translation.py` disables the structure.
|
|
316
|
+
`saber.missing_labels()` lists labels with no matcher.
|
|
317
|
+
|
|
318
|
+
## Testing the matchers
|
|
319
|
+
|
|
320
|
+
`tests/tests.yaml` holds positive and negative example sentences per matcher.
|
|
321
|
+
The harness measures precision and recall for each, both with all patterns
|
|
322
|
+
active and with only the first pattern registered:
|
|
323
|
+
|
|
324
|
+
```bash
|
|
325
|
+
pip install -e ".[dev]"
|
|
326
|
+
python tests/test_suite.py
|
|
327
|
+
```
|
|
328
|
+
|
|
329
|
+
Results are written to `tests/test_results.csv`; the committed copy is the
|
|
330
|
+
reference from the last full run.
|
|
331
|
+
|
|
332
|
+
## Licence and attribution
|
|
333
|
+
|
|
334
|
+
Released under the [MIT License](https://opensource.org/licenses/MIT) — see
|
|
335
|
+
[LICENSE](LICENSE). You may use, modify, and distribute this work, including
|
|
336
|
+
commercially, provided the copyright notice and licence text are retained.
|
|
337
|
+
|
|
338
|
+
The structure inventory and its CEFR mapping derive from the
|
|
339
|
+
[Referencial Camões PLE](https://www.instituto-camoes.pt/activity/centro-virtual/referencial-camoes-ple)
|
|
340
|
+
(Instituto Camões).
|
saber_pt-0.5.4/README.md
ADDED
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
# SABER
|
|
2
|
+
|
|
3
|
+
**SABER** (*Sistema de Análise e Busca de Estruturas Relevantes*) identifies
|
|
4
|
+
European Portuguese grammatical structures in text and reports the CEFR level
|
|
5
|
+
(A1–C2) at which each one is taught, following the
|
|
6
|
+
[**Referencial Camões**](https://www.instituto-camoes.pt/activity/centro-virtual/referencial-camoes-ple).
|
|
7
|
+
|
|
8
|
+
It gives you two things:
|
|
9
|
+
|
|
10
|
+
- **Spans** — every occurrence of every structure, with its text, its token and
|
|
11
|
+
character offsets, and its CEFR level. One row per occurrence.
|
|
12
|
+
- **Features** — per document, the raw frequency, the frequency per 100 tokens,
|
|
13
|
+
and the presence (0/1) of each structure. One row per document, ready to feed
|
|
14
|
+
into a model or a statistical analysis.
|
|
15
|
+
|
|
16
|
+
252 structures are covered, described in Portuguese and organised into 11 broad
|
|
17
|
+
groups (nouns, adjectives, verbs, adverbs, pronouns, determiners, quantifiers,
|
|
18
|
+
relations between constituents, sentence types, sentence polarity, relations
|
|
19
|
+
between sentences) and 31 finer categories. You can extract all of them, or
|
|
20
|
+
filter by group, category, CEFR level, or individual structure.
|
|
21
|
+
|
|
22
|
+
## Table of contents
|
|
23
|
+
|
|
24
|
+
- [Installation](#installation)
|
|
25
|
+
- [Quick start](#quick-start)
|
|
26
|
+
- [Input: strings, files, folders](#input-strings-files-folders)
|
|
27
|
+
- [Choosing which structures to extract](#choosing-which-structures-to-extract)
|
|
28
|
+
- [Output columns](#output-columns)
|
|
29
|
+
- [How it works](#how-it-works)
|
|
30
|
+
- [Performance](#performance)
|
|
31
|
+
- [Working on the matchers](#working-on-the-matchers)
|
|
32
|
+
- [Testing the matchers](#testing-the-matchers)
|
|
33
|
+
- [Licence and attribution](#licence-and-attribution)
|
|
34
|
+
|
|
35
|
+
## Installation
|
|
36
|
+
|
|
37
|
+
Python 3.9 or newer, but not above 3.13.
|
|
38
|
+
|
|
39
|
+
From a clone:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
git clone https://github.com/sorooshakef/SABER-Toolkit.git
|
|
43
|
+
cd SABER-Toolkit
|
|
44
|
+
python -m venv .venv
|
|
45
|
+
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
46
|
+
pip install -e .
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Then download the Portuguese models **once per machine** (~1 GB, mostly Stanza):
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
python -c "import saber; saber.download_models()"
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
> **Note on PyTorch.** `torch` is pinned to 2.5.1 because Stanza's model loading
|
|
56
|
+
> breaks with newer releases. If another package upgrades it, reinstall with
|
|
57
|
+
> `pip install torch==2.5.1`.
|
|
58
|
+
|
|
59
|
+
## Quick start
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
import saber
|
|
63
|
+
|
|
64
|
+
spans = saber.extract("Ele foi ao cinema no domingo com a minha irmã.", mode="spans")
|
|
65
|
+
print(spans[["structure", "level", "text", "description"]])
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
```
|
|
69
|
+
structure level text description
|
|
70
|
+
0 a3d1_12_A2 A2 foi Pretérito perfeito simples do indicativo - ver...
|
|
71
|
+
1 a6d1_6_A1 A1 ao cinema Artigo definido - concordância - em género e n...
|
|
72
|
+
2 a1d1_7_A2 A2 cinema Nomes masculinos terminados em "a"
|
|
73
|
+
3 a6d1_10_A2 A2 no Artigo definido - contração com prep...
|
|
74
|
+
4 a6d1_6_A1 A1 no domingo Artigo definido - concordância - em género e n...
|
|
75
|
+
5 a6d1_5_A1 A1 a minha Artigo definido - uso/valor - antes de determi...
|
|
76
|
+
6 a6d1_6_A1 A1 a minha irmã Artigo definido - concordância - em género e n...
|
|
77
|
+
7 a5d3_1_A1 A1 minha irmã Pronomes Possessivos - variação em pessoa, gén...
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
And the feature matrix for a folder of texts:
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from pathlib import Path
|
|
84
|
+
|
|
85
|
+
features = saber.extract(Path("corpus/"), mode="features")
|
|
86
|
+
features.to_csv("features.csv", index=False)
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
`extract` always returns a `pandas.DataFrame`. `saber.extract_spans(...)` and
|
|
90
|
+
`saber.extract_features(...)` are aliases for the two modes.
|
|
91
|
+
|
|
92
|
+
## Input: strings, files, folders
|
|
93
|
+
|
|
94
|
+
The first argument accepts text, a file, a folder, or a mix of them. The
|
|
95
|
+
distinction between "a string of text" and "a path" is made by **type**, so it
|
|
96
|
+
is never guessed:
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
saber.extract("Um texto qualquer.") # str -> raw text
|
|
100
|
+
saber.extract(Path("texto.txt")) # Path -> one file
|
|
101
|
+
saber.extract(Path("corpus/")) # Path -> every text file in the folder
|
|
102
|
+
saber.extract(["Primeiro texto.", "Segundo."]) # several texts
|
|
103
|
+
saber.extract([Path("a.txt"), Path("corpus/")]) # several paths
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
To read a plain string as a path, pass `source_type="path"`.
|
|
107
|
+
|
|
108
|
+
In a folder, every plain-text file is read: files with a `.txt` extension and
|
|
109
|
+
files with **no file-type extension at all**. A trailing number is not treated
|
|
110
|
+
as an extension, so corpus names like `A1.1` or `texto.2` are read too. Hidden
|
|
111
|
+
files (`.DS_Store`, dotfiles) and binary files are skipped. To pick up something
|
|
112
|
+
else — or to narrow the selection — pass an explicit glob:
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
saber.extract(Path("corpus/"), pattern="*.txt") # only .txt
|
|
116
|
+
saber.extract(Path("corpus/"), pattern="*.tsv") # some other extension
|
|
117
|
+
saber.extract(Path("corpus/"), pattern="aluno_*") # a subset, by name
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Other folder options: `recursive=False`, `encoding="utf-8"` (files that are not
|
|
121
|
+
valid UTF-8 fall back to latin-1 with a warning).
|
|
122
|
+
|
|
123
|
+
Each document is named after its file, without the extension (`texto57.txt` and
|
|
124
|
+
`texto57` both become `texto57`, while `A1.1` keeps its number); inline strings
|
|
125
|
+
become `text`, or `text_1`, `text_2`, … when there are several.
|
|
126
|
+
|
|
127
|
+
## Choosing which structures to extract
|
|
128
|
+
|
|
129
|
+
Four independent filters. Values within one filter are OR-ed, and the filters
|
|
130
|
+
are AND-ed together. Matching ignores case and diacritics.
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
# A broad group: by key or by its Portuguese name
|
|
134
|
+
saber.extract(text, groups="a5")
|
|
135
|
+
saber.extract(text, groups="Pronomes") # the same 29 structures
|
|
136
|
+
|
|
137
|
+
# A finer category
|
|
138
|
+
saber.extract(text, categories="a5d2") # Pronomes > Demonstrativos
|
|
139
|
+
|
|
140
|
+
# A CEFR level, or several
|
|
141
|
+
saber.extract(text, levels="B1")
|
|
142
|
+
saber.extract(text, levels=["A1", "A2"])
|
|
143
|
+
|
|
144
|
+
# Individual structures, by id or by matcher name
|
|
145
|
+
saber.extract(text, structures=["a5d2_1_A2", "b5d2_6"])
|
|
146
|
+
|
|
147
|
+
# Combined: subjunctive-level pronoun structures only
|
|
148
|
+
saber.extract(text, groups="a5", levels="B2")
|
|
149
|
+
|
|
150
|
+
# Everything except a few structures
|
|
151
|
+
saber.extract(text, exclude=["a1d1_7_A2"])
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
To see what the valid values are:
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
saber.list_structures() # all 252, as a DataFrame
|
|
158
|
+
saber.list_structures(groups="a7") # just the quantifiers
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
| structure | matcher | level | group | group_name | category | category_name | description |
|
|
162
|
+
|---|---|---|---|---|---|---|---|
|
|
163
|
+
| a5d2_1_A2 | a5d2_1 | A2 | a5 | Pronomes | a5d2 | Demonstrativos | Pronomes - Demonstrativos - contração com preposições |
|
|
164
|
+
|
|
165
|
+
An unknown filter value raises `ValueError` listing the valid options, and so
|
|
166
|
+
does a combination that selects nothing.
|
|
167
|
+
|
|
168
|
+
Structure ids encode the taxonomy, so you can also filter the output frame
|
|
169
|
+
directly: `a5d2_1_A2` → group `a5`, category `a5d2`, matcher `a5d2_1`, level
|
|
170
|
+
`A2`.
|
|
171
|
+
|
|
172
|
+
## Output columns
|
|
173
|
+
|
|
174
|
+
### `mode="spans"` — one row per occurrence
|
|
175
|
+
|
|
176
|
+
| Column | Meaning |
|
|
177
|
+
|---|---|
|
|
178
|
+
| `document`, `path` | Which text the match came from (`path` is empty for inline strings) |
|
|
179
|
+
| `structure` | Full structure id, e.g. `a5d2_1_A2` |
|
|
180
|
+
| `matcher` | Name of the matcher function, e.g. `a5d2_1` |
|
|
181
|
+
| `level` | CEFR level, `A1`–`C2` |
|
|
182
|
+
| `group`, `group_name` | Broad group key and Portuguese name |
|
|
183
|
+
| `category`, `category_name` | Category key and Portuguese name |
|
|
184
|
+
| `description` | Portuguese description of the structure |
|
|
185
|
+
| `text` | The matched substring, sliced out of the preprocessed text |
|
|
186
|
+
| `reconstructed_text` | The matcher's own rendering of the match |
|
|
187
|
+
| `token_start`, `token_end` | Token offsets, end-exclusive |
|
|
188
|
+
| `char_start`, `char_end` | Character offsets into the preprocessed text |
|
|
189
|
+
| `pipeline` | `stanza` or `spacy`, depending on which parse the matcher needs |
|
|
190
|
+
|
|
191
|
+
Rows are sorted by document and position. Structures overlap freely — a single
|
|
192
|
+
phrase commonly matches several — so occurrences are *not* mutually exclusive.
|
|
193
|
+
|
|
194
|
+
`text` is the authoritative one: it is cut from the text by character offset, so
|
|
195
|
+
`analysis.text[char_start:char_end] == text` always holds and contractions come
|
|
196
|
+
out as written (`ao`, `no`, `daquele`). `reconstructed_text` comes from the
|
|
197
|
+
matcher's own `reconstruct_text` helper and is normally identical, but it is
|
|
198
|
+
rebuilt from tokens rather than sliced, so keep to `text` when exactness matters.
|
|
199
|
+
|
|
200
|
+
To get offsets against your own untouched string, note that they refer to the
|
|
201
|
+
*preprocessed* text — see [How it works](#how-it-works). `saber.analyze()`
|
|
202
|
+
returns that text alongside the matches if you need it:
|
|
203
|
+
|
|
204
|
+
```python
|
|
205
|
+
analysis = saber.analyze("Ele foi ao cinema.", saber.all_structures())
|
|
206
|
+
analysis.text # the preprocessed text the offsets refer to
|
|
207
|
+
analysis.n_tokens # tokens excluding punctuation
|
|
208
|
+
analysis.matches # list of Match objects
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
### `mode="features"` — one row per document
|
|
212
|
+
|
|
213
|
+
| Column | Meaning |
|
|
214
|
+
|---|---|
|
|
215
|
+
| `document`, `path` | Which text the row describes |
|
|
216
|
+
| `n_tokens` | Tokens excluding whitespace and punctuation |
|
|
217
|
+
| `<structure>_count` | Raw number of occurrences |
|
|
218
|
+
| `<structure>_norm` | Occurrences per 100 tokens |
|
|
219
|
+
| `<structure>_present` | `1` if the structure occurs at all, else `0` |
|
|
220
|
+
|
|
221
|
+
Columns are emitted for **every** selected structure, including those that never
|
|
222
|
+
matched, so feature matrices line up across documents and across runs. Their
|
|
223
|
+
order follows the declaration order of `LABEL_TRANSLATIONS` in
|
|
224
|
+
`saber/label_translation.py`, which is curated taxonomically.
|
|
225
|
+
|
|
226
|
+
Selecting all 252 structures therefore gives `3 × 252 + 3 = 759` columns. Use
|
|
227
|
+
`features=` to keep only what you need:
|
|
228
|
+
|
|
229
|
+
```python
|
|
230
|
+
saber.extract(corpus, mode="features", features=["normalized", "presence"])
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
## How it works
|
|
234
|
+
|
|
235
|
+
1. **Preprocessing.** Dialogue dashes are stripped and whitespace collapsed. All
|
|
236
|
+
offsets refer to this preprocessed text, which `saber.analyze()` exposes as
|
|
237
|
+
`analysis.text`.
|
|
238
|
+
2. **Parsing.** The text is parsed twice: by
|
|
239
|
+
[spaCy-Stanza](https://github.com/explosion/spacy-stanza) (`tokenize`, `mwt`,
|
|
240
|
+
`pos`, `lemma`, `depparse`) for morphology and dependencies, and by spaCy's
|
|
241
|
+
`pt_core_news_md` for the handful of matchers that need named entities or
|
|
242
|
+
spaCy's own tokenisation. A matcher declares which parse it needs; you never
|
|
243
|
+
choose.
|
|
244
|
+
3. **Matching.** Each selected structure's matcher runs over the appropriate
|
|
245
|
+
parse using spaCy's `Matcher`, `PhraseMatcher` and `DependencyMatcher`, and
|
|
246
|
+
returns token spans.
|
|
247
|
+
4. **Offsets.** Token spans are converted to character offsets. Stanza expands
|
|
248
|
+
multi-word tokens (`no` → `em o`), which destroys spaCy's own character
|
|
249
|
+
offsets, so they are recovered from the underlying Stanza document — this is
|
|
250
|
+
why `text` reads `no` and not `em o`.
|
|
251
|
+
5. **Tabulating.** Occurrences become the spans table, or are counted and
|
|
252
|
+
normalised into the feature table.
|
|
253
|
+
|
|
254
|
+
A matcher that raises is reported as a warning and skipped, so one broken
|
|
255
|
+
pattern cannot abort a corpus run. Use `on_error="raise"` while debugging, or
|
|
256
|
+
`on_error="ignore"` to silence it.
|
|
257
|
+
|
|
258
|
+
## Performance
|
|
259
|
+
|
|
260
|
+
Expect **a few seconds per short text**, and add a few additional seconds of one-off model loading
|
|
261
|
+
to the first extraction in a process. Two things dominate: the Stanza parse, and
|
|
262
|
+
the fact that each matcher rebuilds its spaCy `Matcher` object on every call.
|
|
263
|
+
|
|
264
|
+
Practical consequences:
|
|
265
|
+
|
|
266
|
+
- Extract once for all the structures you might want, then filter the resulting
|
|
267
|
+
DataFrame — that is much cheaper than re-running with different filters.
|
|
268
|
+
- Restricting `groups`/`categories`/`levels` up front does cut the matching cost
|
|
269
|
+
roughly in proportion, but not the parsing cost.
|
|
270
|
+
- A progress bar over the documents appears whenever there is more than one
|
|
271
|
+
document and you are working in a terminal or a notebook. `progress=False`
|
|
272
|
+
turns it off, `progress=True` forces it on (useful when output is redirected).
|
|
273
|
+
- Progress is also logged per document, which is the better option for scripts
|
|
274
|
+
and log files. Turn it on with `logging.basicConfig(level=logging.INFO)`.
|
|
275
|
+
|
|
276
|
+
## Working on the matchers
|
|
277
|
+
|
|
278
|
+
A matcher function is named after its structure minus the CEFR suffix
|
|
279
|
+
(`a5d2_1` → `a5d2_1_A2`), takes a spaCy `Doc`, and returns a list of
|
|
280
|
+
`(label, text, token_start, token_end)` tuples. Setting
|
|
281
|
+
`<function>.REQUIRES_SPACY = True` after the definition routes it to the
|
|
282
|
+
`pt_core_news_md` parse instead of the Stanza one.
|
|
283
|
+
|
|
284
|
+
A structure is available only if it has **both** an entry in
|
|
285
|
+
`LABEL_TRANSLATIONS` and a matcher function, so commenting out a label in
|
|
286
|
+
`saber/label_translation.py` disables the structure.
|
|
287
|
+
`saber.missing_labels()` lists labels with no matcher.
|
|
288
|
+
|
|
289
|
+
## Testing the matchers
|
|
290
|
+
|
|
291
|
+
`tests/tests.yaml` holds positive and negative example sentences per matcher.
|
|
292
|
+
The harness measures precision and recall for each, both with all patterns
|
|
293
|
+
active and with only the first pattern registered:
|
|
294
|
+
|
|
295
|
+
```bash
|
|
296
|
+
pip install -e ".[dev]"
|
|
297
|
+
python tests/test_suite.py
|
|
298
|
+
```
|
|
299
|
+
|
|
300
|
+
Results are written to `tests/test_results.csv`; the committed copy is the
|
|
301
|
+
reference from the last full run.
|
|
302
|
+
|
|
303
|
+
## Licence and attribution
|
|
304
|
+
|
|
305
|
+
Released under the [MIT License](https://opensource.org/licenses/MIT) — see
|
|
306
|
+
[LICENSE](LICENSE). You may use, modify, and distribute this work, including
|
|
307
|
+
commercially, provided the copyright notice and licence text are retained.
|
|
308
|
+
|
|
309
|
+
The structure inventory and its CEFR mapping derive from the
|
|
310
|
+
[Referencial Camões PLE](https://www.instituto-camoes.pt/activity/centro-virtual/referencial-camoes-ple)
|
|
311
|
+
(Instituto Camões).
|