saber-pt 0.5.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. saber_pt-0.5.4/LICENSE +21 -0
  2. saber_pt-0.5.4/PKG-INFO +340 -0
  3. saber_pt-0.5.4/README.md +311 -0
  4. saber_pt-0.5.4/pyproject.toml +58 -0
  5. saber_pt-0.5.4/saber/__init__.py +58 -0
  6. saber_pt-0.5.4/saber/_compat.py +31 -0
  7. saber_pt-0.5.4/saber/extraction.py +357 -0
  8. saber_pt-0.5.4/saber/label_translation.py +387 -0
  9. saber_pt-0.5.4/saber/matchers/a1_nouns.py +256 -0
  10. saber_pt-0.5.4/saber/matchers/a2_1_adjectives.py +349 -0
  11. saber_pt-0.5.4/saber/matchers/a3_1_verbs_tense_mood.py +2248 -0
  12. saber_pt-0.5.4/saber/matchers/a3_2_compound_verbs.py +1816 -0
  13. saber_pt-0.5.4/saber/matchers/a4_1_adverbs.py +46 -0
  14. saber_pt-0.5.4/saber/matchers/a5_1_personal_pronouns.py +299 -0
  15. saber_pt-0.5.4/saber/matchers/a5_2_demonstratives.py +235 -0
  16. saber_pt-0.5.4/saber/matchers/a5_3_possessives.py +172 -0
  17. saber_pt-0.5.4/saber/matchers/a5_4_indefinite_pronouns.py +96 -0
  18. saber_pt-0.5.4/saber/matchers/a5_5_interrogatives.py +110 -0
  19. saber_pt-0.5.4/saber/matchers/a5_6_relative_pronouns.py +159 -0
  20. saber_pt-0.5.4/saber/matchers/a6_1_articles.py +574 -0
  21. saber_pt-0.5.4/saber/matchers/a6_2_demonstratives.py +269 -0
  22. saber_pt-0.5.4/saber/matchers/a6_3_possessives.py +93 -0
  23. saber_pt-0.5.4/saber/matchers/a6_4_indefinite_determiners.py +70 -0
  24. saber_pt-0.5.4/saber/matchers/a6_6_relative_determiners.py +41 -0
  25. saber_pt-0.5.4/saber/matchers/a7_1_existentials.py +73 -0
  26. saber_pt-0.5.4/saber/matchers/a7_2_universals.py +153 -0
  27. saber_pt-0.5.4/saber/matchers/a7_3_numerals.py +169 -0
  28. saber_pt-0.5.4/saber/matchers/a7_4_interrogatives.py +26 -0
  29. saber_pt-0.5.4/saber/matchers/a7_5_relative_quantifiers.py +21 -0
  30. saber_pt-0.5.4/saber/matchers/b2_1_syntactic_functions.py +1740 -0
  31. saber_pt-0.5.4/saber/matchers/b2_2_order_of_constituents.py +231 -0
  32. saber_pt-0.5.4/saber/matchers/b2_3_agreement.py +808 -0
  33. saber_pt-0.5.4/saber/matchers/b3_2_interrogatives.py +348 -0
  34. saber_pt-0.5.4/saber/matchers/b3_3_exclamatives.py +281 -0
  35. saber_pt-0.5.4/saber/matchers/b3_4_imperatives.py +158 -0
  36. saber_pt-0.5.4/saber/matchers/b3_5_active_passive_sentences.py +359 -0
  37. saber_pt-0.5.4/saber/matchers/b4_1_sentence_polarity.py +27 -0
  38. saber_pt-0.5.4/saber/matchers/b5_1_coordination.py +525 -0
  39. saber_pt-0.5.4/saber/matchers/b5_2_subordination.py +2496 -0
  40. saber_pt-0.5.4/saber/models.py +61 -0
  41. saber_pt-0.5.4/saber/nlp.py +189 -0
  42. saber_pt-0.5.4/saber/process_and_display.py +19 -0
  43. saber_pt-0.5.4/saber/progress.py +103 -0
  44. saber_pt-0.5.4/saber/registry.py +360 -0
  45. saber_pt-0.5.4/saber/sources.py +273 -0
  46. saber_pt-0.5.4/saber/text_reconstruction.py +149 -0
  47. saber_pt-0.5.4/saber_pt.egg-info/PKG-INFO +340 -0
  48. saber_pt-0.5.4/saber_pt.egg-info/SOURCES.txt +51 -0
  49. saber_pt-0.5.4/saber_pt.egg-info/dependency_links.txt +1 -0
  50. saber_pt-0.5.4/saber_pt.egg-info/requires.txt +10 -0
  51. saber_pt-0.5.4/saber_pt.egg-info/top_level.txt +1 -0
  52. saber_pt-0.5.4/setup.cfg +4 -0
  53. saber_pt-0.5.4/tests/test_suite.py +250 -0
saber_pt-0.5.4/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Soroosh Akef
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,340 @@
1
+ Metadata-Version: 2.4
2
+ Name: saber-pt
3
+ Version: 0.5.4
4
+ Summary: Identify European Portuguese grammatical structures and extract CEFR-linked features from text
5
+ Author-email: Soroosh Akef <s.akef@iwm-tuebingen.de>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/sorooshakef/SABER-Toolkit
8
+ Project-URL: Issues, https://github.com/sorooshakef/SABER-Toolkit/issues
9
+ Keywords: portuguese,cefr,nlp,grammatical structures,second language acquisition,readability
10
+ Classifier: Development Status :: 4 - Beta
11
+ Classifier: Intended Audience :: Education
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Natural Language :: Portuguese
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Topic :: Text Processing :: Linguistic
16
+ Requires-Python: <3.14,>=3.9
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Requires-Dist: spacy==3.8.7
20
+ Requires-Dist: spacy_stanza==1.0.4
21
+ Requires-Dist: Unidecode==1.3.8
22
+ Requires-Dist: click>=8.0
23
+ Requires-Dist: torch==2.5.1
24
+ Requires-Dist: pandas>=1.5
25
+ Requires-Dist: tqdm>=4.64
26
+ Provides-Extra: dev
27
+ Requires-Dist: pyyaml>=6.0; extra == "dev"
28
+ Dynamic: license-file
29
+
30
+ # SABER
31
+
32
+ **SABER** (*Sistema de Análise e Busca de Estruturas Relevantes*) identifies
33
+ European Portuguese grammatical structures in text and reports the CEFR level
34
+ (A1–C2) at which each one is taught, following the
35
+ [**Referencial Camões**](https://www.instituto-camoes.pt/activity/centro-virtual/referencial-camoes-ple).
36
+
37
+ It gives you two things:
38
+
39
+ - **Spans** — every occurrence of every structure, with its text, its token and
40
+ character offsets, and its CEFR level. One row per occurrence.
41
+ - **Features** — per document, the raw frequency, the frequency per 100 tokens,
42
+ and the presence (0/1) of each structure. One row per document, ready to feed
43
+ into a model or a statistical analysis.
44
+
45
+ 252 structures are covered, described in Portuguese and organised into 11 broad
46
+ groups (nouns, adjectives, verbs, adverbs, pronouns, determiners, quantifiers,
47
+ relations between constituents, sentence types, sentence polarity, relations
48
+ between sentences) and 31 finer categories. You can extract all of them, or
49
+ filter by group, category, CEFR level, or individual structure.
50
+
51
+ ## Table of contents
52
+
53
+ - [Installation](#installation)
54
+ - [Quick start](#quick-start)
55
+ - [Input: strings, files, folders](#input-strings-files-folders)
56
+ - [Choosing which structures to extract](#choosing-which-structures-to-extract)
57
+ - [Output columns](#output-columns)
58
+ - [How it works](#how-it-works)
59
+ - [Performance](#performance)
60
+ - [Working on the matchers](#working-on-the-matchers)
61
+ - [Testing the matchers](#testing-the-matchers)
62
+ - [Licence and attribution](#licence-and-attribution)
63
+
64
+ ## Installation
65
+
66
+ Python 3.9 or newer, but not above 3.13.
67
+
68
+ From a clone:
69
+
70
+ ```bash
71
+ git clone https://github.com/sorooshakef/SABER-Toolkit.git
72
+ cd SABER-Toolkit
73
+ python -m venv .venv
74
+ source .venv/bin/activate # Windows: .venv\Scripts\activate
75
+ pip install -e .
76
+ ```
77
+
78
+ Then download the Portuguese models **once per machine** (~1 GB, mostly Stanza):
79
+
80
+ ```bash
81
+ python -c "import saber; saber.download_models()"
82
+ ```
83
+
84
+ > **Note on PyTorch.** `torch` is pinned to 2.5.1 because Stanza's model loading
85
+ > breaks with newer releases. If another package upgrades it, reinstall with
86
+ > `pip install torch==2.5.1`.
87
+
88
+ ## Quick start
89
+
90
+ ```python
91
+ import saber
92
+
93
+ spans = saber.extract("Ele foi ao cinema no domingo com a minha irmã.", mode="spans")
94
+ print(spans[["structure", "level", "text", "description"]])
95
+ ```
96
+
97
+ ```
98
+ structure level text description
99
+ 0 a3d1_12_A2 A2 foi Pretérito perfeito simples do indicativo - ver...
100
+ 1 a6d1_6_A1 A1 ao cinema Artigo definido - concordância - em género e n...
101
+ 2 a1d1_7_A2 A2 cinema Nomes masculinos terminados em "a"
102
+ 3 a6d1_10_A2 A2 no Artigo definido - contração com prep...
103
+ 4 a6d1_6_A1 A1 no domingo Artigo definido - concordância - em género e n...
104
+ 5 a6d1_5_A1 A1 a minha Artigo definido - uso/valor - antes de determi...
105
+ 6 a6d1_6_A1 A1 a minha irmã Artigo definido - concordância - em género e n...
106
+ 7 a5d3_1_A1 A1 minha irmã Pronomes Possessivos - variação em pessoa, gén...
107
+ ```
108
+
109
+ And the feature matrix for a folder of texts:
110
+
111
+ ```python
112
+ from pathlib import Path
113
+
114
+ features = saber.extract(Path("corpus/"), mode="features")
115
+ features.to_csv("features.csv", index=False)
116
+ ```
117
+
118
+ `extract` always returns a `pandas.DataFrame`. `saber.extract_spans(...)` and
119
+ `saber.extract_features(...)` are aliases for the two modes.
120
+
121
+ ## Input: strings, files, folders
122
+
123
+ The first argument accepts text, a file, a folder, or a mix of them. The
124
+ distinction between "a string of text" and "a path" is made by **type**, so it
125
+ is never guessed:
126
+
127
+ ```python
128
+ saber.extract("Um texto qualquer.") # str -> raw text
129
+ saber.extract(Path("texto.txt")) # Path -> one file
130
+ saber.extract(Path("corpus/")) # Path -> every text file in the folder
131
+ saber.extract(["Primeiro texto.", "Segundo."]) # several texts
132
+ saber.extract([Path("a.txt"), Path("corpus/")]) # several paths
133
+ ```
134
+
135
+ To read a plain string as a path, pass `source_type="path"`.
136
+
137
+ In a folder, every plain-text file is read: files with a `.txt` extension and
138
+ files with **no file-type extension at all**. A trailing number is not treated
139
+ as an extension, so corpus names like `A1.1` or `texto.2` are read too. Hidden
140
+ files (`.DS_Store`, dotfiles) and binary files are skipped. To pick up something
141
+ else — or to narrow the selection — pass an explicit glob:
142
+
143
+ ```python
144
+ saber.extract(Path("corpus/"), pattern="*.txt") # only .txt
145
+ saber.extract(Path("corpus/"), pattern="*.tsv") # some other extension
146
+ saber.extract(Path("corpus/"), pattern="aluno_*") # a subset, by name
147
+ ```
148
+
149
+ Other folder options: `recursive=False`, `encoding="utf-8"` (files that are not
150
+ valid UTF-8 fall back to latin-1 with a warning).
151
+
152
+ Each document is named after its file, without the extension (`texto57.txt` and
153
+ `texto57` both become `texto57`, while `A1.1` keeps its number); inline strings
154
+ become `text`, or `text_1`, `text_2`, … when there are several.
155
+
156
+ ## Choosing which structures to extract
157
+
158
+ Four independent filters. Values within one filter are OR-ed, and the filters
159
+ are AND-ed together. Matching ignores case and diacritics.
160
+
161
+ ```python
162
+ # A broad group: by key or by its Portuguese name
163
+ saber.extract(text, groups="a5")
164
+ saber.extract(text, groups="Pronomes") # the same 29 structures
165
+
166
+ # A finer category
167
+ saber.extract(text, categories="a5d2") # Pronomes > Demonstrativos
168
+
169
+ # A CEFR level, or several
170
+ saber.extract(text, levels="B1")
171
+ saber.extract(text, levels=["A1", "A2"])
172
+
173
+ # Individual structures, by id or by matcher name
174
+ saber.extract(text, structures=["a5d2_1_A2", "b5d2_6"])
175
+
176
+ # Combined: subjunctive-level pronoun structures only
177
+ saber.extract(text, groups="a5", levels="B2")
178
+
179
+ # Everything except a few structures
180
+ saber.extract(text, exclude=["a1d1_7_A2"])
181
+ ```
182
+
183
+ To see what the valid values are:
184
+
185
+ ```python
186
+ saber.list_structures() # all 252, as a DataFrame
187
+ saber.list_structures(groups="a7") # just the quantifiers
188
+ ```
189
+
190
+ | structure | matcher | level | group | group_name | category | category_name | description |
191
+ |---|---|---|---|---|---|---|---|
192
+ | a5d2_1_A2 | a5d2_1 | A2 | a5 | Pronomes | a5d2 | Demonstrativos | Pronomes - Demonstrativos - contração com preposições |
193
+
194
+ An unknown filter value raises `ValueError` listing the valid options, and so
195
+ does a combination that selects nothing.
196
+
197
+ Structure ids encode the taxonomy, so you can also filter the output frame
198
+ directly: `a5d2_1_A2` → group `a5`, category `a5d2`, matcher `a5d2_1`, level
199
+ `A2`.
200
+
201
+ ## Output columns
202
+
203
+ ### `mode="spans"` — one row per occurrence
204
+
205
+ | Column | Meaning |
206
+ |---|---|
207
+ | `document`, `path` | Which text the match came from (`path` is empty for inline strings) |
208
+ | `structure` | Full structure id, e.g. `a5d2_1_A2` |
209
+ | `matcher` | Name of the matcher function, e.g. `a5d2_1` |
210
+ | `level` | CEFR level, `A1`–`C2` |
211
+ | `group`, `group_name` | Broad group key and Portuguese name |
212
+ | `category`, `category_name` | Category key and Portuguese name |
213
+ | `description` | Portuguese description of the structure |
214
+ | `text` | The matched substring, sliced out of the preprocessed text |
215
+ | `reconstructed_text` | The matcher's own rendering of the match |
216
+ | `token_start`, `token_end` | Token offsets, end-exclusive |
217
+ | `char_start`, `char_end` | Character offsets into the preprocessed text |
218
+ | `pipeline` | `stanza` or `spacy`, depending on which parse the matcher needs |
219
+
220
+ Rows are sorted by document and position. Structures overlap freely — a single
221
+ phrase commonly matches several — so occurrences are *not* mutually exclusive.
222
+
223
+ `text` is the authoritative one: it is cut from the text by character offset, so
224
+ `analysis.text[char_start:char_end] == text` always holds and contractions come
225
+ out as written (`ao`, `no`, `daquele`). `reconstructed_text` comes from the
226
+ matcher's own `reconstruct_text` helper and is normally identical, but it is
227
+ rebuilt from tokens rather than sliced, so keep to `text` when exactness matters.
228
+
229
+ To get offsets against your own untouched string, note that they refer to the
230
+ *preprocessed* text — see [How it works](#how-it-works). `saber.analyze()`
231
+ returns that text alongside the matches if you need it:
232
+
233
+ ```python
234
+ analysis = saber.analyze("Ele foi ao cinema.", saber.all_structures())
235
+ analysis.text # the preprocessed text the offsets refer to
236
+ analysis.n_tokens # tokens excluding punctuation
237
+ analysis.matches # list of Match objects
238
+ ```
239
+
240
+ ### `mode="features"` — one row per document
241
+
242
+ | Column | Meaning |
243
+ |---|---|
244
+ | `document`, `path` | Which text the row describes |
245
+ | `n_tokens` | Tokens excluding whitespace and punctuation |
246
+ | `<structure>_count` | Raw number of occurrences |
247
+ | `<structure>_norm` | Occurrences per 100 tokens |
248
+ | `<structure>_present` | `1` if the structure occurs at all, else `0` |
249
+
250
+ Columns are emitted for **every** selected structure, including those that never
251
+ matched, so feature matrices line up across documents and across runs. Their
252
+ order follows the declaration order of `LABEL_TRANSLATIONS` in
253
+ `saber/label_translation.py`, which is curated taxonomically.
254
+
255
+ Selecting all 252 structures therefore gives `3 × 252 + 3 = 759` columns. Use
256
+ `features=` to keep only what you need:
257
+
258
+ ```python
259
+ saber.extract(corpus, mode="features", features=["normalized", "presence"])
260
+ ```
261
+
262
+ ## How it works
263
+
264
+ 1. **Preprocessing.** Dialogue dashes are stripped and whitespace collapsed. All
265
+ offsets refer to this preprocessed text, which `saber.analyze()` exposes as
266
+ `analysis.text`.
267
+ 2. **Parsing.** The text is parsed twice: by
268
+ [spaCy-Stanza](https://github.com/explosion/spacy-stanza) (`tokenize`, `mwt`,
269
+ `pos`, `lemma`, `depparse`) for morphology and dependencies, and by spaCy's
270
+ `pt_core_news_md` for the handful of matchers that need named entities or
271
+ spaCy's own tokenisation. A matcher declares which parse it needs; you never
272
+ choose.
273
+ 3. **Matching.** Each selected structure's matcher runs over the appropriate
274
+ parse using spaCy's `Matcher`, `PhraseMatcher` and `DependencyMatcher`, and
275
+ returns token spans.
276
+ 4. **Offsets.** Token spans are converted to character offsets. Stanza expands
277
+ multi-word tokens (`no` → `em o`), which destroys spaCy's own character
278
+ offsets, so they are recovered from the underlying Stanza document — this is
279
+ why `text` reads `no` and not `em o`.
280
+ 5. **Tabulating.** Occurrences become the spans table, or are counted and
281
+ normalised into the feature table.
282
+
283
+ A matcher that raises is reported as a warning and skipped, so one broken
284
+ pattern cannot abort a corpus run. Use `on_error="raise"` while debugging, or
285
+ `on_error="ignore"` to silence it.
286
+
287
+ ## Performance
288
+
289
+ Expect **a few seconds per short text**, and add a few additional seconds of one-off model loading
290
+ to the first extraction in a process. Two things dominate: the Stanza parse, and
291
+ the fact that each matcher rebuilds its spaCy `Matcher` object on every call.
292
+
293
+ Practical consequences:
294
+
295
+ - Extract once for all the structures you might want, then filter the resulting
296
+ DataFrame — that is much cheaper than re-running with different filters.
297
+ - Restricting `groups`/`categories`/`levels` up front does cut the matching cost
298
+ roughly in proportion, but not the parsing cost.
299
+ - A progress bar over the documents appears whenever there is more than one
300
+ document and you are working in a terminal or a notebook. `progress=False`
301
+ turns it off, `progress=True` forces it on (useful when output is redirected).
302
+ - Progress is also logged per document, which is the better option for scripts
303
+ and log files. Turn it on with `logging.basicConfig(level=logging.INFO)`.
304
+
305
+ ## Working on the matchers
306
+
307
+ A matcher function is named after its structure minus the CEFR suffix
308
+ (`a5d2_1` → `a5d2_1_A2`), takes a spaCy `Doc`, and returns a list of
309
+ `(label, text, token_start, token_end)` tuples. Setting
310
+ `<function>.REQUIRES_SPACY = True` after the definition routes it to the
311
+ `pt_core_news_md` parse instead of the Stanza one.
312
+
313
+ A structure is available only if it has **both** an entry in
314
+ `LABEL_TRANSLATIONS` and a matcher function, so commenting out a label in
315
+ `saber/label_translation.py` disables the structure.
316
+ `saber.missing_labels()` lists labels with no matcher.
317
+
318
+ ## Testing the matchers
319
+
320
+ `tests/tests.yaml` holds positive and negative example sentences per matcher.
321
+ The harness measures precision and recall for each, both with all patterns
322
+ active and with only the first pattern registered:
323
+
324
+ ```bash
325
+ pip install -e ".[dev]"
326
+ python tests/test_suite.py
327
+ ```
328
+
329
+ Results are written to `tests/test_results.csv`; the committed copy is the
330
+ reference from the last full run.
331
+
332
+ ## Licence and attribution
333
+
334
+ Released under the [MIT License](https://opensource.org/licenses/MIT) — see
335
+ [LICENSE](LICENSE). You may use, modify, and distribute this work, including
336
+ commercially, provided the copyright notice and licence text are retained.
337
+
338
+ The structure inventory and its CEFR mapping derive from the
339
+ [Referencial Camões PLE](https://www.instituto-camoes.pt/activity/centro-virtual/referencial-camoes-ple)
340
+ (Instituto Camões).
@@ -0,0 +1,311 @@
1
+ # SABER
2
+
3
+ **SABER** (*Sistema de Análise e Busca de Estruturas Relevantes*) identifies
4
+ European Portuguese grammatical structures in text and reports the CEFR level
5
+ (A1–C2) at which each one is taught, following the
6
+ [**Referencial Camões**](https://www.instituto-camoes.pt/activity/centro-virtual/referencial-camoes-ple).
7
+
8
+ It gives you two things:
9
+
10
+ - **Spans** — every occurrence of every structure, with its text, its token and
11
+ character offsets, and its CEFR level. One row per occurrence.
12
+ - **Features** — per document, the raw frequency, the frequency per 100 tokens,
13
+ and the presence (0/1) of each structure. One row per document, ready to feed
14
+ into a model or a statistical analysis.
15
+
16
+ 252 structures are covered, described in Portuguese and organised into 11 broad
17
+ groups (nouns, adjectives, verbs, adverbs, pronouns, determiners, quantifiers,
18
+ relations between constituents, sentence types, sentence polarity, relations
19
+ between sentences) and 31 finer categories. You can extract all of them, or
20
+ filter by group, category, CEFR level, or individual structure.
21
+
22
+ ## Table of contents
23
+
24
+ - [Installation](#installation)
25
+ - [Quick start](#quick-start)
26
+ - [Input: strings, files, folders](#input-strings-files-folders)
27
+ - [Choosing which structures to extract](#choosing-which-structures-to-extract)
28
+ - [Output columns](#output-columns)
29
+ - [How it works](#how-it-works)
30
+ - [Performance](#performance)
31
+ - [Working on the matchers](#working-on-the-matchers)
32
+ - [Testing the matchers](#testing-the-matchers)
33
+ - [Licence and attribution](#licence-and-attribution)
34
+
35
+ ## Installation
36
+
37
+ Python 3.9 or newer, but not above 3.13.
38
+
39
+ From a clone:
40
+
41
+ ```bash
42
+ git clone https://github.com/sorooshakef/SABER-Toolkit.git
43
+ cd SABER-Toolkit
44
+ python -m venv .venv
45
+ source .venv/bin/activate # Windows: .venv\Scripts\activate
46
+ pip install -e .
47
+ ```
48
+
49
+ Then download the Portuguese models **once per machine** (~1 GB, mostly Stanza):
50
+
51
+ ```bash
52
+ python -c "import saber; saber.download_models()"
53
+ ```
54
+
55
+ > **Note on PyTorch.** `torch` is pinned to 2.5.1 because Stanza's model loading
56
+ > breaks with newer releases. If another package upgrades it, reinstall with
57
+ > `pip install torch==2.5.1`.
58
+
59
+ ## Quick start
60
+
61
+ ```python
62
+ import saber
63
+
64
+ spans = saber.extract("Ele foi ao cinema no domingo com a minha irmã.", mode="spans")
65
+ print(spans[["structure", "level", "text", "description"]])
66
+ ```
67
+
68
+ ```
69
+ structure level text description
70
+ 0 a3d1_12_A2 A2 foi Pretérito perfeito simples do indicativo - ver...
71
+ 1 a6d1_6_A1 A1 ao cinema Artigo definido - concordância - em género e n...
72
+ 2 a1d1_7_A2 A2 cinema Nomes masculinos terminados em "a"
73
+ 3 a6d1_10_A2 A2 no Artigo definido - contração com prep...
74
+ 4 a6d1_6_A1 A1 no domingo Artigo definido - concordância - em género e n...
75
+ 5 a6d1_5_A1 A1 a minha Artigo definido - uso/valor - antes de determi...
76
+ 6 a6d1_6_A1 A1 a minha irmã Artigo definido - concordância - em género e n...
77
+ 7 a5d3_1_A1 A1 minha irmã Pronomes Possessivos - variação em pessoa, gén...
78
+ ```
79
+
80
+ And the feature matrix for a folder of texts:
81
+
82
+ ```python
83
+ from pathlib import Path
84
+
85
+ features = saber.extract(Path("corpus/"), mode="features")
86
+ features.to_csv("features.csv", index=False)
87
+ ```
88
+
89
+ `extract` always returns a `pandas.DataFrame`. `saber.extract_spans(...)` and
90
+ `saber.extract_features(...)` are aliases for the two modes.
91
+
92
+ ## Input: strings, files, folders
93
+
94
+ The first argument accepts text, a file, a folder, or a mix of them. The
95
+ distinction between "a string of text" and "a path" is made by **type**, so it
96
+ is never guessed:
97
+
98
+ ```python
99
+ saber.extract("Um texto qualquer.") # str -> raw text
100
+ saber.extract(Path("texto.txt")) # Path -> one file
101
+ saber.extract(Path("corpus/")) # Path -> every text file in the folder
102
+ saber.extract(["Primeiro texto.", "Segundo."]) # several texts
103
+ saber.extract([Path("a.txt"), Path("corpus/")]) # several paths
104
+ ```
105
+
106
+ To read a plain string as a path, pass `source_type="path"`.
107
+
108
+ In a folder, every plain-text file is read: files with a `.txt` extension and
109
+ files with **no file-type extension at all**. A trailing number is not treated
110
+ as an extension, so corpus names like `A1.1` or `texto.2` are read too. Hidden
111
+ files (`.DS_Store`, dotfiles) and binary files are skipped. To pick up something
112
+ else — or to narrow the selection — pass an explicit glob:
113
+
114
+ ```python
115
+ saber.extract(Path("corpus/"), pattern="*.txt") # only .txt
116
+ saber.extract(Path("corpus/"), pattern="*.tsv") # some other extension
117
+ saber.extract(Path("corpus/"), pattern="aluno_*") # a subset, by name
118
+ ```
119
+
120
+ Other folder options: `recursive=False`, `encoding="utf-8"` (files that are not
121
+ valid UTF-8 fall back to latin-1 with a warning).
122
+
123
+ Each document is named after its file, without the extension (`texto57.txt` and
124
+ `texto57` both become `texto57`, while `A1.1` keeps its number); inline strings
125
+ become `text`, or `text_1`, `text_2`, … when there are several.
126
+
127
+ ## Choosing which structures to extract
128
+
129
+ Four independent filters. Values within one filter are OR-ed, and the filters
130
+ are AND-ed together. Matching ignores case and diacritics.
131
+
132
+ ```python
133
+ # A broad group: by key or by its Portuguese name
134
+ saber.extract(text, groups="a5")
135
+ saber.extract(text, groups="Pronomes") # the same 29 structures
136
+
137
+ # A finer category
138
+ saber.extract(text, categories="a5d2") # Pronomes > Demonstrativos
139
+
140
+ # A CEFR level, or several
141
+ saber.extract(text, levels="B1")
142
+ saber.extract(text, levels=["A1", "A2"])
143
+
144
+ # Individual structures, by id or by matcher name
145
+ saber.extract(text, structures=["a5d2_1_A2", "b5d2_6"])
146
+
147
+ # Combined: subjunctive-level pronoun structures only
148
+ saber.extract(text, groups="a5", levels="B2")
149
+
150
+ # Everything except a few structures
151
+ saber.extract(text, exclude=["a1d1_7_A2"])
152
+ ```
153
+
154
+ To see what the valid values are:
155
+
156
+ ```python
157
+ saber.list_structures() # all 252, as a DataFrame
158
+ saber.list_structures(groups="a7") # just the quantifiers
159
+ ```
160
+
161
+ | structure | matcher | level | group | group_name | category | category_name | description |
162
+ |---|---|---|---|---|---|---|---|
163
+ | a5d2_1_A2 | a5d2_1 | A2 | a5 | Pronomes | a5d2 | Demonstrativos | Pronomes - Demonstrativos - contração com preposições |
164
+
165
+ An unknown filter value raises `ValueError` listing the valid options, and so
166
+ does a combination that selects nothing.
167
+
168
+ Structure ids encode the taxonomy, so you can also filter the output frame
169
+ directly: `a5d2_1_A2` → group `a5`, category `a5d2`, matcher `a5d2_1`, level
170
+ `A2`.
171
+
172
+ ## Output columns
173
+
174
+ ### `mode="spans"` — one row per occurrence
175
+
176
+ | Column | Meaning |
177
+ |---|---|
178
+ | `document`, `path` | Which text the match came from (`path` is empty for inline strings) |
179
+ | `structure` | Full structure id, e.g. `a5d2_1_A2` |
180
+ | `matcher` | Name of the matcher function, e.g. `a5d2_1` |
181
+ | `level` | CEFR level, `A1`–`C2` |
182
+ | `group`, `group_name` | Broad group key and Portuguese name |
183
+ | `category`, `category_name` | Category key and Portuguese name |
184
+ | `description` | Portuguese description of the structure |
185
+ | `text` | The matched substring, sliced out of the preprocessed text |
186
+ | `reconstructed_text` | The matcher's own rendering of the match |
187
+ | `token_start`, `token_end` | Token offsets, end-exclusive |
188
+ | `char_start`, `char_end` | Character offsets into the preprocessed text |
189
+ | `pipeline` | `stanza` or `spacy`, depending on which parse the matcher needs |
190
+
191
+ Rows are sorted by document and position. Structures overlap freely — a single
192
+ phrase commonly matches several — so occurrences are *not* mutually exclusive.
193
+
194
+ `text` is the authoritative one: it is cut from the text by character offset, so
195
+ `analysis.text[char_start:char_end] == text` always holds and contractions come
196
+ out as written (`ao`, `no`, `daquele`). `reconstructed_text` comes from the
197
+ matcher's own `reconstruct_text` helper and is normally identical, but it is
198
+ rebuilt from tokens rather than sliced, so keep to `text` when exactness matters.
199
+
200
+ To get offsets against your own untouched string, note that they refer to the
201
+ *preprocessed* text — see [How it works](#how-it-works). `saber.analyze()`
202
+ returns that text alongside the matches if you need it:
203
+
204
+ ```python
205
+ analysis = saber.analyze("Ele foi ao cinema.", saber.all_structures())
206
+ analysis.text # the preprocessed text the offsets refer to
207
+ analysis.n_tokens # tokens excluding punctuation
208
+ analysis.matches # list of Match objects
209
+ ```
210
+
211
+ ### `mode="features"` — one row per document
212
+
213
+ | Column | Meaning |
214
+ |---|---|
215
+ | `document`, `path` | Which text the row describes |
216
+ | `n_tokens` | Tokens excluding whitespace and punctuation |
217
+ | `<structure>_count` | Raw number of occurrences |
218
+ | `<structure>_norm` | Occurrences per 100 tokens |
219
+ | `<structure>_present` | `1` if the structure occurs at all, else `0` |
220
+
221
+ Columns are emitted for **every** selected structure, including those that never
222
+ matched, so feature matrices line up across documents and across runs. Their
223
+ order follows the declaration order of `LABEL_TRANSLATIONS` in
224
+ `saber/label_translation.py`, which is curated taxonomically.
225
+
226
+ Selecting all 252 structures therefore gives `3 × 252 + 3 = 759` columns. Use
227
+ `features=` to keep only what you need:
228
+
229
+ ```python
230
+ saber.extract(corpus, mode="features", features=["normalized", "presence"])
231
+ ```
232
+
233
+ ## How it works
234
+
235
+ 1. **Preprocessing.** Dialogue dashes are stripped and whitespace collapsed. All
236
+ offsets refer to this preprocessed text, which `saber.analyze()` exposes as
237
+ `analysis.text`.
238
+ 2. **Parsing.** The text is parsed twice: by
239
+ [spaCy-Stanza](https://github.com/explosion/spacy-stanza) (`tokenize`, `mwt`,
240
+ `pos`, `lemma`, `depparse`) for morphology and dependencies, and by spaCy's
241
+ `pt_core_news_md` for the handful of matchers that need named entities or
242
+ spaCy's own tokenisation. A matcher declares which parse it needs; you never
243
+ choose.
244
+ 3. **Matching.** Each selected structure's matcher runs over the appropriate
245
+ parse using spaCy's `Matcher`, `PhraseMatcher` and `DependencyMatcher`, and
246
+ returns token spans.
247
+ 4. **Offsets.** Token spans are converted to character offsets. Stanza expands
248
+ multi-word tokens (`no` → `em o`), which destroys spaCy's own character
249
+ offsets, so they are recovered from the underlying Stanza document — this is
250
+ why `text` reads `no` and not `em o`.
251
+ 5. **Tabulating.** Occurrences become the spans table, or are counted and
252
+ normalised into the feature table.
253
+
254
+ A matcher that raises is reported as a warning and skipped, so one broken
255
+ pattern cannot abort a corpus run. Use `on_error="raise"` while debugging, or
256
+ `on_error="ignore"` to silence it.
257
+
258
+ ## Performance
259
+
260
+ Expect **a few seconds per short text**, and add a few additional seconds of one-off model loading
261
+ to the first extraction in a process. Two things dominate: the Stanza parse, and
262
+ the fact that each matcher rebuilds its spaCy `Matcher` object on every call.
263
+
264
+ Practical consequences:
265
+
266
+ - Extract once for all the structures you might want, then filter the resulting
267
+ DataFrame — that is much cheaper than re-running with different filters.
268
+ - Restricting `groups`/`categories`/`levels` up front does cut the matching cost
269
+ roughly in proportion, but not the parsing cost.
270
+ - A progress bar over the documents appears whenever there is more than one
271
+ document and you are working in a terminal or a notebook. `progress=False`
272
+ turns it off, `progress=True` forces it on (useful when output is redirected).
273
+ - Progress is also logged per document, which is the better option for scripts
274
+ and log files. Turn it on with `logging.basicConfig(level=logging.INFO)`.
275
+
276
+ ## Working on the matchers
277
+
278
+ A matcher function is named after its structure minus the CEFR suffix
279
+ (`a5d2_1` → `a5d2_1_A2`), takes a spaCy `Doc`, and returns a list of
280
+ `(label, text, token_start, token_end)` tuples. Setting
281
+ `<function>.REQUIRES_SPACY = True` after the definition routes it to the
282
+ `pt_core_news_md` parse instead of the Stanza one.
283
+
284
+ A structure is available only if it has **both** an entry in
285
+ `LABEL_TRANSLATIONS` and a matcher function, so commenting out a label in
286
+ `saber/label_translation.py` disables the structure.
287
+ `saber.missing_labels()` lists labels with no matcher.
288
+
289
+ ## Testing the matchers
290
+
291
+ `tests/tests.yaml` holds positive and negative example sentences per matcher.
292
+ The harness measures precision and recall for each, both with all patterns
293
+ active and with only the first pattern registered:
294
+
295
+ ```bash
296
+ pip install -e ".[dev]"
297
+ python tests/test_suite.py
298
+ ```
299
+
300
+ Results are written to `tests/test_results.csv`; the committed copy is the
301
+ reference from the last full run.
302
+
303
+ ## Licence and attribution
304
+
305
+ Released under the [MIT License](https://opensource.org/licenses/MIT) — see
306
+ [LICENSE](LICENSE). You may use, modify, and distribute this work, including
307
+ commercially, provided the copyright notice and licence text are retained.
308
+
309
+ The structure inventory and its CEFR mapping derive from the
310
+ [Referencial Camões PLE](https://www.instituto-camoes.pt/activity/centro-virtual/referencial-camoes-ple)
311
+ (Instituto Camões).