pyests 0.7.0__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyests-0.7.0 → pyests-0.8.0}/PKG-INFO +33 -33
- {pyests-0.7.0 → pyests-0.8.0}/README.es.md +30 -30
- {pyests-0.7.0 → pyests-0.8.0}/README.md +30 -30
- {pyests-0.7.0 → pyests-0.8.0}/ests/__init__.py +44 -22
- {pyests-0.7.0 → pyests-0.8.0}/ests/basic_stats.py +4 -2
- pyests-0.8.0/ests/cohesion_stats.py +399 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/components.py +50 -132
- {pyests-0.7.0 → pyests-0.8.0}/ests/constants.py +185 -35
- pyests-0.8.0/ests/corpus/__init__.py +109 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/corpus/compare.py +62 -145
- {pyests-0.7.0 → pyests-0.8.0}/ests/corpus/keyness.py +2 -2
- {pyests-0.7.0 → pyests-0.8.0}/ests/corpus/kwic.py +8 -3
- {pyests-0.7.0 → pyests-0.8.0}/ests/corpus/stylometry.py +20 -16
- {pyests-0.7.0 → pyests-0.8.0}/ests/datasets/freq_dict.py +3 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/datasets/spanish_literature.py +3 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/datasets/spanish_sonnets.py +3 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/diversity_stats.py +17 -9
- {pyests-0.7.0 → pyests-0.8.0}/ests/extractors.py +3 -2
- {pyests-0.7.0 → pyests-0.8.0}/ests/lexical_stats.py +99 -185
- {pyests-0.7.0 → pyests-0.8.0}/ests/morph_stats.py +46 -47
- {pyests-0.7.0 → pyests-0.8.0}/ests/phon_stats.py +61 -170
- {pyests-0.7.0 → pyests-0.8.0}/ests/readability_stats.py +30 -44
- {pyests-0.7.0 → pyests-0.8.0}/ests/resources/README.md +1 -1
- {pyests-0.7.0 → pyests-0.8.0}/ests/resources/connectors.tsv +23 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/style_stats.py +223 -501
- {pyests-0.7.0 → pyests-0.8.0}/ests/syllables.py +1 -1
- {pyests-0.7.0 → pyests-0.8.0}/ests/syntax_stats.py +148 -280
- {pyests-0.7.0 → pyests-0.8.0}/ests/utils.py +59 -113
- {pyests-0.7.0 → pyests-0.8.0}/ests/verse_stats.py +21 -25
- pyests-0.8.0/ests/visualizers/__init__.py +58 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/visualizers/highlight.py +173 -146
- {pyests-0.7.0 → pyests-0.8.0}/ests/visualizers/sentences.py +10 -3
- {pyests-0.7.0 → pyests-0.8.0}/pyproject.toml +3 -3
- pyests-0.7.0/ests/cohesion_stats.py +0 -590
- pyests-0.7.0/ests/corpus/__init__.py +0 -65
- pyests-0.7.0/ests/visualizers/__init__.py +0 -29
- {pyests-0.7.0 → pyests-0.8.0}/.gitignore +0 -0
- {pyests-0.7.0 → pyests-0.8.0}/LICENSE.txt +0 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/corpus/collocations.py +0 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/corpus/dispersion.py +0 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/datasets/__init__.py +0 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/exceptions.py +0 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/py.typed +0 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/resources/google_books_top10000.txt +0 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/visualizers/corpus.py +0 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/visualizers/fingerprinting.py +0 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/visualizers/stylometry.py +0 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/visualizers/vocabulary.py +0 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/visualizers/word_tree.py +0 -0
- {pyests-0.7.0 → pyests-0.8.0}/ests/visualizers/zipf.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: pyests
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: Spanish Texts Statistics
|
|
5
5
|
Project-URL: Homepage, https://github.com/SergeyShk/esTS
|
|
6
6
|
Project-URL: Repository, https://github.com/SergeyShk/esTS
|
|
@@ -28,14 +28,14 @@ Classifier: Topic :: Text Processing
|
|
|
28
28
|
Classifier: Topic :: Text Processing :: Linguistic
|
|
29
29
|
Classifier: Typing :: Typed
|
|
30
30
|
Requires-Python: >=3.11
|
|
31
|
-
Requires-Dist: anyts<0.
|
|
31
|
+
Requires-Dist: anyts<0.4,>=0.3.9
|
|
32
32
|
Requires-Dist: graphviz>=0.20.3
|
|
33
33
|
Requires-Dist: matplotlib>=3.8
|
|
34
34
|
Requires-Dist: numpy>=1.26
|
|
35
35
|
Requires-Dist: pandas>=2.1
|
|
36
36
|
Requires-Dist: scipy>=1.11
|
|
37
37
|
Requires-Dist: simplemma<3,>=2.0
|
|
38
|
-
Requires-Dist: spacy
|
|
38
|
+
Requires-Dist: spacy<4,>=3.8.15
|
|
39
39
|
Description-Content-Type: text/markdown
|
|
40
40
|
|
|
41
41
|
<p align="center">
|
|
@@ -87,7 +87,7 @@ The library works both with raw strings and with `Doc` objects of [spaCy](https:
|
|
|
87
87
|
* **[Visualizers](https://sergeyshk.github.io/esTS/visualizers/zipf/)** - Zipf's law, literature fingerprinting, a word tree, lexical dispersion and keywords, a network of collocations, a dendrogram, PCA and MDS by Delta, vocabulary growth, sentence lengths, and the highlighting of the fragments the statistics count, such as long sentences and passives
|
|
88
88
|
* **[Datasets](https://sergeyshk.github.io/esTS/datasets/spanishliterature/)** - Spanish-language literature in the public domain: 150 works by 33 authors in four genres; 4,259 sonnets of the 15th-20th centuries with the metrical pattern and the rhyme of every line; a frequency dictionary of 83,785 lemmas by Google Books Ngram
|
|
89
89
|
* **[spaCy components](https://sergeyshk.github.io/esTS/components/)** - every statistics class as a component of a pipeline, the statistics attached to the `Doc` in one pass
|
|
90
|
-
* **[Cohesion statistics](https://sergeyshk.github.io/esTS/stats/cohesion_stats/)** - the overlap of nouns, arguments and content words between sentences, givenness and temporal cohesion in the manner of Coh-Metrix, with the density of
|
|
90
|
+
* **[Cohesion statistics](https://sergeyshk.github.io/esTS/stats/cohesion_stats/)** - the overlap of nouns, arguments and content words between sentences, givenness and temporal cohesion in the manner of Coh-Metrix, with the density of 273 Spanish discourse markers by class and by place
|
|
91
91
|
* **[Lexical sophistication statistics](https://sergeyshk.github.io/esTS/stats/lexical_stats/)** - how rare the words of a text are in the language: the frequency, range and dispersion of the lemmas by a dictionary of Google Books Ngram, the frequency bands top-1000 to 10000, surprisal, perplexity and lexical density
|
|
92
92
|
* **[Style metrics](https://sergeyshk.github.io/esTS/stats/style_stats/)** - the SEO indicators of Advego and Text.ru (nausea, water content, spam score, naturalness by Zipf's law, keyword density) and the markers of the officialese style by the Spanish-language guides to plain language: verbal nouns, compound prepositions, parenthetical expressions and clichés
|
|
93
93
|
* **[Phonostatistics](https://sergeyshk.github.io/esTS/stats/phon_stats/)** - the shares of the classes of sounds, consonant clusters, hiatuses, open syllables, hardness and the indices of alliteration and assonance, over the sounds of a rule-based transcription
|
|
@@ -178,7 +178,7 @@ More in the [documentation](https://sergeyshk.github.io/esTS/extractors/sentence
|
|
|
178
178
|
|
|
179
179
|
<br>
|
|
180
180
|
|
|
181
|
-
The syllables and the stress follow from the Spanish spelling, with no dictionary: the written accent, or the ending of the word when there is none, gives the stressed syllable. Adverbs in `-mente` and hyphenated compounds
|
|
181
|
+
The syllables and the stress follow from the Spanish spelling, with no dictionary: the written accent, or the ending of the word when there is none, gives the stressed syllable. Adverbs in `-mente` carry two stresses, and hyphenated compounds one per part.
|
|
182
182
|
|
|
183
183
|
```python
|
|
184
184
|
>>> from ests.syllables import stress_type, syllabify, word_stress, word_stresses
|
|
@@ -230,21 +230,21 @@ A complex word has three or more syllables and a long word seven or more letters
|
|
|
230
230
|
|
|
231
231
|
>>> text = "Hay tres clases de mentiras: mentiras, malditas mentiras y estadísticas"
|
|
232
232
|
>>> BasicStats(text).print_stats()
|
|
233
|
-
Statistic
|
|
233
|
+
Statistic | Value
|
|
234
234
|
------------------------------
|
|
235
|
-
Sentences
|
|
236
|
-
Words
|
|
237
|
-
Unique words
|
|
238
|
-
Long words
|
|
239
|
-
Complex words
|
|
240
|
-
Simple words
|
|
241
|
-
Monosyllabic words
|
|
242
|
-
Polysyllabic words
|
|
243
|
-
Characters
|
|
244
|
-
Letters
|
|
245
|
-
Spaces
|
|
246
|
-
Syllables
|
|
247
|
-
Punctuation marks
|
|
235
|
+
Sentences | 1
|
|
236
|
+
Words | 10
|
|
237
|
+
Unique words | 8
|
|
238
|
+
Long words | 5
|
|
239
|
+
Complex words | 5
|
|
240
|
+
Simple words | 5
|
|
241
|
+
Monosyllabic words | 4
|
|
242
|
+
Polysyllabic words | 6
|
|
243
|
+
Characters | 71
|
|
244
|
+
Letters | 60
|
|
245
|
+
Spaces | 9
|
|
246
|
+
Syllables | 23
|
|
247
|
+
Punctuation marks | 2
|
|
248
248
|
```
|
|
249
249
|
|
|
250
250
|
More in the [documentation](https://sergeyshk.github.io/esTS/stats/basic_stats/).
|
|
@@ -289,17 +289,17 @@ The coefficients of the Flesch reading ease are selected by `preset`: by default
|
|
|
289
289
|
'reading_time': 0.03597122302158273}
|
|
290
290
|
|
|
291
291
|
>>> rs.print_stats()
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
Flesch reading ease (Szigriszt-Pazos)
|
|
295
|
-
Gutiérrez de Polini comprehensibility
|
|
296
|
-
Crawford grade
|
|
297
|
-
Legibilidad µ
|
|
298
|
-
SOL grade (SMOG for Spanish)
|
|
299
|
-
LIX readability index
|
|
300
|
-
RIX readability index
|
|
301
|
-
Consensus grade
|
|
302
|
-
Reading time (min)
|
|
292
|
+
Metric | Value
|
|
293
|
+
-------------------------------------------------
|
|
294
|
+
Flesch reading ease (Szigriszt-Pazos) | 53.55
|
|
295
|
+
Gutiérrez de Polini comprehensibility | 33.50
|
|
296
|
+
Crawford grade | 5.81
|
|
297
|
+
Legibilidad µ | 50.94
|
|
298
|
+
SOL grade (SMOG for Spanish) | 9.26
|
|
299
|
+
LIX readability index | 60.00
|
|
300
|
+
RIX readability index | 5.00
|
|
301
|
+
Consensus grade | 9.00
|
|
302
|
+
Reading time (min) | 0.04
|
|
303
303
|
|
|
304
304
|
>>> rs.describe_level()
|
|
305
305
|
'algo difícil'
|
|
@@ -398,7 +398,7 @@ More in the [documentation](https://sergeyshk.github.io/esTS/stats/morph_stats/)
|
|
|
398
398
|
|
|
399
399
|
The dependency tree of Universal Dependencies and the constructions that the Spanish-language guides to plain language warn about:
|
|
400
400
|
|
|
401
|
-
* the complexity of the tree: dependency distances, depth, leaves and subtrees, valency of the finite verbs, coordination chains, clauses and
|
|
401
|
+
* the complexity of the tree: dependency distances, depth, leaves and subtrees, valency of the finite verbs, coordination chains, clauses, subordinate and infinitive clauses, modifiers per noun
|
|
402
402
|
* the constructions: the passive with `ser` and with `se`, the participial and the gerund clauses, the chains of `de`, the split predicates, the impersonal `se`, the words of negation, the ratio of nouns to verbs
|
|
403
403
|
|
|
404
404
|
```python
|
|
@@ -437,7 +437,7 @@ Referential cohesion in the manner of Coh-Metrix and its Spanish adaptation Coh-
|
|
|
437
437
|
* the overlap of nouns, of arguments and of content words between adjacent sentences and between all pairs of sentences, binary and proportional
|
|
438
438
|
* givenness: pronouns, demonstratives and the content words whose lemma was already used
|
|
439
439
|
* temporal cohesion: the repetition of the tense and of the mood of the verbs of adjacent sentences
|
|
440
|
-
* the density of
|
|
440
|
+
* the density of 273 Spanish discourse markers by class - causal, adversative, concessive, temporal, additive, conditional, reformulative - by kind and by place: linking a sentence to the ones before or the parts of a sentence
|
|
441
441
|
|
|
442
442
|
```python
|
|
443
443
|
>>> from ests import CohesionStats
|
|
@@ -595,7 +595,7 @@ The measures of corpus linguistics that compare corpora and describe the use of
|
|
|
595
595
|
* the dispersion of a word over the parts of a text: DP of Gries, normalized DP, Juilland's D, Carroll's D2, Rosengren's S and the Kullback-Leibler divergence
|
|
596
596
|
* a KWIC concordance by word form or by lemma
|
|
597
597
|
* stylometry: the distances between texts by Burrows's Delta and its variants, with the attribution of a text to reference authors, the markers of preferred and avoided words by Zeta, Kilgarriff's chi-square, the Mendenhall curve and the profile of the function words
|
|
598
|
-
* the comparison of two corpora by 132 features of a text over windows of
|
|
598
|
+
* the comparison of two corpora by 132 features of a text over windows of the same size: Cliff's delta, Cohen's d and the AUC of every feature, the Mann-Whitney test with Holm's correction and a bootstrap interval of the difference of the medians
|
|
599
599
|
|
|
600
600
|
```python
|
|
601
601
|
>>> from ests import WordsExtractor
|
|
@@ -47,7 +47,7 @@ La biblioteca trabaja tanto con cadenas como con objetos `Doc` de [spaCy](https:
|
|
|
47
47
|
* **[Visualizaciones](https://sergeyshk.github.io/esTS/es/visualizers/zipf/)** - ley de Zipf, huella literaria, árbol de palabras, dispersión léxica y palabras clave, red de colocaciones, dendrograma, PCA y MDS por la Delta, crecimiento del vocabulario, longitudes de las oraciones y el resaltado de los fragmentos que cuentan las estadísticas, como las oraciones largas y las pasivas
|
|
48
48
|
* **[Conjuntos de datos](https://sergeyshk.github.io/esTS/es/datasets/spanishliterature/)** - literatura en español de dominio público: 150 obras de 33 autores en cuatro géneros; 4259 sonetos de los siglos XV-XX con el patrón métrico y la rima de cada verso; un diccionario de frecuencias de 83 785 lemas según Google Books Ngram
|
|
49
49
|
* **[Componentes de spaCy](https://sergeyshk.github.io/esTS/es/components/)** - cada clase de estadísticas como componente de un pipeline, con las estadísticas puestas en el `Doc` en una sola pasada
|
|
50
|
-
* **[Estadísticas de cohesión](https://sergeyshk.github.io/esTS/es/stats/cohesion_stats/)** - la repetición de sustantivos, argumentos y palabras con contenido entre oraciones, la información dada y la cohesión temporal a la manera de Coh-Metrix, con la densidad de
|
|
50
|
+
* **[Estadísticas de cohesión](https://sergeyshk.github.io/esTS/es/stats/cohesion_stats/)** - la repetición de sustantivos, argumentos y palabras con contenido entre oraciones, la información dada y la cohesión temporal a la manera de Coh-Metrix, con la densidad de 273 marcadores del discurso españoles por clase y por lugar
|
|
51
51
|
* **[Estadísticas de complejidad léxica](https://sergeyshk.github.io/esTS/es/stats/lexical_stats/)** - cuán raras son las palabras de un texto en la lengua: la frecuencia, el rango y la dispersión de los lemas según un diccionario de Google Books Ngram, las bandas de frecuencia del top-1000 al 10000, la sorpresa, la perplejidad y la densidad léxica
|
|
52
52
|
* **[Métricas de estilo](https://sergeyshk.github.io/esTS/es/stats/style_stats/)** - los indicadores SEO de Advego y Text.ru (náusea, contenido de agua, índice de spam, naturalidad según Zipf, densidad de palabras clave) y los marcadores del estilo burocrático según las guías de lenguaje claro en español: sustantivos deverbales, locuciones prepositivas, expresiones parentéticas y clichés
|
|
53
53
|
* **[Fonoestadística](https://sergeyshk.github.io/esTS/es/stats/phon_stats/)** - proporciones de las clases de sonidos, grupos consonánticos, hiatos, sílabas abiertas, dureza e índices de aliteración y de asonancia, sobre los sonidos de una transcripción por reglas
|
|
@@ -138,7 +138,7 @@ Más en la [documentación](https://sergeyshk.github.io/esTS/es/extractors/sente
|
|
|
138
138
|
|
|
139
139
|
<br>
|
|
140
140
|
|
|
141
|
-
Las sílabas y el acento se deducen de la ortografía española, sin diccionario: la tilde, o la terminación de la palabra cuando no la hay, da la sílaba tónica. Los adverbios en `-mente` y los compuestos con guion
|
|
141
|
+
Las sílabas y el acento se deducen de la ortografía española, sin diccionario: la tilde, o la terminación de la palabra cuando no la hay, da la sílaba tónica. Los adverbios en `-mente` llevan dos acentos, y los compuestos con guion uno por parte.
|
|
142
142
|
|
|
143
143
|
```python
|
|
144
144
|
>>> from ests.syllables import stress_type, syllabify, word_stress, word_stresses
|
|
@@ -190,21 +190,21 @@ Una palabra compleja tiene tres o más sílabas y una palabra larga siete o más
|
|
|
190
190
|
|
|
191
191
|
>>> text = "Hay tres clases de mentiras: mentiras, malditas mentiras y estadísticas"
|
|
192
192
|
>>> BasicStats(text).print_stats()
|
|
193
|
-
Statistic
|
|
193
|
+
Statistic | Value
|
|
194
194
|
------------------------------
|
|
195
|
-
Sentences
|
|
196
|
-
Words
|
|
197
|
-
Unique words
|
|
198
|
-
Long words
|
|
199
|
-
Complex words
|
|
200
|
-
Simple words
|
|
201
|
-
Monosyllabic words
|
|
202
|
-
Polysyllabic words
|
|
203
|
-
Characters
|
|
204
|
-
Letters
|
|
205
|
-
Spaces
|
|
206
|
-
Syllables
|
|
207
|
-
Punctuation marks
|
|
195
|
+
Sentences | 1
|
|
196
|
+
Words | 10
|
|
197
|
+
Unique words | 8
|
|
198
|
+
Long words | 5
|
|
199
|
+
Complex words | 5
|
|
200
|
+
Simple words | 5
|
|
201
|
+
Monosyllabic words | 4
|
|
202
|
+
Polysyllabic words | 6
|
|
203
|
+
Characters | 71
|
|
204
|
+
Letters | 60
|
|
205
|
+
Spaces | 9
|
|
206
|
+
Syllables | 23
|
|
207
|
+
Punctuation marks | 2
|
|
208
208
|
```
|
|
209
209
|
|
|
210
210
|
Más en la [documentación](https://sergeyshk.github.io/esTS/es/stats/basic_stats/).
|
|
@@ -249,17 +249,17 @@ Los coeficientes de la facilidad de lectura de Flesch se eligen con `preset`: po
|
|
|
249
249
|
'reading_time': 0.03597122302158273}
|
|
250
250
|
|
|
251
251
|
>>> rs.print_stats()
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
Flesch reading ease (Szigriszt-Pazos)
|
|
255
|
-
Gutiérrez de Polini comprehensibility
|
|
256
|
-
Crawford grade
|
|
257
|
-
Legibilidad µ
|
|
258
|
-
SOL grade (SMOG for Spanish)
|
|
259
|
-
LIX readability index
|
|
260
|
-
RIX readability index
|
|
261
|
-
Consensus grade
|
|
262
|
-
Reading time (min)
|
|
252
|
+
Metric | Value
|
|
253
|
+
-------------------------------------------------
|
|
254
|
+
Flesch reading ease (Szigriszt-Pazos) | 53.55
|
|
255
|
+
Gutiérrez de Polini comprehensibility | 33.50
|
|
256
|
+
Crawford grade | 5.81
|
|
257
|
+
Legibilidad µ | 50.94
|
|
258
|
+
SOL grade (SMOG for Spanish) | 9.26
|
|
259
|
+
LIX readability index | 60.00
|
|
260
|
+
RIX readability index | 5.00
|
|
261
|
+
Consensus grade | 9.00
|
|
262
|
+
Reading time (min) | 0.04
|
|
263
263
|
|
|
264
264
|
>>> rs.describe_level()
|
|
265
265
|
'algo difícil'
|
|
@@ -358,7 +358,7 @@ Más en la [documentación](https://sergeyshk.github.io/esTS/es/stats/morph_stat
|
|
|
358
358
|
|
|
359
359
|
El árbol de dependencias de Universal Dependencies y las construcciones contra las que advierten las guías de lenguaje claro en español:
|
|
360
360
|
|
|
361
|
-
* la complejidad del árbol: distancias de dependencia, profundidad, hojas y subárboles, valencia de los verbos personales, cadenas de coordinación, cláusulas
|
|
361
|
+
* la complejidad del árbol: distancias de dependencia, profundidad, hojas y subárboles, valencia de los verbos personales, cadenas de coordinación, cláusulas, subordinadas e infinitivas, modificadores por sustantivo
|
|
362
362
|
* las construcciones: la pasiva con `ser` y con `se`, las cláusulas de participio y de gerundio, las cadenas de `de`, los predicados escindidos, el `se` impersonal, las palabras de negación, la razón entre sustantivos y verbos
|
|
363
363
|
|
|
364
364
|
```python
|
|
@@ -397,7 +397,7 @@ La cohesión referencial a la manera de Coh-Metrix y de su adaptación española
|
|
|
397
397
|
* la repetición de sustantivos, de argumentos y de palabras con contenido entre oraciones contiguas y entre todos los pares de oraciones, binaria y proporcional
|
|
398
398
|
* la información dada: pronombres, demostrativos y palabras con contenido cuyo lema ya se había usado
|
|
399
399
|
* la cohesión temporal: la repetición del tiempo y del modo de los verbos de oraciones contiguas
|
|
400
|
-
* la densidad de
|
|
400
|
+
* la densidad de 273 marcadores del discurso españoles por clase - causales, adversativos, concesivos, temporales, aditivos, condicionales, reformulativos -, por tipo y por lugar: los que unen una oración con las anteriores o las partes de una oración
|
|
401
401
|
|
|
402
402
|
```python
|
|
403
403
|
>>> from ests import CohesionStats
|
|
@@ -555,7 +555,7 @@ Las medidas de la lingüística de corpus que comparan corpus y describen el uso
|
|
|
555
555
|
* la dispersión de una palabra por las partes de un texto: la DP de Gries, la DP normalizada, la D de Juilland, la D2 de Carroll, la S de Rosengren y la divergencia de Kullback-Leibler
|
|
556
556
|
* una concordancia KWIC por forma o por lema
|
|
557
557
|
* estilometría: las distancias entre textos por la Delta de Burrows y sus variantes, con la atribución de un texto a autores de referencia, los marcadores de palabras preferidas y evitadas por Zeta, el ji cuadrado de Kilgarriff, la curva de Mendenhall y el perfil de las palabras funcionales
|
|
558
|
-
* la comparación de dos corpus por 132 rasgos de un texto en ventanas
|
|
558
|
+
* la comparación de dos corpus por 132 rasgos de un texto en ventanas del mismo tamaño: la delta de Cliff, la d de Cohen y el AUC de cada rasgo, la prueba de Mann-Whitney con la corrección de Holm y un intervalo bootstrap de la diferencia de las medianas
|
|
559
559
|
|
|
560
560
|
```python
|
|
561
561
|
>>> from ests import WordsExtractor
|
|
@@ -47,7 +47,7 @@ The library works both with raw strings and with `Doc` objects of [spaCy](https:
|
|
|
47
47
|
* **[Visualizers](https://sergeyshk.github.io/esTS/visualizers/zipf/)** - Zipf's law, literature fingerprinting, a word tree, lexical dispersion and keywords, a network of collocations, a dendrogram, PCA and MDS by Delta, vocabulary growth, sentence lengths, and the highlighting of the fragments the statistics count, such as long sentences and passives
|
|
48
48
|
* **[Datasets](https://sergeyshk.github.io/esTS/datasets/spanishliterature/)** - Spanish-language literature in the public domain: 150 works by 33 authors in four genres; 4,259 sonnets of the 15th-20th centuries with the metrical pattern and the rhyme of every line; a frequency dictionary of 83,785 lemmas by Google Books Ngram
|
|
49
49
|
* **[spaCy components](https://sergeyshk.github.io/esTS/components/)** - every statistics class as a component of a pipeline, the statistics attached to the `Doc` in one pass
|
|
50
|
-
* **[Cohesion statistics](https://sergeyshk.github.io/esTS/stats/cohesion_stats/)** - the overlap of nouns, arguments and content words between sentences, givenness and temporal cohesion in the manner of Coh-Metrix, with the density of
|
|
50
|
+
* **[Cohesion statistics](https://sergeyshk.github.io/esTS/stats/cohesion_stats/)** - the overlap of nouns, arguments and content words between sentences, givenness and temporal cohesion in the manner of Coh-Metrix, with the density of 273 Spanish discourse markers by class and by place
|
|
51
51
|
* **[Lexical sophistication statistics](https://sergeyshk.github.io/esTS/stats/lexical_stats/)** - how rare the words of a text are in the language: the frequency, range and dispersion of the lemmas by a dictionary of Google Books Ngram, the frequency bands top-1000 to 10000, surprisal, perplexity and lexical density
|
|
52
52
|
* **[Style metrics](https://sergeyshk.github.io/esTS/stats/style_stats/)** - the SEO indicators of Advego and Text.ru (nausea, water content, spam score, naturalness by Zipf's law, keyword density) and the markers of the officialese style by the Spanish-language guides to plain language: verbal nouns, compound prepositions, parenthetical expressions and clichés
|
|
53
53
|
* **[Phonostatistics](https://sergeyshk.github.io/esTS/stats/phon_stats/)** - the shares of the classes of sounds, consonant clusters, hiatuses, open syllables, hardness and the indices of alliteration and assonance, over the sounds of a rule-based transcription
|
|
@@ -138,7 +138,7 @@ More in the [documentation](https://sergeyshk.github.io/esTS/extractors/sentence
|
|
|
138
138
|
|
|
139
139
|
<br>
|
|
140
140
|
|
|
141
|
-
The syllables and the stress follow from the Spanish spelling, with no dictionary: the written accent, or the ending of the word when there is none, gives the stressed syllable. Adverbs in `-mente` and hyphenated compounds
|
|
141
|
+
The syllables and the stress follow from the Spanish spelling, with no dictionary: the written accent, or the ending of the word when there is none, gives the stressed syllable. Adverbs in `-mente` carry two stresses, and hyphenated compounds one per part.
|
|
142
142
|
|
|
143
143
|
```python
|
|
144
144
|
>>> from ests.syllables import stress_type, syllabify, word_stress, word_stresses
|
|
@@ -190,21 +190,21 @@ A complex word has three or more syllables and a long word seven or more letters
|
|
|
190
190
|
|
|
191
191
|
>>> text = "Hay tres clases de mentiras: mentiras, malditas mentiras y estadísticas"
|
|
192
192
|
>>> BasicStats(text).print_stats()
|
|
193
|
-
Statistic
|
|
193
|
+
Statistic | Value
|
|
194
194
|
------------------------------
|
|
195
|
-
Sentences
|
|
196
|
-
Words
|
|
197
|
-
Unique words
|
|
198
|
-
Long words
|
|
199
|
-
Complex words
|
|
200
|
-
Simple words
|
|
201
|
-
Monosyllabic words
|
|
202
|
-
Polysyllabic words
|
|
203
|
-
Characters
|
|
204
|
-
Letters
|
|
205
|
-
Spaces
|
|
206
|
-
Syllables
|
|
207
|
-
Punctuation marks
|
|
195
|
+
Sentences | 1
|
|
196
|
+
Words | 10
|
|
197
|
+
Unique words | 8
|
|
198
|
+
Long words | 5
|
|
199
|
+
Complex words | 5
|
|
200
|
+
Simple words | 5
|
|
201
|
+
Monosyllabic words | 4
|
|
202
|
+
Polysyllabic words | 6
|
|
203
|
+
Characters | 71
|
|
204
|
+
Letters | 60
|
|
205
|
+
Spaces | 9
|
|
206
|
+
Syllables | 23
|
|
207
|
+
Punctuation marks | 2
|
|
208
208
|
```
|
|
209
209
|
|
|
210
210
|
More in the [documentation](https://sergeyshk.github.io/esTS/stats/basic_stats/).
|
|
@@ -249,17 +249,17 @@ The coefficients of the Flesch reading ease are selected by `preset`: by default
|
|
|
249
249
|
'reading_time': 0.03597122302158273}
|
|
250
250
|
|
|
251
251
|
>>> rs.print_stats()
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
Flesch reading ease (Szigriszt-Pazos)
|
|
255
|
-
Gutiérrez de Polini comprehensibility
|
|
256
|
-
Crawford grade
|
|
257
|
-
Legibilidad µ
|
|
258
|
-
SOL grade (SMOG for Spanish)
|
|
259
|
-
LIX readability index
|
|
260
|
-
RIX readability index
|
|
261
|
-
Consensus grade
|
|
262
|
-
Reading time (min)
|
|
252
|
+
Metric | Value
|
|
253
|
+
-------------------------------------------------
|
|
254
|
+
Flesch reading ease (Szigriszt-Pazos) | 53.55
|
|
255
|
+
Gutiérrez de Polini comprehensibility | 33.50
|
|
256
|
+
Crawford grade | 5.81
|
|
257
|
+
Legibilidad µ | 50.94
|
|
258
|
+
SOL grade (SMOG for Spanish) | 9.26
|
|
259
|
+
LIX readability index | 60.00
|
|
260
|
+
RIX readability index | 5.00
|
|
261
|
+
Consensus grade | 9.00
|
|
262
|
+
Reading time (min) | 0.04
|
|
263
263
|
|
|
264
264
|
>>> rs.describe_level()
|
|
265
265
|
'algo difícil'
|
|
@@ -358,7 +358,7 @@ More in the [documentation](https://sergeyshk.github.io/esTS/stats/morph_stats/)
|
|
|
358
358
|
|
|
359
359
|
The dependency tree of Universal Dependencies and the constructions that the Spanish-language guides to plain language warn about:
|
|
360
360
|
|
|
361
|
-
* the complexity of the tree: dependency distances, depth, leaves and subtrees, valency of the finite verbs, coordination chains, clauses and
|
|
361
|
+
* the complexity of the tree: dependency distances, depth, leaves and subtrees, valency of the finite verbs, coordination chains, clauses, subordinate and infinitive clauses, modifiers per noun
|
|
362
362
|
* the constructions: the passive with `ser` and with `se`, the participial and the gerund clauses, the chains of `de`, the split predicates, the impersonal `se`, the words of negation, the ratio of nouns to verbs
|
|
363
363
|
|
|
364
364
|
```python
|
|
@@ -397,7 +397,7 @@ Referential cohesion in the manner of Coh-Metrix and its Spanish adaptation Coh-
|
|
|
397
397
|
* the overlap of nouns, of arguments and of content words between adjacent sentences and between all pairs of sentences, binary and proportional
|
|
398
398
|
* givenness: pronouns, demonstratives and the content words whose lemma was already used
|
|
399
399
|
* temporal cohesion: the repetition of the tense and of the mood of the verbs of adjacent sentences
|
|
400
|
-
* the density of
|
|
400
|
+
* the density of 273 Spanish discourse markers by class - causal, adversative, concessive, temporal, additive, conditional, reformulative - by kind and by place: linking a sentence to the ones before or the parts of a sentence
|
|
401
401
|
|
|
402
402
|
```python
|
|
403
403
|
>>> from ests import CohesionStats
|
|
@@ -555,7 +555,7 @@ The measures of corpus linguistics that compare corpora and describe the use of
|
|
|
555
555
|
* the dispersion of a word over the parts of a text: DP of Gries, normalized DP, Juilland's D, Carroll's D2, Rosengren's S and the Kullback-Leibler divergence
|
|
556
556
|
* a KWIC concordance by word form or by lemma
|
|
557
557
|
* stylometry: the distances between texts by Burrows's Delta and its variants, with the attribution of a text to reference authors, the markers of preferred and avoided words by Zeta, Kilgarriff's chi-square, the Mendenhall curve and the profile of the function words
|
|
558
|
-
* the comparison of two corpora by 132 features of a text over windows of
|
|
558
|
+
* the comparison of two corpora by 132 features of a text over windows of the same size: Cliff's delta, Cohen's d and the AUC of every feature, the Mann-Whitney test with Holm's correction and a bootstrap interval of the difference of the medians
|
|
559
559
|
|
|
560
560
|
```python
|
|
561
561
|
>>> from ests import WordsExtractor
|
|
@@ -5,22 +5,14 @@
|
|
|
5
5
|
# URL: <https://github.com/SergeyShk/esTS>
|
|
6
6
|
|
|
7
7
|
import logging
|
|
8
|
+
import sys
|
|
8
9
|
from importlib.metadata import PackageNotFoundError, version
|
|
10
|
+
from typing import TYPE_CHECKING
|
|
11
|
+
|
|
12
|
+
from anyts.utils import make_lazy
|
|
9
13
|
|
|
10
14
|
from .basic_stats import BasicStats
|
|
11
15
|
from .cohesion_stats import CohesionStats
|
|
12
|
-
from .components import (
|
|
13
|
-
BasicStatsComponent,
|
|
14
|
-
CohesionStatsComponent,
|
|
15
|
-
DiversityStatsComponent,
|
|
16
|
-
LexicalStatsComponent,
|
|
17
|
-
MorphStatsComponent,
|
|
18
|
-
PhonStatsComponent,
|
|
19
|
-
ReadabilityStatsComponent,
|
|
20
|
-
StyleStatsComponent,
|
|
21
|
-
SyntaxStatsComponent,
|
|
22
|
-
VerseStatsComponent,
|
|
23
|
-
)
|
|
24
16
|
from .diversity_stats import DiversityStats
|
|
25
17
|
from .exceptions import (
|
|
26
18
|
DataFileError,
|
|
@@ -41,6 +33,20 @@ from .style_stats import StyleStats
|
|
|
41
33
|
from .syntax_stats import SyntaxStats
|
|
42
34
|
from .verse_stats import VerseStats
|
|
43
35
|
|
|
36
|
+
if TYPE_CHECKING:
|
|
37
|
+
from .components import (
|
|
38
|
+
BasicStatsComponent as BasicStatsComponent,
|
|
39
|
+
CohesionStatsComponent as CohesionStatsComponent,
|
|
40
|
+
DiversityStatsComponent as DiversityStatsComponent,
|
|
41
|
+
LexicalStatsComponent as LexicalStatsComponent,
|
|
42
|
+
MorphStatsComponent as MorphStatsComponent,
|
|
43
|
+
PhonStatsComponent as PhonStatsComponent,
|
|
44
|
+
ReadabilityStatsComponent as ReadabilityStatsComponent,
|
|
45
|
+
StyleStatsComponent as StyleStatsComponent,
|
|
46
|
+
SyntaxStatsComponent as SyntaxStatsComponent,
|
|
47
|
+
VerseStatsComponent as VerseStatsComponent,
|
|
48
|
+
)
|
|
49
|
+
|
|
44
50
|
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
|
45
51
|
|
|
46
52
|
# The distribution is pyests, the package it installs is ests
|
|
@@ -56,35 +62,51 @@ __author_email__ = "kouki.sergey@gmail.com"
|
|
|
56
62
|
|
|
57
63
|
__all__ = [
|
|
58
64
|
"BasicStats",
|
|
59
|
-
"BasicStatsComponent",
|
|
60
65
|
"CharNgramsExtractor",
|
|
61
66
|
"CohesionStats",
|
|
62
|
-
"CohesionStatsComponent",
|
|
63
67
|
"DataFileError",
|
|
64
68
|
"DatasetNotFoundError",
|
|
65
69
|
"DiversityStats",
|
|
66
|
-
"DiversityStatsComponent",
|
|
67
70
|
"DownloadError",
|
|
68
71
|
"EstsError",
|
|
69
72
|
"LexicalStats",
|
|
70
|
-
"LexicalStatsComponent",
|
|
71
73
|
"MorphStats",
|
|
72
|
-
"MorphStatsComponent",
|
|
73
74
|
"ParameterError",
|
|
74
75
|
"PhonStats",
|
|
75
|
-
"PhonStatsComponent",
|
|
76
76
|
"ReadabilityStats",
|
|
77
|
-
"ReadabilityStatsComponent",
|
|
78
77
|
"SentsExtractor",
|
|
79
78
|
"SourceError",
|
|
80
79
|
"SourceTypeError",
|
|
81
80
|
"StyleStats",
|
|
82
|
-
"StyleStatsComponent",
|
|
83
81
|
"SyntaxStats",
|
|
84
|
-
"SyntaxStatsComponent",
|
|
85
82
|
"UnknownStatError",
|
|
86
83
|
"VerseStats",
|
|
87
|
-
"VerseStatsComponent",
|
|
88
84
|
"WordsExtractor",
|
|
89
85
|
"__version__",
|
|
90
86
|
]
|
|
87
|
+
|
|
88
|
+
# The components register the factories of spaCy and import it: the module is loaded on first
|
|
89
|
+
# use, or at once when spaCy is loaded already, as it may have read the entry points before
|
|
90
|
+
# esTS was installed. The components are out of __all__, so import * and help() load no spaCy;
|
|
91
|
+
# a stub of spaCy is no spaCy
|
|
92
|
+
if sys.modules.get("spacy.language") is not None:
|
|
93
|
+
from . import components as components
|
|
94
|
+
|
|
95
|
+
make_lazy(
|
|
96
|
+
__name__,
|
|
97
|
+
dict.fromkeys(
|
|
98
|
+
(
|
|
99
|
+
"BasicStatsComponent",
|
|
100
|
+
"CohesionStatsComponent",
|
|
101
|
+
"DiversityStatsComponent",
|
|
102
|
+
"LexicalStatsComponent",
|
|
103
|
+
"MorphStatsComponent",
|
|
104
|
+
"PhonStatsComponent",
|
|
105
|
+
"ReadabilityStatsComponent",
|
|
106
|
+
"StyleStatsComponent",
|
|
107
|
+
"SyntaxStatsComponent",
|
|
108
|
+
"VerseStatsComponent",
|
|
109
|
+
),
|
|
110
|
+
"components",
|
|
111
|
+
),
|
|
112
|
+
)
|
|
@@ -95,8 +95,9 @@ class BasicStats(anyts.basic_stats.BasicStats):
|
|
|
95
95
|
Raises:
|
|
96
96
|
SourceTypeError: If the source is neither a string nor a Doc object, or an extractor
|
|
97
97
|
is of another type
|
|
98
|
-
SourceError: If the source has no words
|
|
99
|
-
ParameterError: If a factor is not an integer or is below one
|
|
98
|
+
SourceError: If the source has no words or is a string with a character without a UTF-8 form
|
|
99
|
+
ParameterError: If a factor is not an integer or is below one, or normalize is not
|
|
100
|
+
a bool
|
|
100
101
|
"""
|
|
101
102
|
|
|
102
103
|
sents_extractor_class = SentsExtractor
|
|
@@ -137,6 +138,7 @@ def punctuation_profile(text: str, n_words: int | None = None) -> dict[str, floa
|
|
|
137
138
|
|
|
138
139
|
Raises:
|
|
139
140
|
SourceTypeError: If the text is not a string
|
|
141
|
+
SourceError: If the text has a character without a UTF-8 form
|
|
140
142
|
ParameterError: If the number of words is not an integer or is negative
|
|
141
143
|
"""
|
|
142
144
|
if n_words is not None:
|