lixity 0.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lixity/__init__.py +29 -0
- lixity/__main__.py +5 -0
- lixity/_version.py +3 -0
- lixity/analyzer.py +569 -0
- lixity/api.py +497 -0
- lixity/characters.py +126 -0
- lixity/cli.py +1444 -0
- lixity/config.py +396 -0
- lixity/dialogue.py +181 -0
- lixity/diversity.py +189 -0
- lixity/format.py +44 -0
- lixity/formatters.py +615 -0
- lixity/io.py +165 -0
- lixity/language.py +225 -0
- lixity/language_data.py +4822 -0
- lixity/layout.py +237 -0
- lixity/markdown_parser.py +565 -0
- lixity/markers.py +140 -0
- lixity/math_core.py +108 -0
- lixity/models.py +288 -0
- lixity/motifs.py +168 -0
- lixity/nda.py +164 -0
- lixity/nda_templates.py +472 -0
- lixity/pacing.py +259 -0
- lixity/pdf_document.py +945 -0
- lixity/pdf_export.py +71 -0
- lixity/pdf_font.py +690 -0
- lixity/pdf_layouts.py +711 -0
- lixity/pipeline.py +74 -0
- lixity/project_identity.py +153 -0
- lixity/publication.py +157 -0
- lixity/publishing.py +113 -0
- lixity/py.typed +0 -0
- lixity/research/__init__.py +1 -0
- lixity/research/acknowledgements.py +41 -0
- lixity/research/analysis.py +603 -0
- lixity/research/api.py +1932 -0
- lixity/research/archive.py +261 -0
- lixity/research/catalogue.py +197 -0
- lixity/research/checkpoints.py +496 -0
- lixity/research/citation_selection.py +60 -0
- lixity/research/cli.py +926 -0
- lixity/research/editorial.py +147 -0
- lixity/research/images.py +316 -0
- lixity/research/limits.py +10 -0
- lixity/research/manuscript.py +225 -0
- lixity/research/models.py +461 -0
- lixity/research/ocr.py +642 -0
- lixity/research/presentation.py +256 -0
- lixity/research/repository.py +552 -0
- lixity/research/revisions.py +592 -0
- lixity/research/sections.py +108 -0
- lixity/research/timeline.py +140 -0
- lixity/research/zotero.py +445 -0
- lixity/research/zotero_backup.py +111 -0
- lixity/scenes.py +144 -0
- lixity/sentences.py +136 -0
- lixity/server/__init__.py +47 -0
- lixity/server/_base.py +82 -0
- lixity/server/_state.py +98 -0
- lixity/server/constants.py +28 -0
- lixity/server/discovery.py +31 -0
- lixity/server/handler.py +300 -0
- lixity/server/pdf_context.py +80 -0
- lixity/server/routes_manuscript_research.py +108 -0
- lixity/server/routes_markers.py +72 -0
- lixity/server/routes_nda.py +38 -0
- lixity/server/routes_pdf.py +78 -0
- lixity/server/routes_project.py +410 -0
- lixity/server/routes_research.py +829 -0
- lixity/server/runtime.py +119 -0
- lixity/server/views.py +246 -0
- lixity/service.py +828 -0
- lixity/showing.py +174 -0
- lixity/site.py +16 -0
- lixity/status.py +136 -0
- lixity/style_fingerprint.py +1645 -0
- lixity/style_profile.py +360 -0
- lixity/syllables.py +227 -0
- lixity/ui/__init__.py +12 -0
- lixity/ui/assets/dashboard.css +2348 -0
- lixity/ui/assets/dashboard.js +5524 -0
- lixity/ui/assets/design-tokens.css +68 -0
- lixity/ui/assets/dossier-images.css +25 -0
- lixity/ui/assets/dossier-images.js +341 -0
- lixity/ui/assets/manuscript-links.css +16 -0
- lixity/ui/assets/manuscript-links.js +265 -0
- lixity/ui/assets/style-space.js +550 -0
- lixity/ui/assets/workspace-reader.css +23 -0
- lixity/ui/assets/workspace-reader.js +110 -0
- lixity/ui/components.py +329 -0
- lixity/ui/dashboard.py +1320 -0
- lixity/ui/dimensions.py +202 -0
- lixity/ui/dossier_images.py +24 -0
- lixity/ui/heatmap.py +234 -0
- lixity/ui/markers_panel.py +64 -0
- lixity/ui/modals.py +291 -0
- lixity/ui/narration.py +396 -0
- lixity/ui/research_panel.py +312 -0
- lixity/ui/settings.py +194 -0
- lixity/ui/structural.py +193 -0
- lixity/workspace.py +204 -0
- lixity/workspace_labels.py +846 -0
- lixity-0.4.2.dist-info/METADATA +397 -0
- lixity-0.4.2.dist-info/RECORD +109 -0
- lixity-0.4.2.dist-info/WHEEL +5 -0
- lixity-0.4.2.dist-info/entry_points.txt +2 -0
- lixity-0.4.2.dist-info/licenses/LICENSE +661 -0
- lixity-0.4.2.dist-info/top_level.txt +1 -0
lixity/__init__.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Lixity – Quantitative text linguistics, stylometry and self-calibrating style passports."""
|
|
2
|
+
|
|
3
|
+
from ._version import __version__
|
|
4
|
+
from .analyzer import CorpusAnalyzer
|
|
5
|
+
from .formatters import ReportFormatter
|
|
6
|
+
from .io import FileUtils
|
|
7
|
+
from .models import (
|
|
8
|
+
ChapterMetrics,
|
|
9
|
+
CorpusAuditReport,
|
|
10
|
+
CorpusConfig,
|
|
11
|
+
CorpusMetrics,
|
|
12
|
+
DossierStatus,
|
|
13
|
+
SentenceDistribution,
|
|
14
|
+
)
|
|
15
|
+
from .style_fingerprint import StyleFingerprint
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"ChapterMetrics",
|
|
19
|
+
"CorpusAnalyzer",
|
|
20
|
+
"CorpusAuditReport",
|
|
21
|
+
"CorpusConfig",
|
|
22
|
+
"CorpusMetrics",
|
|
23
|
+
"DossierStatus",
|
|
24
|
+
"FileUtils",
|
|
25
|
+
"ReportFormatter",
|
|
26
|
+
"SentenceDistribution",
|
|
27
|
+
"StyleFingerprint",
|
|
28
|
+
"__version__",
|
|
29
|
+
]
|
lixity/__main__.py
ADDED
lixity/_version.py
ADDED
lixity/analyzer.py
ADDED
|
@@ -0,0 +1,569 @@
|
|
|
1
|
+
"""lixity.analyzer – High-performance text analysis and corpus linguistics engine.
|
|
2
|
+
|
|
3
|
+
Computes sentence-length architecture (ASL, CV, short/long sentence shares), lexical diversity
|
|
4
|
+
(TTR, Guiraud R, HD-D, MTLD, MATTR, Maas, Yule's K), language-calibrated readability
|
|
5
|
+
(Flesch family, LIX), dialogue ratios, register signals, and per-chapter metrics
|
|
6
|
+
with standard errors.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
from collections import Counter
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
|
|
15
|
+
from .diversity import (
|
|
16
|
+
hd_d as hd_d_value,
|
|
17
|
+
)
|
|
18
|
+
from .diversity import (
|
|
19
|
+
maas_a2 as maas_a2_value,
|
|
20
|
+
)
|
|
21
|
+
from .diversity import (
|
|
22
|
+
mattr as mattr_value,
|
|
23
|
+
)
|
|
24
|
+
from .diversity import (
|
|
25
|
+
mtld as mtld_value,
|
|
26
|
+
)
|
|
27
|
+
from .diversity import (
|
|
28
|
+
yules_k as yules_k_value,
|
|
29
|
+
)
|
|
30
|
+
from .language import compile_pattern, compile_word_pattern, count_nominal_matches, resolve_language
|
|
31
|
+
from .language_data import READABILITY
|
|
32
|
+
from .markdown_parser import chapter_heading_spans, prose_paragraphs, prose_text, split_chapters
|
|
33
|
+
from .math_core import binomial_se_percent, per_mille, poisson_se_per_mille
|
|
34
|
+
from .models import (
|
|
35
|
+
ChapterMetrics,
|
|
36
|
+
CorpusConfig,
|
|
37
|
+
CorpusMetrics,
|
|
38
|
+
SentenceDistribution,
|
|
39
|
+
)
|
|
40
|
+
from .sentences import (
|
|
41
|
+
LONG_SENTENCE_MAX,
|
|
42
|
+
MEDIUM_SENTENCE_MAX,
|
|
43
|
+
SHORT_SENTENCE_MAX,
|
|
44
|
+
split_sentences,
|
|
45
|
+
)
|
|
46
|
+
from .style_profile import dominance_from_hits
|
|
47
|
+
from .syllables import count_syllables as count_syllables_for
|
|
48
|
+
|
|
49
|
+
_RE_HTML_COMMENT = re.compile(r"<!--.*?-->", re.DOTALL)
|
|
50
|
+
_RE_HEADING_LINE = re.compile(r"(?m)^#+.*$")
|
|
51
|
+
_RE_DASHES = re.compile(r"[–—]")
|
|
52
|
+
_RE_ELLIPSES = re.compile(r"(?:…|\.{3})")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class SentenceStats:
|
|
57
|
+
"""Sentence-length architecture of a text (shared by corpus and chapters)."""
|
|
58
|
+
|
|
59
|
+
total: int
|
|
60
|
+
asl: float
|
|
61
|
+
median: int
|
|
62
|
+
median_exact: float
|
|
63
|
+
std: float
|
|
64
|
+
cv: float
|
|
65
|
+
distribution: SentenceDistribution
|
|
66
|
+
staccato_pct: float
|
|
67
|
+
kaskade_pct: float
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def share_se(pct: float, n: int) -> float:
|
|
71
|
+
"""Binomial standard error of a share in percentage points (plug-in)."""
|
|
72
|
+
return binomial_se_percent(pct, n)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def count_se(cnt: int, words: int) -> float:
|
|
76
|
+
"""Poisson standard error of a count density (per 1,000 words)."""
|
|
77
|
+
return poisson_se_per_mille(cnt, words)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class CorpusAnalyzer:
|
|
81
|
+
"""
|
|
82
|
+
Stateless, thread-safe analysis engine for literary Markdown texts.
|
|
83
|
+
Performs quantitative corpus linguistics, stylometry and chapter segmentation.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
def __init__(self, config: CorpusConfig | None = None, *, sample_text: str | None = None):
|
|
87
|
+
"""Initialises the analyzer with a configuration.
|
|
88
|
+
|
|
89
|
+
If no configuration is passed, the default parameters apply.
|
|
90
|
+
``sample_text`` lets an explicit ``language="auto"`` resolve against
|
|
91
|
+
the text to analyse; without it ``auto`` falls back to ``generic``
|
|
92
|
+
(documented in docs/LOCALIZATION.md).
|
|
93
|
+
"""
|
|
94
|
+
self.config = config or CorpusConfig()
|
|
95
|
+
self.lang = resolve_language(self.config, sample_text=sample_text)
|
|
96
|
+
self._praes_re = compile_pattern(self.lang.praesens_regex)
|
|
97
|
+
self._praet_re = compile_pattern(self.lang.praeteritum_regex)
|
|
98
|
+
self._filter_re = compile_pattern(self.lang.filter_verbs_regex)
|
|
99
|
+
self._word_re = compile_word_pattern(self.lang.word_regex)
|
|
100
|
+
self._dialogue_re = compile_pattern(self.lang.dialogue_regex)
|
|
101
|
+
# Style heuristics (self-calibrating house-style fingerprint)
|
|
102
|
+
self._passive_re = compile_pattern(self.lang.passive_regex)
|
|
103
|
+
self._nominal_re = compile_pattern(self.lang.nominal_regex)
|
|
104
|
+
self._nominal_exclusions = (
|
|
105
|
+
self.lang.function_words if self.config.nominal_regex is None else frozenset()
|
|
106
|
+
)
|
|
107
|
+
self._adjective_re = compile_pattern(self.lang.adjective_regex)
|
|
108
|
+
self._modals = frozenset(w.lower() for w in self.lang.lexicon.get("modals", ()))
|
|
109
|
+
self._starters = frozenset(w.lower() for w in self.lang.first_person_starters)
|
|
110
|
+
self._content_blacklist = self.lang.function_words | self.lang.stopwords
|
|
111
|
+
|
|
112
|
+
@staticmethod
|
|
113
|
+
def _entropy(counts: Counter[str]) -> float:
|
|
114
|
+
"""Shannon entropy in bits over a token-type counter."""
|
|
115
|
+
total = sum(counts.values())
|
|
116
|
+
if not total:
|
|
117
|
+
return 0.0
|
|
118
|
+
return math.fsum(-(c / total) * math.log2(c / total) for c in counts.values())
|
|
119
|
+
|
|
120
|
+
def _starter_stats(self, sentences: list[str]) -> tuple[float, float, float]:
|
|
121
|
+
"""(start entropy in bits, first-person-start rate, entropy standard error).
|
|
122
|
+
|
|
123
|
+
Entropy uncertainty uses a first-order plug-in variance approximation:
|
|
124
|
+
Var(H) = [sum(p*log2^2(p)) - H^2] / n. It is not a calibrated interval.
|
|
125
|
+
"""
|
|
126
|
+
starters: Counter[str] = Counter()
|
|
127
|
+
first_person = 0
|
|
128
|
+
for s in sentences:
|
|
129
|
+
tokens = self._word_re.findall(s)
|
|
130
|
+
if not tokens:
|
|
131
|
+
continue
|
|
132
|
+
# Elision-aware match (fr: "J'aime" -> "je"/"j", en: "I'm" -> "i")
|
|
133
|
+
first = tokens[0].lower()
|
|
134
|
+
stem = first.split("'", 1)[0]
|
|
135
|
+
starters[first] += 1
|
|
136
|
+
if first in self._starters or stem in self._starters:
|
|
137
|
+
first_person += 1
|
|
138
|
+
total = sum(starters.values())
|
|
139
|
+
rate = (first_person / total * 100.0) if total else 0.0
|
|
140
|
+
entropy = self._entropy(starters)
|
|
141
|
+
if not total:
|
|
142
|
+
return entropy, rate, 0.0
|
|
143
|
+
second = math.fsum(
|
|
144
|
+
(c / total) * math.log2(c / total) ** 2 for c in starters.values()
|
|
145
|
+
)
|
|
146
|
+
variance = (second - entropy**2) / total
|
|
147
|
+
return entropy, rate, math.sqrt(max(0.0, variance))
|
|
148
|
+
|
|
149
|
+
@staticmethod
|
|
150
|
+
def _density(matches: int, words: int) -> float:
|
|
151
|
+
"""Occurrences per 1,000 tokens (length-comparable density)."""
|
|
152
|
+
return per_mille(matches, words)
|
|
153
|
+
|
|
154
|
+
def _jsd_chapter(
|
|
155
|
+
self,
|
|
156
|
+
chapter_counter: Counter[str],
|
|
157
|
+
corpus_counter: Counter[str],
|
|
158
|
+
n_chapter: int,
|
|
159
|
+
n_corpus: int,
|
|
160
|
+
top_n: int = 5,
|
|
161
|
+
) -> tuple[float, list[str]]:
|
|
162
|
+
"""
|
|
163
|
+
Jensen-Shannon divergence of the chapter's word distribution to the rest
|
|
164
|
+
of the corpus (leave-one-out, so small chapters are not self-biased).
|
|
165
|
+
|
|
166
|
+
Returns (jsd, top driver words): contributions are additive per type,
|
|
167
|
+
so the most divergent content words are interpretable.
|
|
168
|
+
|
|
169
|
+
Only the chapter's own vocabulary is iterated. Types the chapter does
|
|
170
|
+
not contain contribute exactly ``q * ln(2) / 2`` each, so their total
|
|
171
|
+
is ``ln(2)/2 * (1 - covered_rest_mass)`` in closed form – an O(chapter
|
|
172
|
+
types) computation instead of O(corpus types).
|
|
173
|
+
"""
|
|
174
|
+
rest = n_corpus - n_chapter
|
|
175
|
+
if rest < 1 or n_chapter < 1:
|
|
176
|
+
return 0.0, []
|
|
177
|
+
jsd = 0.0
|
|
178
|
+
contribs: dict[str, float] = {}
|
|
179
|
+
covered_rest_mass = 0.0
|
|
180
|
+
for w in sorted(chapter_counter): # deterministic order: bit-identical across runs
|
|
181
|
+
p = chapter_counter[w] / n_chapter
|
|
182
|
+
q = (corpus_counter[w] - chapter_counter[w]) / rest
|
|
183
|
+
covered_rest_mass += q
|
|
184
|
+
m = 0.5 * (p + q)
|
|
185
|
+
term = p * math.log(p / m)
|
|
186
|
+
if q > 0.0:
|
|
187
|
+
term += q * math.log(q / m)
|
|
188
|
+
contrib = 0.5 * term
|
|
189
|
+
contribs[w] = contrib
|
|
190
|
+
jsd += contrib
|
|
191
|
+
# Types absent from the chapter: p = 0, m = q/2 -> contribution q*ln(2)/2.
|
|
192
|
+
jsd += 0.5 * math.log(2.0) * max(0.0, 1.0 - covered_rest_mass)
|
|
193
|
+
top = [
|
|
194
|
+
w
|
|
195
|
+
for w, _ in sorted(contribs.items(), key=lambda kv: kv[1], reverse=True)
|
|
196
|
+
if len(w) > 1 and w not in self._content_blacklist
|
|
197
|
+
][:top_n]
|
|
198
|
+
return jsd, top
|
|
199
|
+
|
|
200
|
+
def count_syllables(self, word: str) -> int:
|
|
201
|
+
"""Language-sensitive syllable counting (see :mod:`lixity.syllables`)."""
|
|
202
|
+
return count_syllables_for(word, self.lang.key)
|
|
203
|
+
|
|
204
|
+
def readability(self, asl: float, asw: float) -> tuple[float, str]:
|
|
205
|
+
"""Language-calibrated Reading Ease score and formula name; not clipped to 0–100."""
|
|
206
|
+
rd = READABILITY.get(self.lang.key, READABILITY["generic"])
|
|
207
|
+
if asl == 0.0 and asw == 0.0:
|
|
208
|
+
return 0.0, rd["name"]
|
|
209
|
+
score = rd["constant"] - rd["asl_coef"] * asl - rd["asw_coef"] * asw
|
|
210
|
+
return score, rd["name"]
|
|
211
|
+
|
|
212
|
+
def long_word_min(self) -> int:
|
|
213
|
+
"""Language-calibrated minimum letter count for LIX 'long words' (Björnsson)."""
|
|
214
|
+
rd = READABILITY.get(self.lang.key, READABILITY["generic"])
|
|
215
|
+
return int(rd["long_word_min"])
|
|
216
|
+
|
|
217
|
+
def _sentence_stats(self, sentences: list[str]) -> SentenceStats:
|
|
218
|
+
"""Sentence-length architecture (lengths, ASL, median, CV, distribution)."""
|
|
219
|
+
lengths = [n for n in (len(self._word_re.findall(s)) for s in sentences) if n > 0]
|
|
220
|
+
total = len(lengths)
|
|
221
|
+
asl = sum(lengths) / total if total else 0.0
|
|
222
|
+
sorted_lengths = sorted(lengths)
|
|
223
|
+
median_sl = sorted_lengths[total // 2] if total else 0
|
|
224
|
+
median_exact = (
|
|
225
|
+
(sorted_lengths[(total - 1) // 2] + sorted_lengths[total // 2]) / 2.0
|
|
226
|
+
if total
|
|
227
|
+
else 0.0
|
|
228
|
+
)
|
|
229
|
+
variance = sum((x - asl) ** 2 for x in lengths) / total if total else 0.0
|
|
230
|
+
std = math.sqrt(variance)
|
|
231
|
+
short = sum(1 for x in lengths if x <= SHORT_SENTENCE_MAX)
|
|
232
|
+
medium = sum(1 for x in lengths if SHORT_SENTENCE_MAX < x <= MEDIUM_SENTENCE_MAX)
|
|
233
|
+
long_ = sum(1 for x in lengths if MEDIUM_SENTENCE_MAX < x <= LONG_SENTENCE_MAX)
|
|
234
|
+
complex_ = sum(1 for x in lengths if x > LONG_SENTENCE_MAX)
|
|
235
|
+
|
|
236
|
+
def _pct(count: int) -> float:
|
|
237
|
+
return (count / total * 100.0) if total else 0.0
|
|
238
|
+
|
|
239
|
+
return SentenceStats(
|
|
240
|
+
total=total,
|
|
241
|
+
asl=asl,
|
|
242
|
+
median=median_sl,
|
|
243
|
+
median_exact=median_exact,
|
|
244
|
+
std=std,
|
|
245
|
+
cv=(std / asl) if asl else 0.0,
|
|
246
|
+
distribution=SentenceDistribution(
|
|
247
|
+
short_count=short,
|
|
248
|
+
short_pct=_pct(short),
|
|
249
|
+
medium_count=medium,
|
|
250
|
+
medium_pct=_pct(medium),
|
|
251
|
+
long_count=long_,
|
|
252
|
+
long_pct=_pct(long_),
|
|
253
|
+
complex_count=complex_,
|
|
254
|
+
complex_pct=_pct(complex_),
|
|
255
|
+
),
|
|
256
|
+
staccato_pct=_pct(short),
|
|
257
|
+
kaskade_pct=_pct(complex_),
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
def _paragraph_stats(self, paragraphs: list[str]) -> tuple[int, float, int]:
|
|
261
|
+
"""(prose paragraphs, average words per paragraph, one-liner count)."""
|
|
262
|
+
lengths = [len(p.split()) for p in paragraphs]
|
|
263
|
+
total = len(lengths)
|
|
264
|
+
average = sum(lengths) / total if total else 0.0
|
|
265
|
+
one_liners = sum(
|
|
266
|
+
1
|
|
267
|
+
for length in lengths
|
|
268
|
+
if length <= self.config.min_paragraph_length_for_oneliner
|
|
269
|
+
)
|
|
270
|
+
return total, average, one_liners
|
|
271
|
+
|
|
272
|
+
def _punctuation_profile(self, text: str) -> dict[str, int]:
|
|
273
|
+
"""Language-neutral punctuation counts for the JSON contract."""
|
|
274
|
+
return {
|
|
275
|
+
"periods": text.count("."),
|
|
276
|
+
"commas": text.count(","),
|
|
277
|
+
"dashes": len(_RE_DASHES.findall(text)),
|
|
278
|
+
"colons": text.count(":"),
|
|
279
|
+
"semicolons": text.count(";"),
|
|
280
|
+
"questions": text.count("?"),
|
|
281
|
+
"exclamations": text.count("!"),
|
|
282
|
+
"ellipses": len(_RE_ELLIPSES.findall(text)),
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
def measure_unit(self, text: str) -> ChapterMetrics | None:
|
|
286
|
+
"""Measure one prose unit using the chapter feature engine once.
|
|
287
|
+
|
|
288
|
+
This omits corpus-level readability, cross-chapter divergence and title
|
|
289
|
+
parsing. It is useful for scenes already segmented by the shared parser.
|
|
290
|
+
"""
|
|
291
|
+
return self._chapter_metrics(1, "", text, self.long_word_min())[0]
|
|
292
|
+
|
|
293
|
+
def _chapter_metrics(
|
|
294
|
+
self, num: int, title: str, body: str, lw_min: int
|
|
295
|
+
) -> tuple[ChapterMetrics | None, list[str]]:
|
|
296
|
+
"""Per-chapter metrics (style features, uncertainty, tense, JSD input)."""
|
|
297
|
+
cl_b = prose_text(body, _front_matter=False)
|
|
298
|
+
c_words = self._word_re.findall(cl_b)
|
|
299
|
+
if not c_words:
|
|
300
|
+
return None, []
|
|
301
|
+
|
|
302
|
+
n_cw = len(c_words)
|
|
303
|
+
c_lower = [w.lower() for w in c_words]
|
|
304
|
+
c_sentences = split_sentences(cl_b, self.lang.key)
|
|
305
|
+
stats = self._sentence_stats(c_sentences)
|
|
306
|
+
|
|
307
|
+
c_dial = self._dialogue_re.findall(cl_b)
|
|
308
|
+
c_dial_words = sum(len(self._word_re.findall(m)) for m in c_dial)
|
|
309
|
+
c_dial_pct = (c_dial_words / n_cw) * 100.0 if n_cw else 0.0
|
|
310
|
+
c_ttr = len(set(c_lower)) / n_cw if n_cw else 0.0
|
|
311
|
+
c_signals = {
|
|
312
|
+
s_name: len(re.findall(s_pat, cl_b, re.IGNORECASE))
|
|
313
|
+
for s_name, s_pat in self.lang.signal_keywords.items()
|
|
314
|
+
}
|
|
315
|
+
c_motifs = {
|
|
316
|
+
m_name: len(re.findall(m_pat, cl_b, re.IGNORECASE))
|
|
317
|
+
for m_name, m_pat in self.config.motif_regexes.items()
|
|
318
|
+
}
|
|
319
|
+
c_fil = len(self._filter_re.findall(cl_b))
|
|
320
|
+
dominance = dominance_from_hits(
|
|
321
|
+
len(self._praes_re.findall(cl_b)), len(self._praet_re.findall(cl_b))
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
c_start_entropy, c_first_rate, c_entropy_se = self._starter_stats(c_sentences)
|
|
325
|
+
c_passive_cnt = len(self._passive_re.findall(cl_b))
|
|
326
|
+
c_nominal_cnt = count_nominal_matches(cl_b, self._nominal_re, self._nominal_exclusions)
|
|
327
|
+
c_adjective_cnt = len(self._adjective_re.findall(cl_b))
|
|
328
|
+
c_modal_cnt = sum(1 for t in c_lower if t in self._modals)
|
|
329
|
+
c_long_words = sum(1 for t in c_words if len(t) > lw_min)
|
|
330
|
+
c_long_pct = (c_long_words / n_cw * 100.0) if n_cw else 0.0
|
|
331
|
+
c_guiraud = len(set(c_lower)) / math.sqrt(n_cw) if n_cw else 0.0
|
|
332
|
+
c_hd_d = hd_d_value(c_lower)
|
|
333
|
+
c_func_pct = (
|
|
334
|
+
sum(1 for t in c_lower if t in self.lang.function_words) / n_cw * 100.0 if n_cw else 0.0
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
# Measurement uncertainty per feature (documented plug-ins)
|
|
338
|
+
c_se: dict[str, float] = {
|
|
339
|
+
"asl": (stats.std / math.sqrt(stats.total)) if stats.total else 0.0,
|
|
340
|
+
"staccato_pct": share_se(stats.staccato_pct, stats.total),
|
|
341
|
+
"kaskade_pct": share_se(stats.kaskade_pct, stats.total),
|
|
342
|
+
"sentence_cv": (
|
|
343
|
+
(stats.cv / math.sqrt(2.0 * stats.total)) * math.sqrt(1.0 + 2.0 * stats.cv**2)
|
|
344
|
+
if stats.total
|
|
345
|
+
else 0.0
|
|
346
|
+
),
|
|
347
|
+
"dialog_pct": share_se(c_dial_pct, n_cw),
|
|
348
|
+
"function_word_pct": share_se(c_func_pct, n_cw),
|
|
349
|
+
"filter_density": count_se(c_fil, n_cw),
|
|
350
|
+
"modal_density": count_se(c_modal_cnt, n_cw),
|
|
351
|
+
"passive_density": count_se(c_passive_cnt, n_cw),
|
|
352
|
+
"nominalization_density": count_se(c_nominal_cnt, n_cw),
|
|
353
|
+
"adjective_density": count_se(c_adjective_cnt, n_cw),
|
|
354
|
+
"long_word_pct": share_se(c_long_pct, n_cw),
|
|
355
|
+
"start_entropy": c_entropy_se,
|
|
356
|
+
"first_person_start_rate": share_se(c_first_rate, stats.total),
|
|
357
|
+
"guiraud_r": (0.5 * c_guiraud / math.sqrt(n_cw)) if n_cw else 0.0,
|
|
358
|
+
}
|
|
359
|
+
# Exact rarefaction has no Monte Carlo SE; population uncertainty is not estimated.
|
|
360
|
+
|
|
361
|
+
chapter = ChapterMetrics(
|
|
362
|
+
num=num,
|
|
363
|
+
title=title,
|
|
364
|
+
words=n_cw,
|
|
365
|
+
sentences=stats.total,
|
|
366
|
+
asl=stats.asl,
|
|
367
|
+
dialog_pct=c_dial_pct,
|
|
368
|
+
ttr=c_ttr,
|
|
369
|
+
motif_counts=c_motifs,
|
|
370
|
+
filter_verbs=c_fil,
|
|
371
|
+
dominance=dominance,
|
|
372
|
+
signal_matches=c_signals,
|
|
373
|
+
staccato_pct=stats.staccato_pct,
|
|
374
|
+
kaskade_pct=stats.kaskade_pct,
|
|
375
|
+
sentence_cv=stats.cv,
|
|
376
|
+
start_entropy=c_start_entropy,
|
|
377
|
+
first_person_start_rate=c_first_rate,
|
|
378
|
+
passive_density=self._density(c_passive_cnt, n_cw),
|
|
379
|
+
nominalization_density=self._density(c_nominal_cnt, n_cw),
|
|
380
|
+
adjective_density=self._density(c_adjective_cnt, n_cw),
|
|
381
|
+
modal_density=self._density(c_modal_cnt, n_cw),
|
|
382
|
+
filter_density=self._density(c_fil, n_cw),
|
|
383
|
+
long_word_pct=c_long_pct,
|
|
384
|
+
guiraud_r=c_guiraud,
|
|
385
|
+
hd_d=c_hd_d,
|
|
386
|
+
function_word_pct=c_func_pct,
|
|
387
|
+
style_se=c_se,
|
|
388
|
+
)
|
|
389
|
+
return chapter, c_lower
|
|
390
|
+
|
|
391
|
+
def analyze_text(self, full_text: str) -> CorpusMetrics:
|
|
392
|
+
"""
|
|
393
|
+
Performs the complete stylometric and quantitative analysis on
|
|
394
|
+
a passed text string.
|
|
395
|
+
|
|
396
|
+
Workflow:
|
|
397
|
+
1. Separation of the scholarly appendix (single source of truth).
|
|
398
|
+
2. Filtering of Markdown HTML comments (<!-- ... -->).
|
|
399
|
+
3. Tokenisation and capture of vocabulary types.
|
|
400
|
+
4. Computation of lexical diversity metrics (TTR, Guiraud, Yule).
|
|
401
|
+
5. Sentence segmentation and sentence-length architecture (without heading artefacts).
|
|
402
|
+
6. Readability indices (Flesch family, LIX).
|
|
403
|
+
7. Dialogue and paragraph economy.
|
|
404
|
+
8. Punctuation and signal word frequencies.
|
|
405
|
+
9. Chapter-wise segmentation and tense classification.
|
|
406
|
+
|
|
407
|
+
Returns:
|
|
408
|
+
CorpusMetrics object with all computed metrics.
|
|
409
|
+
"""
|
|
410
|
+
# 1. Main text vs. appendix, comments removed
|
|
411
|
+
if self.config.appendix_marker and self.config.appendix_marker in full_text:
|
|
412
|
+
main_text, _ = full_text.split(self.config.appendix_marker, 1)
|
|
413
|
+
else:
|
|
414
|
+
main_text = full_text
|
|
415
|
+
cleaned_full = _RE_HTML_COMMENT.sub("", full_text)
|
|
416
|
+
cleaned_main = _RE_HTML_COMMENT.sub("", main_text)
|
|
417
|
+
# Configured chapter headings are metadata, just like Markdown headings.
|
|
418
|
+
# Retain line separators so removing one never joins adjacent prose.
|
|
419
|
+
prose_parts: list[str] = []
|
|
420
|
+
cursor = 0
|
|
421
|
+
for start, end, _line, _title in chapter_heading_spans(cleaned_main, self.config):
|
|
422
|
+
prose_parts.append(cleaned_main[cursor:start])
|
|
423
|
+
prose_parts.append("\n" if cleaned_main[end - 1:end] == "\n" else "")
|
|
424
|
+
cursor = end
|
|
425
|
+
prose_parts.append(cleaned_main[cursor:])
|
|
426
|
+
legacy_clean_main = _RE_HEADING_LINE.sub("", "".join(prose_parts))
|
|
427
|
+
|
|
428
|
+
raw_words = len(cleaned_full.split())
|
|
429
|
+
clean_words = len(legacy_clean_main.split())
|
|
430
|
+
raw_chars = len(cleaned_full)
|
|
431
|
+
clean_chars = len(legacy_clean_main)
|
|
432
|
+
# Token-based measurements share the supported semantic-block scope.
|
|
433
|
+
# Legacy whitespace words/chars intentionally retain their own contract.
|
|
434
|
+
prose_units = prose_paragraphs(main_text, self.config)
|
|
435
|
+
prose_main = "\n\n".join(prose_units)
|
|
436
|
+
|
|
437
|
+
# 2. Tokenisation
|
|
438
|
+
tokens = self._word_re.findall(prose_main)
|
|
439
|
+
n_tokens = len(tokens)
|
|
440
|
+
lower_tokens = [t.lower() for t in tokens]
|
|
441
|
+
v_types = len(set(lower_tokens))
|
|
442
|
+
freqs = Counter(lower_tokens)
|
|
443
|
+
|
|
444
|
+
# 3. Lexical range & stability
|
|
445
|
+
ttr = v_types / n_tokens if n_tokens else 0.0
|
|
446
|
+
guiraud_r = v_types / math.sqrt(n_tokens) if n_tokens else 0.0
|
|
447
|
+
|
|
448
|
+
# 4. Sentence-length architecture on the same prose as other metrics.
|
|
449
|
+
sentences = split_sentences(prose_main, self.lang.key)
|
|
450
|
+
stats = self._sentence_stats(sentences)
|
|
451
|
+
start_entropy, first_person_start_rate, _entropy_se = self._starter_stats(sentences)
|
|
452
|
+
|
|
453
|
+
# 5. Readability & complexity (language-calibrated Flesch family + LIX)
|
|
454
|
+
total_syllables = sum(self.count_syllables(t) * count for t, count in freqs.items())
|
|
455
|
+
asw = total_syllables / n_tokens if n_tokens else 0.0
|
|
456
|
+
if n_tokens == 0 or stats.total == 0:
|
|
457
|
+
flesch_de = 0.0
|
|
458
|
+
flesch_variant = self.readability(0.0, 0.0)[1]
|
|
459
|
+
lix = 0.0
|
|
460
|
+
else:
|
|
461
|
+
flesch_de, flesch_variant = self.readability(stats.asl, asw)
|
|
462
|
+
lw_min = self.long_word_min()
|
|
463
|
+
long_words = sum(1 for t in tokens if len(t) > lw_min)
|
|
464
|
+
pct_long_words = (long_words / n_tokens) * 100.0 if n_tokens else 0.0
|
|
465
|
+
lix = stats.asl + pct_long_words
|
|
466
|
+
|
|
467
|
+
# 6. Dialogue ratio
|
|
468
|
+
dialog_matches = self._dialogue_re.findall(prose_main)
|
|
469
|
+
dialog_words = sum(len(self._word_re.findall(m)) for m in dialog_matches)
|
|
470
|
+
dialog_ratio = (dialog_words / n_tokens) * 100.0 if n_tokens else 0.0
|
|
471
|
+
|
|
472
|
+
# 7. Paragraph economy, punctuation, signals
|
|
473
|
+
total_paras, avg_para_len, single_line_paras = self._paragraph_stats(prose_units)
|
|
474
|
+
punctuation = self._punctuation_profile(prose_main)
|
|
475
|
+
signal_counts = {
|
|
476
|
+
name: len(re.findall(pat, prose_main, re.IGNORECASE))
|
|
477
|
+
for name, pat in self.lang.signal_keywords.items()
|
|
478
|
+
}
|
|
479
|
+
filter_cnt = len(self._filter_re.findall(prose_main))
|
|
480
|
+
|
|
481
|
+
# 8. Style densities (per 1,000 tokens, length-comparable)
|
|
482
|
+
passive_density = self._density(len(self._passive_re.findall(prose_main)), n_tokens)
|
|
483
|
+
nominalization_density = self._density(
|
|
484
|
+
count_nominal_matches(prose_main, self._nominal_re, self._nominal_exclusions), n_tokens
|
|
485
|
+
)
|
|
486
|
+
adjective_density = self._density(len(self._adjective_re.findall(prose_main)), n_tokens)
|
|
487
|
+
modal_density = self._density(sum(1 for t in lower_tokens if t in self._modals), n_tokens)
|
|
488
|
+
filter_density = self._density(filter_cnt, n_tokens)
|
|
489
|
+
|
|
490
|
+
# 9. Chapter-wise segmentation (shared split_chapters: front matter +
|
|
491
|
+
# appendix + empty-body rules identical to the structure modules).
|
|
492
|
+
chapters: list[ChapterMetrics] = []
|
|
493
|
+
chapter_tokens: list[list[str]] = []
|
|
494
|
+
for _split_num, title, body in split_chapters(main_text, self.config):
|
|
495
|
+
chapter, c_lower = self._chapter_metrics(len(chapters) + 1, title, body, lw_min)
|
|
496
|
+
if chapter is None:
|
|
497
|
+
continue
|
|
498
|
+
chapters.append(chapter)
|
|
499
|
+
chapter_tokens.append(c_lower)
|
|
500
|
+
|
|
501
|
+
# 10. Jensen-Shannon divergence per chapter vs. the rest (leave-one-out)
|
|
502
|
+
for chapter, c_lower in zip(chapters, chapter_tokens, strict=False):
|
|
503
|
+
chapter.jsd, chapter.jsd_top_words = self._jsd_chapter(
|
|
504
|
+
Counter(c_lower), freqs, len(c_lower), n_tokens
|
|
505
|
+
)
|
|
506
|
+
|
|
507
|
+
return CorpusMetrics(
|
|
508
|
+
raw_words=raw_words,
|
|
509
|
+
clean_words=clean_words,
|
|
510
|
+
raw_chars=raw_chars,
|
|
511
|
+
clean_chars=clean_chars,
|
|
512
|
+
tokens=n_tokens,
|
|
513
|
+
vocab_types=v_types,
|
|
514
|
+
ttr=ttr,
|
|
515
|
+
guiraud_r=guiraud_r,
|
|
516
|
+
yules_k=yules_k_value(lower_tokens),
|
|
517
|
+
total_sentences=stats.total,
|
|
518
|
+
asl=stats.asl,
|
|
519
|
+
median_sl=stats.median,
|
|
520
|
+
median_sl_exact=stats.median_exact,
|
|
521
|
+
std_sl=stats.std,
|
|
522
|
+
sentence_dist=stats.distribution,
|
|
523
|
+
asw=asw,
|
|
524
|
+
flesch_de=flesch_de,
|
|
525
|
+
flesch_variant=flesch_variant,
|
|
526
|
+
lix=lix,
|
|
527
|
+
dialog_words=dialog_words,
|
|
528
|
+
dialog_ratio=dialog_ratio,
|
|
529
|
+
total_paragraphs=total_paras,
|
|
530
|
+
avg_paragraph_len=avg_para_len,
|
|
531
|
+
single_line_paragraphs=single_line_paras,
|
|
532
|
+
punctuation=punctuation,
|
|
533
|
+
signal_counts=signal_counts,
|
|
534
|
+
filter_count=filter_cnt,
|
|
535
|
+
chapters=chapters,
|
|
536
|
+
staccato_pct=stats.staccato_pct,
|
|
537
|
+
kaskade_pct=stats.kaskade_pct,
|
|
538
|
+
sentence_cv=stats.cv,
|
|
539
|
+
start_entropy=start_entropy,
|
|
540
|
+
first_person_start_rate=first_person_start_rate,
|
|
541
|
+
passive_density=passive_density,
|
|
542
|
+
nominalization_density=nominalization_density,
|
|
543
|
+
adjective_density=adjective_density,
|
|
544
|
+
modal_density=modal_density,
|
|
545
|
+
filter_density=filter_density,
|
|
546
|
+
hd_d=hd_d_value(lower_tokens),
|
|
547
|
+
mtld=mtld_value(lower_tokens),
|
|
548
|
+
mattr=mattr_value(lower_tokens),
|
|
549
|
+
maas_a2=maas_a2_value(n_tokens, v_types),
|
|
550
|
+
)
|
|
551
|
+
|
|
552
|
+
def analyze_file(self, filepath: str) -> CorpusMetrics:
|
|
553
|
+
"""
|
|
554
|
+
Reads a UTF-8 text file and delegates to analyze_text.
|
|
555
|
+
|
|
556
|
+
Parameters:
|
|
557
|
+
filepath: Absolute or relative file path to the Markdown manuscript.
|
|
558
|
+
|
|
559
|
+
Returns:
|
|
560
|
+
CorpusMetrics object.
|
|
561
|
+
|
|
562
|
+
Raises:
|
|
563
|
+
FileNotFoundError: If the file does not exist at the given path.
|
|
564
|
+
"""
|
|
565
|
+
if not os.path.isfile(filepath):
|
|
566
|
+
raise FileNotFoundError(f"Manuscript file not found: {filepath}")
|
|
567
|
+
with open(filepath, encoding="utf-8") as f:
|
|
568
|
+
content = f.read()
|
|
569
|
+
return self.analyze_text(content)
|