lixity 0.4.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. lixity/__init__.py +29 -0
  2. lixity/__main__.py +5 -0
  3. lixity/_version.py +3 -0
  4. lixity/analyzer.py +569 -0
  5. lixity/api.py +497 -0
  6. lixity/characters.py +126 -0
  7. lixity/cli.py +1444 -0
  8. lixity/config.py +396 -0
  9. lixity/dialogue.py +181 -0
  10. lixity/diversity.py +189 -0
  11. lixity/format.py +44 -0
  12. lixity/formatters.py +615 -0
  13. lixity/io.py +165 -0
  14. lixity/language.py +225 -0
  15. lixity/language_data.py +4822 -0
  16. lixity/layout.py +237 -0
  17. lixity/markdown_parser.py +565 -0
  18. lixity/markers.py +140 -0
  19. lixity/math_core.py +108 -0
  20. lixity/models.py +288 -0
  21. lixity/motifs.py +168 -0
  22. lixity/nda.py +164 -0
  23. lixity/nda_templates.py +472 -0
  24. lixity/pacing.py +259 -0
  25. lixity/pdf_document.py +945 -0
  26. lixity/pdf_export.py +71 -0
  27. lixity/pdf_font.py +690 -0
  28. lixity/pdf_layouts.py +711 -0
  29. lixity/pipeline.py +74 -0
  30. lixity/project_identity.py +153 -0
  31. lixity/publication.py +157 -0
  32. lixity/publishing.py +113 -0
  33. lixity/py.typed +0 -0
  34. lixity/research/__init__.py +1 -0
  35. lixity/research/acknowledgements.py +41 -0
  36. lixity/research/analysis.py +603 -0
  37. lixity/research/api.py +1932 -0
  38. lixity/research/archive.py +261 -0
  39. lixity/research/catalogue.py +197 -0
  40. lixity/research/checkpoints.py +496 -0
  41. lixity/research/citation_selection.py +60 -0
  42. lixity/research/cli.py +926 -0
  43. lixity/research/editorial.py +147 -0
  44. lixity/research/images.py +316 -0
  45. lixity/research/limits.py +10 -0
  46. lixity/research/manuscript.py +225 -0
  47. lixity/research/models.py +461 -0
  48. lixity/research/ocr.py +642 -0
  49. lixity/research/presentation.py +256 -0
  50. lixity/research/repository.py +552 -0
  51. lixity/research/revisions.py +592 -0
  52. lixity/research/sections.py +108 -0
  53. lixity/research/timeline.py +140 -0
  54. lixity/research/zotero.py +445 -0
  55. lixity/research/zotero_backup.py +111 -0
  56. lixity/scenes.py +144 -0
  57. lixity/sentences.py +136 -0
  58. lixity/server/__init__.py +47 -0
  59. lixity/server/_base.py +82 -0
  60. lixity/server/_state.py +98 -0
  61. lixity/server/constants.py +28 -0
  62. lixity/server/discovery.py +31 -0
  63. lixity/server/handler.py +300 -0
  64. lixity/server/pdf_context.py +80 -0
  65. lixity/server/routes_manuscript_research.py +108 -0
  66. lixity/server/routes_markers.py +72 -0
  67. lixity/server/routes_nda.py +38 -0
  68. lixity/server/routes_pdf.py +78 -0
  69. lixity/server/routes_project.py +410 -0
  70. lixity/server/routes_research.py +829 -0
  71. lixity/server/runtime.py +119 -0
  72. lixity/server/views.py +246 -0
  73. lixity/service.py +828 -0
  74. lixity/showing.py +174 -0
  75. lixity/site.py +16 -0
  76. lixity/status.py +136 -0
  77. lixity/style_fingerprint.py +1645 -0
  78. lixity/style_profile.py +360 -0
  79. lixity/syllables.py +227 -0
  80. lixity/ui/__init__.py +12 -0
  81. lixity/ui/assets/dashboard.css +2348 -0
  82. lixity/ui/assets/dashboard.js +5524 -0
  83. lixity/ui/assets/design-tokens.css +68 -0
  84. lixity/ui/assets/dossier-images.css +25 -0
  85. lixity/ui/assets/dossier-images.js +341 -0
  86. lixity/ui/assets/manuscript-links.css +16 -0
  87. lixity/ui/assets/manuscript-links.js +265 -0
  88. lixity/ui/assets/style-space.js +550 -0
  89. lixity/ui/assets/workspace-reader.css +23 -0
  90. lixity/ui/assets/workspace-reader.js +110 -0
  91. lixity/ui/components.py +329 -0
  92. lixity/ui/dashboard.py +1320 -0
  93. lixity/ui/dimensions.py +202 -0
  94. lixity/ui/dossier_images.py +24 -0
  95. lixity/ui/heatmap.py +234 -0
  96. lixity/ui/markers_panel.py +64 -0
  97. lixity/ui/modals.py +291 -0
  98. lixity/ui/narration.py +396 -0
  99. lixity/ui/research_panel.py +312 -0
  100. lixity/ui/settings.py +194 -0
  101. lixity/ui/structural.py +193 -0
  102. lixity/workspace.py +204 -0
  103. lixity/workspace_labels.py +846 -0
  104. lixity-0.4.2.dist-info/METADATA +397 -0
  105. lixity-0.4.2.dist-info/RECORD +109 -0
  106. lixity-0.4.2.dist-info/WHEEL +5 -0
  107. lixity-0.4.2.dist-info/entry_points.txt +2 -0
  108. lixity-0.4.2.dist-info/licenses/LICENSE +661 -0
  109. lixity-0.4.2.dist-info/top_level.txt +1 -0
lixity/__init__.py ADDED
@@ -0,0 +1,29 @@
1
+ """Lixity – Quantitative text linguistics, stylometry and self-calibrating style passports."""
2
+
3
+ from ._version import __version__
4
+ from .analyzer import CorpusAnalyzer
5
+ from .formatters import ReportFormatter
6
+ from .io import FileUtils
7
+ from .models import (
8
+ ChapterMetrics,
9
+ CorpusAuditReport,
10
+ CorpusConfig,
11
+ CorpusMetrics,
12
+ DossierStatus,
13
+ SentenceDistribution,
14
+ )
15
+ from .style_fingerprint import StyleFingerprint
16
+
17
+ __all__ = [
18
+ "ChapterMetrics",
19
+ "CorpusAnalyzer",
20
+ "CorpusAuditReport",
21
+ "CorpusConfig",
22
+ "CorpusMetrics",
23
+ "DossierStatus",
24
+ "FileUtils",
25
+ "ReportFormatter",
26
+ "SentenceDistribution",
27
+ "StyleFingerprint",
28
+ "__version__",
29
+ ]
lixity/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ """Run the installed Lixity CLI with ``python -m lixity``."""
2
+
3
+ from .cli import main
4
+
5
+ raise SystemExit(main())
lixity/_version.py ADDED
@@ -0,0 +1,3 @@
1
+ """Single source of truth for the package version (readable by setuptools without imports)."""
2
+
3
+ __version__ = "0.4.2"
lixity/analyzer.py ADDED
@@ -0,0 +1,569 @@
1
+ """lixity.analyzer – High-performance text analysis and corpus linguistics engine.
2
+
3
+ Computes sentence-length architecture (ASL, CV, short/long sentence shares), lexical diversity
4
+ (TTR, Guiraud R, HD-D, MTLD, MATTR, Maas, Yule's K), language-calibrated readability
5
+ (Flesch family, LIX), dialogue ratios, register signals, and per-chapter metrics
6
+ with standard errors.
7
+ """
8
+
9
+ import math
10
+ import os
11
+ import re
12
+ from collections import Counter
13
+ from dataclasses import dataclass
14
+
15
+ from .diversity import (
16
+ hd_d as hd_d_value,
17
+ )
18
+ from .diversity import (
19
+ maas_a2 as maas_a2_value,
20
+ )
21
+ from .diversity import (
22
+ mattr as mattr_value,
23
+ )
24
+ from .diversity import (
25
+ mtld as mtld_value,
26
+ )
27
+ from .diversity import (
28
+ yules_k as yules_k_value,
29
+ )
30
+ from .language import compile_pattern, compile_word_pattern, count_nominal_matches, resolve_language
31
+ from .language_data import READABILITY
32
+ from .markdown_parser import chapter_heading_spans, prose_paragraphs, prose_text, split_chapters
33
+ from .math_core import binomial_se_percent, per_mille, poisson_se_per_mille
34
+ from .models import (
35
+ ChapterMetrics,
36
+ CorpusConfig,
37
+ CorpusMetrics,
38
+ SentenceDistribution,
39
+ )
40
+ from .sentences import (
41
+ LONG_SENTENCE_MAX,
42
+ MEDIUM_SENTENCE_MAX,
43
+ SHORT_SENTENCE_MAX,
44
+ split_sentences,
45
+ )
46
+ from .style_profile import dominance_from_hits
47
+ from .syllables import count_syllables as count_syllables_for
48
+
49
+ _RE_HTML_COMMENT = re.compile(r"<!--.*?-->", re.DOTALL)
50
+ _RE_HEADING_LINE = re.compile(r"(?m)^#+.*$")
51
+ _RE_DASHES = re.compile(r"[–—]")
52
+ _RE_ELLIPSES = re.compile(r"(?:…|\.{3})")
53
+
54
+
55
+ @dataclass(frozen=True)
56
+ class SentenceStats:
57
+ """Sentence-length architecture of a text (shared by corpus and chapters)."""
58
+
59
+ total: int
60
+ asl: float
61
+ median: int
62
+ median_exact: float
63
+ std: float
64
+ cv: float
65
+ distribution: SentenceDistribution
66
+ staccato_pct: float
67
+ kaskade_pct: float
68
+
69
+
70
+ def share_se(pct: float, n: int) -> float:
71
+ """Binomial standard error of a share in percentage points (plug-in)."""
72
+ return binomial_se_percent(pct, n)
73
+
74
+
75
+ def count_se(cnt: int, words: int) -> float:
76
+ """Poisson standard error of a count density (per 1,000 words)."""
77
+ return poisson_se_per_mille(cnt, words)
78
+
79
+
80
+ class CorpusAnalyzer:
81
+ """
82
+ Stateless, thread-safe analysis engine for literary Markdown texts.
83
+ Performs quantitative corpus linguistics, stylometry and chapter segmentation.
84
+ """
85
+
86
+ def __init__(self, config: CorpusConfig | None = None, *, sample_text: str | None = None):
87
+ """Initialises the analyzer with a configuration.
88
+
89
+ If no configuration is passed, the default parameters apply.
90
+ ``sample_text`` lets an explicit ``language="auto"`` resolve against
91
+ the text to analyse; without it ``auto`` falls back to ``generic``
92
+ (documented in docs/LOCALIZATION.md).
93
+ """
94
+ self.config = config or CorpusConfig()
95
+ self.lang = resolve_language(self.config, sample_text=sample_text)
96
+ self._praes_re = compile_pattern(self.lang.praesens_regex)
97
+ self._praet_re = compile_pattern(self.lang.praeteritum_regex)
98
+ self._filter_re = compile_pattern(self.lang.filter_verbs_regex)
99
+ self._word_re = compile_word_pattern(self.lang.word_regex)
100
+ self._dialogue_re = compile_pattern(self.lang.dialogue_regex)
101
+ # Style heuristics (self-calibrating house-style fingerprint)
102
+ self._passive_re = compile_pattern(self.lang.passive_regex)
103
+ self._nominal_re = compile_pattern(self.lang.nominal_regex)
104
+ self._nominal_exclusions = (
105
+ self.lang.function_words if self.config.nominal_regex is None else frozenset()
106
+ )
107
+ self._adjective_re = compile_pattern(self.lang.adjective_regex)
108
+ self._modals = frozenset(w.lower() for w in self.lang.lexicon.get("modals", ()))
109
+ self._starters = frozenset(w.lower() for w in self.lang.first_person_starters)
110
+ self._content_blacklist = self.lang.function_words | self.lang.stopwords
111
+
112
+ @staticmethod
113
+ def _entropy(counts: Counter[str]) -> float:
114
+ """Shannon entropy in bits over a token-type counter."""
115
+ total = sum(counts.values())
116
+ if not total:
117
+ return 0.0
118
+ return math.fsum(-(c / total) * math.log2(c / total) for c in counts.values())
119
+
120
+ def _starter_stats(self, sentences: list[str]) -> tuple[float, float, float]:
121
+ """(start entropy in bits, first-person-start rate, entropy standard error).
122
+
123
+ Entropy uncertainty uses a first-order plug-in variance approximation:
124
+ Var(H) = [sum(p*log2^2(p)) - H^2] / n. It is not a calibrated interval.
125
+ """
126
+ starters: Counter[str] = Counter()
127
+ first_person = 0
128
+ for s in sentences:
129
+ tokens = self._word_re.findall(s)
130
+ if not tokens:
131
+ continue
132
+ # Elision-aware match (fr: "J'aime" -> "je"/"j", en: "I'm" -> "i")
133
+ first = tokens[0].lower()
134
+ stem = first.split("'", 1)[0]
135
+ starters[first] += 1
136
+ if first in self._starters or stem in self._starters:
137
+ first_person += 1
138
+ total = sum(starters.values())
139
+ rate = (first_person / total * 100.0) if total else 0.0
140
+ entropy = self._entropy(starters)
141
+ if not total:
142
+ return entropy, rate, 0.0
143
+ second = math.fsum(
144
+ (c / total) * math.log2(c / total) ** 2 for c in starters.values()
145
+ )
146
+ variance = (second - entropy**2) / total
147
+ return entropy, rate, math.sqrt(max(0.0, variance))
148
+
149
+ @staticmethod
150
+ def _density(matches: int, words: int) -> float:
151
+ """Occurrences per 1,000 tokens (length-comparable density)."""
152
+ return per_mille(matches, words)
153
+
154
+ def _jsd_chapter(
155
+ self,
156
+ chapter_counter: Counter[str],
157
+ corpus_counter: Counter[str],
158
+ n_chapter: int,
159
+ n_corpus: int,
160
+ top_n: int = 5,
161
+ ) -> tuple[float, list[str]]:
162
+ """
163
+ Jensen-Shannon divergence of the chapter's word distribution to the rest
164
+ of the corpus (leave-one-out, so small chapters are not self-biased).
165
+
166
+ Returns (jsd, top driver words): contributions are additive per type,
167
+ so the most divergent content words are interpretable.
168
+
169
+ Only the chapter's own vocabulary is iterated. Types the chapter does
170
+ not contain contribute exactly ``q * ln(2) / 2`` each, so their total
171
+ is ``ln(2)/2 * (1 - covered_rest_mass)`` in closed form – an O(chapter
172
+ types) computation instead of O(corpus types).
173
+ """
174
+ rest = n_corpus - n_chapter
175
+ if rest < 1 or n_chapter < 1:
176
+ return 0.0, []
177
+ jsd = 0.0
178
+ contribs: dict[str, float] = {}
179
+ covered_rest_mass = 0.0
180
+ for w in sorted(chapter_counter): # deterministic order: bit-identical across runs
181
+ p = chapter_counter[w] / n_chapter
182
+ q = (corpus_counter[w] - chapter_counter[w]) / rest
183
+ covered_rest_mass += q
184
+ m = 0.5 * (p + q)
185
+ term = p * math.log(p / m)
186
+ if q > 0.0:
187
+ term += q * math.log(q / m)
188
+ contrib = 0.5 * term
189
+ contribs[w] = contrib
190
+ jsd += contrib
191
+ # Types absent from the chapter: p = 0, m = q/2 -> contribution q*ln(2)/2.
192
+ jsd += 0.5 * math.log(2.0) * max(0.0, 1.0 - covered_rest_mass)
193
+ top = [
194
+ w
195
+ for w, _ in sorted(contribs.items(), key=lambda kv: kv[1], reverse=True)
196
+ if len(w) > 1 and w not in self._content_blacklist
197
+ ][:top_n]
198
+ return jsd, top
199
+
200
+ def count_syllables(self, word: str) -> int:
201
+ """Language-sensitive syllable counting (see :mod:`lixity.syllables`)."""
202
+ return count_syllables_for(word, self.lang.key)
203
+
204
+ def readability(self, asl: float, asw: float) -> tuple[float, str]:
205
+ """Language-calibrated Reading Ease score and formula name; not clipped to 0–100."""
206
+ rd = READABILITY.get(self.lang.key, READABILITY["generic"])
207
+ if asl == 0.0 and asw == 0.0:
208
+ return 0.0, rd["name"]
209
+ score = rd["constant"] - rd["asl_coef"] * asl - rd["asw_coef"] * asw
210
+ return score, rd["name"]
211
+
212
+ def long_word_min(self) -> int:
213
+ """Language-calibrated minimum letter count for LIX 'long words' (Björnsson)."""
214
+ rd = READABILITY.get(self.lang.key, READABILITY["generic"])
215
+ return int(rd["long_word_min"])
216
+
217
+ def _sentence_stats(self, sentences: list[str]) -> SentenceStats:
218
+ """Sentence-length architecture (lengths, ASL, median, CV, distribution)."""
219
+ lengths = [n for n in (len(self._word_re.findall(s)) for s in sentences) if n > 0]
220
+ total = len(lengths)
221
+ asl = sum(lengths) / total if total else 0.0
222
+ sorted_lengths = sorted(lengths)
223
+ median_sl = sorted_lengths[total // 2] if total else 0
224
+ median_exact = (
225
+ (sorted_lengths[(total - 1) // 2] + sorted_lengths[total // 2]) / 2.0
226
+ if total
227
+ else 0.0
228
+ )
229
+ variance = sum((x - asl) ** 2 for x in lengths) / total if total else 0.0
230
+ std = math.sqrt(variance)
231
+ short = sum(1 for x in lengths if x <= SHORT_SENTENCE_MAX)
232
+ medium = sum(1 for x in lengths if SHORT_SENTENCE_MAX < x <= MEDIUM_SENTENCE_MAX)
233
+ long_ = sum(1 for x in lengths if MEDIUM_SENTENCE_MAX < x <= LONG_SENTENCE_MAX)
234
+ complex_ = sum(1 for x in lengths if x > LONG_SENTENCE_MAX)
235
+
236
+ def _pct(count: int) -> float:
237
+ return (count / total * 100.0) if total else 0.0
238
+
239
+ return SentenceStats(
240
+ total=total,
241
+ asl=asl,
242
+ median=median_sl,
243
+ median_exact=median_exact,
244
+ std=std,
245
+ cv=(std / asl) if asl else 0.0,
246
+ distribution=SentenceDistribution(
247
+ short_count=short,
248
+ short_pct=_pct(short),
249
+ medium_count=medium,
250
+ medium_pct=_pct(medium),
251
+ long_count=long_,
252
+ long_pct=_pct(long_),
253
+ complex_count=complex_,
254
+ complex_pct=_pct(complex_),
255
+ ),
256
+ staccato_pct=_pct(short),
257
+ kaskade_pct=_pct(complex_),
258
+ )
259
+
260
+ def _paragraph_stats(self, paragraphs: list[str]) -> tuple[int, float, int]:
261
+ """(prose paragraphs, average words per paragraph, one-liner count)."""
262
+ lengths = [len(p.split()) for p in paragraphs]
263
+ total = len(lengths)
264
+ average = sum(lengths) / total if total else 0.0
265
+ one_liners = sum(
266
+ 1
267
+ for length in lengths
268
+ if length <= self.config.min_paragraph_length_for_oneliner
269
+ )
270
+ return total, average, one_liners
271
+
272
+ def _punctuation_profile(self, text: str) -> dict[str, int]:
273
+ """Language-neutral punctuation counts for the JSON contract."""
274
+ return {
275
+ "periods": text.count("."),
276
+ "commas": text.count(","),
277
+ "dashes": len(_RE_DASHES.findall(text)),
278
+ "colons": text.count(":"),
279
+ "semicolons": text.count(";"),
280
+ "questions": text.count("?"),
281
+ "exclamations": text.count("!"),
282
+ "ellipses": len(_RE_ELLIPSES.findall(text)),
283
+ }
284
+
285
+ def measure_unit(self, text: str) -> ChapterMetrics | None:
286
+ """Measure one prose unit using the chapter feature engine once.
287
+
288
+ This omits corpus-level readability, cross-chapter divergence and title
289
+ parsing. It is useful for scenes already segmented by the shared parser.
290
+ """
291
+ return self._chapter_metrics(1, "", text, self.long_word_min())[0]
292
+
293
+ def _chapter_metrics(
294
+ self, num: int, title: str, body: str, lw_min: int
295
+ ) -> tuple[ChapterMetrics | None, list[str]]:
296
+ """Per-chapter metrics (style features, uncertainty, tense, JSD input)."""
297
+ cl_b = prose_text(body, _front_matter=False)
298
+ c_words = self._word_re.findall(cl_b)
299
+ if not c_words:
300
+ return None, []
301
+
302
+ n_cw = len(c_words)
303
+ c_lower = [w.lower() for w in c_words]
304
+ c_sentences = split_sentences(cl_b, self.lang.key)
305
+ stats = self._sentence_stats(c_sentences)
306
+
307
+ c_dial = self._dialogue_re.findall(cl_b)
308
+ c_dial_words = sum(len(self._word_re.findall(m)) for m in c_dial)
309
+ c_dial_pct = (c_dial_words / n_cw) * 100.0 if n_cw else 0.0
310
+ c_ttr = len(set(c_lower)) / n_cw if n_cw else 0.0
311
+ c_signals = {
312
+ s_name: len(re.findall(s_pat, cl_b, re.IGNORECASE))
313
+ for s_name, s_pat in self.lang.signal_keywords.items()
314
+ }
315
+ c_motifs = {
316
+ m_name: len(re.findall(m_pat, cl_b, re.IGNORECASE))
317
+ for m_name, m_pat in self.config.motif_regexes.items()
318
+ }
319
+ c_fil = len(self._filter_re.findall(cl_b))
320
+ dominance = dominance_from_hits(
321
+ len(self._praes_re.findall(cl_b)), len(self._praet_re.findall(cl_b))
322
+ )
323
+
324
+ c_start_entropy, c_first_rate, c_entropy_se = self._starter_stats(c_sentences)
325
+ c_passive_cnt = len(self._passive_re.findall(cl_b))
326
+ c_nominal_cnt = count_nominal_matches(cl_b, self._nominal_re, self._nominal_exclusions)
327
+ c_adjective_cnt = len(self._adjective_re.findall(cl_b))
328
+ c_modal_cnt = sum(1 for t in c_lower if t in self._modals)
329
+ c_long_words = sum(1 for t in c_words if len(t) > lw_min)
330
+ c_long_pct = (c_long_words / n_cw * 100.0) if n_cw else 0.0
331
+ c_guiraud = len(set(c_lower)) / math.sqrt(n_cw) if n_cw else 0.0
332
+ c_hd_d = hd_d_value(c_lower)
333
+ c_func_pct = (
334
+ sum(1 for t in c_lower if t in self.lang.function_words) / n_cw * 100.0 if n_cw else 0.0
335
+ )
336
+
337
+ # Measurement uncertainty per feature (documented plug-ins)
338
+ c_se: dict[str, float] = {
339
+ "asl": (stats.std / math.sqrt(stats.total)) if stats.total else 0.0,
340
+ "staccato_pct": share_se(stats.staccato_pct, stats.total),
341
+ "kaskade_pct": share_se(stats.kaskade_pct, stats.total),
342
+ "sentence_cv": (
343
+ (stats.cv / math.sqrt(2.0 * stats.total)) * math.sqrt(1.0 + 2.0 * stats.cv**2)
344
+ if stats.total
345
+ else 0.0
346
+ ),
347
+ "dialog_pct": share_se(c_dial_pct, n_cw),
348
+ "function_word_pct": share_se(c_func_pct, n_cw),
349
+ "filter_density": count_se(c_fil, n_cw),
350
+ "modal_density": count_se(c_modal_cnt, n_cw),
351
+ "passive_density": count_se(c_passive_cnt, n_cw),
352
+ "nominalization_density": count_se(c_nominal_cnt, n_cw),
353
+ "adjective_density": count_se(c_adjective_cnt, n_cw),
354
+ "long_word_pct": share_se(c_long_pct, n_cw),
355
+ "start_entropy": c_entropy_se,
356
+ "first_person_start_rate": share_se(c_first_rate, stats.total),
357
+ "guiraud_r": (0.5 * c_guiraud / math.sqrt(n_cw)) if n_cw else 0.0,
358
+ }
359
+ # Exact rarefaction has no Monte Carlo SE; population uncertainty is not estimated.
360
+
361
+ chapter = ChapterMetrics(
362
+ num=num,
363
+ title=title,
364
+ words=n_cw,
365
+ sentences=stats.total,
366
+ asl=stats.asl,
367
+ dialog_pct=c_dial_pct,
368
+ ttr=c_ttr,
369
+ motif_counts=c_motifs,
370
+ filter_verbs=c_fil,
371
+ dominance=dominance,
372
+ signal_matches=c_signals,
373
+ staccato_pct=stats.staccato_pct,
374
+ kaskade_pct=stats.kaskade_pct,
375
+ sentence_cv=stats.cv,
376
+ start_entropy=c_start_entropy,
377
+ first_person_start_rate=c_first_rate,
378
+ passive_density=self._density(c_passive_cnt, n_cw),
379
+ nominalization_density=self._density(c_nominal_cnt, n_cw),
380
+ adjective_density=self._density(c_adjective_cnt, n_cw),
381
+ modal_density=self._density(c_modal_cnt, n_cw),
382
+ filter_density=self._density(c_fil, n_cw),
383
+ long_word_pct=c_long_pct,
384
+ guiraud_r=c_guiraud,
385
+ hd_d=c_hd_d,
386
+ function_word_pct=c_func_pct,
387
+ style_se=c_se,
388
+ )
389
+ return chapter, c_lower
390
+
391
+ def analyze_text(self, full_text: str) -> CorpusMetrics:
392
+ """
393
+ Performs the complete stylometric and quantitative analysis on
394
+ a passed text string.
395
+
396
+ Workflow:
397
+ 1. Separation of the scholarly appendix (single source of truth).
398
+ 2. Filtering of Markdown HTML comments (<!-- ... -->).
399
+ 3. Tokenisation and capture of vocabulary types.
400
+ 4. Computation of lexical diversity metrics (TTR, Guiraud, Yule).
401
+ 5. Sentence segmentation and sentence-length architecture (without heading artefacts).
402
+ 6. Readability indices (Flesch family, LIX).
403
+ 7. Dialogue and paragraph economy.
404
+ 8. Punctuation and signal word frequencies.
405
+ 9. Chapter-wise segmentation and tense classification.
406
+
407
+ Returns:
408
+ CorpusMetrics object with all computed metrics.
409
+ """
410
+ # 1. Main text vs. appendix, comments removed
411
+ if self.config.appendix_marker and self.config.appendix_marker in full_text:
412
+ main_text, _ = full_text.split(self.config.appendix_marker, 1)
413
+ else:
414
+ main_text = full_text
415
+ cleaned_full = _RE_HTML_COMMENT.sub("", full_text)
416
+ cleaned_main = _RE_HTML_COMMENT.sub("", main_text)
417
+ # Configured chapter headings are metadata, just like Markdown headings.
418
+ # Retain line separators so removing one never joins adjacent prose.
419
+ prose_parts: list[str] = []
420
+ cursor = 0
421
+ for start, end, _line, _title in chapter_heading_spans(cleaned_main, self.config):
422
+ prose_parts.append(cleaned_main[cursor:start])
423
+ prose_parts.append("\n" if cleaned_main[end - 1:end] == "\n" else "")
424
+ cursor = end
425
+ prose_parts.append(cleaned_main[cursor:])
426
+ legacy_clean_main = _RE_HEADING_LINE.sub("", "".join(prose_parts))
427
+
428
+ raw_words = len(cleaned_full.split())
429
+ clean_words = len(legacy_clean_main.split())
430
+ raw_chars = len(cleaned_full)
431
+ clean_chars = len(legacy_clean_main)
432
+ # Token-based measurements share the supported semantic-block scope.
433
+ # Legacy whitespace words/chars intentionally retain their own contract.
434
+ prose_units = prose_paragraphs(main_text, self.config)
435
+ prose_main = "\n\n".join(prose_units)
436
+
437
+ # 2. Tokenisation
438
+ tokens = self._word_re.findall(prose_main)
439
+ n_tokens = len(tokens)
440
+ lower_tokens = [t.lower() for t in tokens]
441
+ v_types = len(set(lower_tokens))
442
+ freqs = Counter(lower_tokens)
443
+
444
+ # 3. Lexical range & stability
445
+ ttr = v_types / n_tokens if n_tokens else 0.0
446
+ guiraud_r = v_types / math.sqrt(n_tokens) if n_tokens else 0.0
447
+
448
+ # 4. Sentence-length architecture on the same prose as other metrics.
449
+ sentences = split_sentences(prose_main, self.lang.key)
450
+ stats = self._sentence_stats(sentences)
451
+ start_entropy, first_person_start_rate, _entropy_se = self._starter_stats(sentences)
452
+
453
+ # 5. Readability & complexity (language-calibrated Flesch family + LIX)
454
+ total_syllables = sum(self.count_syllables(t) * count for t, count in freqs.items())
455
+ asw = total_syllables / n_tokens if n_tokens else 0.0
456
+ if n_tokens == 0 or stats.total == 0:
457
+ flesch_de = 0.0
458
+ flesch_variant = self.readability(0.0, 0.0)[1]
459
+ lix = 0.0
460
+ else:
461
+ flesch_de, flesch_variant = self.readability(stats.asl, asw)
462
+ lw_min = self.long_word_min()
463
+ long_words = sum(1 for t in tokens if len(t) > lw_min)
464
+ pct_long_words = (long_words / n_tokens) * 100.0 if n_tokens else 0.0
465
+ lix = stats.asl + pct_long_words
466
+
467
+ # 6. Dialogue ratio
468
+ dialog_matches = self._dialogue_re.findall(prose_main)
469
+ dialog_words = sum(len(self._word_re.findall(m)) for m in dialog_matches)
470
+ dialog_ratio = (dialog_words / n_tokens) * 100.0 if n_tokens else 0.0
471
+
472
+ # 7. Paragraph economy, punctuation, signals
473
+ total_paras, avg_para_len, single_line_paras = self._paragraph_stats(prose_units)
474
+ punctuation = self._punctuation_profile(prose_main)
475
+ signal_counts = {
476
+ name: len(re.findall(pat, prose_main, re.IGNORECASE))
477
+ for name, pat in self.lang.signal_keywords.items()
478
+ }
479
+ filter_cnt = len(self._filter_re.findall(prose_main))
480
+
481
+ # 8. Style densities (per 1,000 tokens, length-comparable)
482
+ passive_density = self._density(len(self._passive_re.findall(prose_main)), n_tokens)
483
+ nominalization_density = self._density(
484
+ count_nominal_matches(prose_main, self._nominal_re, self._nominal_exclusions), n_tokens
485
+ )
486
+ adjective_density = self._density(len(self._adjective_re.findall(prose_main)), n_tokens)
487
+ modal_density = self._density(sum(1 for t in lower_tokens if t in self._modals), n_tokens)
488
+ filter_density = self._density(filter_cnt, n_tokens)
489
+
490
+ # 9. Chapter-wise segmentation (shared split_chapters: front matter +
491
+ # appendix + empty-body rules identical to the structure modules).
492
+ chapters: list[ChapterMetrics] = []
493
+ chapter_tokens: list[list[str]] = []
494
+ for _split_num, title, body in split_chapters(main_text, self.config):
495
+ chapter, c_lower = self._chapter_metrics(len(chapters) + 1, title, body, lw_min)
496
+ if chapter is None:
497
+ continue
498
+ chapters.append(chapter)
499
+ chapter_tokens.append(c_lower)
500
+
501
+ # 10. Jensen-Shannon divergence per chapter vs. the rest (leave-one-out)
502
+ for chapter, c_lower in zip(chapters, chapter_tokens, strict=False):
503
+ chapter.jsd, chapter.jsd_top_words = self._jsd_chapter(
504
+ Counter(c_lower), freqs, len(c_lower), n_tokens
505
+ )
506
+
507
+ return CorpusMetrics(
508
+ raw_words=raw_words,
509
+ clean_words=clean_words,
510
+ raw_chars=raw_chars,
511
+ clean_chars=clean_chars,
512
+ tokens=n_tokens,
513
+ vocab_types=v_types,
514
+ ttr=ttr,
515
+ guiraud_r=guiraud_r,
516
+ yules_k=yules_k_value(lower_tokens),
517
+ total_sentences=stats.total,
518
+ asl=stats.asl,
519
+ median_sl=stats.median,
520
+ median_sl_exact=stats.median_exact,
521
+ std_sl=stats.std,
522
+ sentence_dist=stats.distribution,
523
+ asw=asw,
524
+ flesch_de=flesch_de,
525
+ flesch_variant=flesch_variant,
526
+ lix=lix,
527
+ dialog_words=dialog_words,
528
+ dialog_ratio=dialog_ratio,
529
+ total_paragraphs=total_paras,
530
+ avg_paragraph_len=avg_para_len,
531
+ single_line_paragraphs=single_line_paras,
532
+ punctuation=punctuation,
533
+ signal_counts=signal_counts,
534
+ filter_count=filter_cnt,
535
+ chapters=chapters,
536
+ staccato_pct=stats.staccato_pct,
537
+ kaskade_pct=stats.kaskade_pct,
538
+ sentence_cv=stats.cv,
539
+ start_entropy=start_entropy,
540
+ first_person_start_rate=first_person_start_rate,
541
+ passive_density=passive_density,
542
+ nominalization_density=nominalization_density,
543
+ adjective_density=adjective_density,
544
+ modal_density=modal_density,
545
+ filter_density=filter_density,
546
+ hd_d=hd_d_value(lower_tokens),
547
+ mtld=mtld_value(lower_tokens),
548
+ mattr=mattr_value(lower_tokens),
549
+ maas_a2=maas_a2_value(n_tokens, v_types),
550
+ )
551
+
552
+ def analyze_file(self, filepath: str) -> CorpusMetrics:
553
+ """
554
+ Reads a UTF-8 text file and delegates to analyze_text.
555
+
556
+ Parameters:
557
+ filepath: Absolute or relative file path to the Markdown manuscript.
558
+
559
+ Returns:
560
+ CorpusMetrics object.
561
+
562
+ Raises:
563
+ FileNotFoundError: If the file does not exist at the given path.
564
+ """
565
+ if not os.path.isfile(filepath):
566
+ raise FileNotFoundError(f"Manuscript file not found: {filepath}")
567
+ with open(filepath, encoding="utf-8") as f:
568
+ content = f.read()
569
+ return self.analyze_text(content)