markdown-extractor 0.1.1__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {markdown_extractor-0.1.1/src/markdown_extractor.egg-info → markdown_extractor-0.2.0}/PKG-INFO +117 -1
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/README.md +116 -0
- markdown_extractor-0.2.0/src/markdown_extractor/__init__.py +25 -0
- markdown_extractor-0.2.0/src/markdown_extractor/_collections.py +166 -0
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/src/markdown_extractor/blocks.py +42 -14
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/src/markdown_extractor/extractor.py +12 -9
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/src/markdown_extractor/html_renderer.py +13 -9
- markdown_extractor-0.2.0/src/markdown_extractor/inline.py +122 -0
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/src/markdown_extractor/section.py +39 -10
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/src/markdown_extractor/text_renderer.py +9 -9
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0/src/markdown_extractor.egg-info}/PKG-INFO +117 -1
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/src/markdown_extractor.egg-info/SOURCES.txt +4 -1
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/tests/test_extractor.py +326 -1
- markdown_extractor-0.2.0/tests/test_inlines.py +255 -0
- markdown_extractor-0.1.1/src/markdown_extractor/__init__.py +0 -14
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/LICENSE +0 -0
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/pyproject.toml +0 -0
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/setup.cfg +0 -0
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/src/markdown_extractor/parser.py +0 -0
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/src/markdown_extractor/py.typed +0 -0
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/src/markdown_extractor.egg-info/dependency_links.txt +0 -0
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/src/markdown_extractor.egg-info/requires.txt +0 -0
- {markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/src/markdown_extractor.egg-info/top_level.txt +0 -0
{markdown_extractor-0.1.1/src/markdown_extractor.egg-info → markdown_extractor-0.2.0}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: markdown-extractor
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Extract structured sections from Markdown by header — bracket-style access with robust handling of code blocks, tables, math, and YAML front matter.
|
|
5
5
|
Author-email: Fasil <fasilwdr@hotmail.com>
|
|
6
6
|
License: Apache License
|
|
@@ -417,6 +417,122 @@ bool(s.block(99)) # False — null sentinel
|
|
|
417
417
|
out-of-range — strict access stays strict. Use `block()` / `.get()` only
|
|
418
418
|
when you want soft fall-through.
|
|
419
419
|
|
|
420
|
+
### `.filtered(**kwargs)` — narrow any collection in place
|
|
421
|
+
|
|
422
|
+
Collection accessors return `BlockList` / `SectionList`, both `list`
|
|
423
|
+
subclasses that add a `.filtered(**kwargs)` method for chainable,
|
|
424
|
+
attribute-based narrowing. The kwargs form keeps the API usable from
|
|
425
|
+
Jinja2 / Django templates (which can't define lambdas). Existing list
|
|
426
|
+
operations (indexing, iteration, `len()`, `isinstance(x, list)`) keep
|
|
427
|
+
working unchanged.
|
|
428
|
+
|
|
429
|
+
```python
|
|
430
|
+
s = e["Section 1"]
|
|
431
|
+
|
|
432
|
+
# Equality (most common):
|
|
433
|
+
s.blocks.filtered(kind="paragraph")
|
|
434
|
+
e["Section 1"].children.filtered(level=2)
|
|
435
|
+
|
|
436
|
+
# Multiple kwargs AND together:
|
|
437
|
+
s.blocks.filtered(kind="code", info="python")
|
|
438
|
+
|
|
439
|
+
# Operator suffixes:
|
|
440
|
+
e.headers().filtered(level__gte=2)
|
|
441
|
+
e.headers().filtered(level__in=[2, 3])
|
|
442
|
+
e.headers().filtered(title__startswith="Sub")
|
|
443
|
+
|
|
444
|
+
# Chains keep the typed return — both results are SectionList:
|
|
445
|
+
e.headers() \
|
|
446
|
+
.filtered(level=2) \
|
|
447
|
+
.filtered(title__startswith="A")
|
|
448
|
+
|
|
449
|
+
# No kwargs → a shallow copy of the same subclass.
|
|
450
|
+
e.headers().filtered()
|
|
451
|
+
```
|
|
452
|
+
|
|
453
|
+
Supported operator suffixes:
|
|
454
|
+
|
|
455
|
+
| Suffix | Meaning |
|
|
456
|
+
|--------|---------|
|
|
457
|
+
| *(none)* | `==` |
|
|
458
|
+
| `__ne` | `!=` |
|
|
459
|
+
| `__lt`, `__lte`, `__gt`, `__gte` | comparison |
|
|
460
|
+
| `__in` | membership in iterable |
|
|
461
|
+
| `__contains` | substring (`b in a`) or container-`in` |
|
|
462
|
+
| `__startswith`, `__endswith` | string prefix / suffix |
|
|
463
|
+
|
|
464
|
+
Items missing the requested attribute are treated as non-matches — no
|
|
465
|
+
exception. Slices preserve the subclass too, so
|
|
466
|
+
`section.children[1:].filtered(level=2)` works.
|
|
467
|
+
|
|
468
|
+
The method is available on:
|
|
469
|
+
|
|
470
|
+
| Returns `BlockList` | Returns `SectionList` |
|
|
471
|
+
|---------------------|------------------------|
|
|
472
|
+
| `Section.blocks` | `Section.children` |
|
|
473
|
+
| `Block.children` | `Section.find(title)` |
|
|
474
|
+
| `Block.walk()` | `Section.walk()` |
|
|
475
|
+
| | `MDExtractor.find(title)` |
|
|
476
|
+
| | `MDExtractor.walk()` |
|
|
477
|
+
| | `MDExtractor.headers()` |
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
### `.mapped(path)` — extract / flatten across a collection
|
|
481
|
+
|
|
482
|
+
`BlockList`, `SectionList`, `Section`, and `Block` all expose a
|
|
483
|
+
`.mapped(path)` method — dotted-path attribute traversal that flattens
|
|
484
|
+
list-valued attributes and keeps the typed return so you can keep
|
|
485
|
+
chaining `.filtered(...)`. Calling it on a single `Section` / `Block`
|
|
486
|
+
behaves like a one-element collection, so the same path rules apply.
|
|
487
|
+
|
|
488
|
+
```python
|
|
489
|
+
s = e["Section 1"]
|
|
490
|
+
|
|
491
|
+
# Pull every body block from every direct subsection (flat BlockList):
|
|
492
|
+
s.children.mapped("blocks")
|
|
493
|
+
|
|
494
|
+
# Dotted paths walk further — every list_item inside every top-level
|
|
495
|
+
# block of Section 1, fully flattened:
|
|
496
|
+
s.blocks.mapped("children") # → 3 list_items (the bullets)
|
|
497
|
+
|
|
498
|
+
# Two-level dotted path — every inline token under those bullets:
|
|
499
|
+
s.blocks.mapped("children.inlines")
|
|
500
|
+
# → [bold 'Lightweight', text ' — …', em 'Flexible', text ' — …',
|
|
501
|
+
# code 'Tested', text ' — …']
|
|
502
|
+
|
|
503
|
+
# Equivalent — chained calls produce the same result as the dotted path:
|
|
504
|
+
s.blocks.mapped("children.inlines") == s.blocks.mapped("children").mapped("inlines")
|
|
505
|
+
|
|
506
|
+
# Chain with .filtered() — still a BlockList:
|
|
507
|
+
s.blocks.mapped("children").filtered(kind="list_item")
|
|
508
|
+
|
|
509
|
+
# Scalar attributes return a plain list:
|
|
510
|
+
s.children.mapped("title") # ['Subsection 1.1', 'FAQ']
|
|
511
|
+
e.headers().mapped("title") # ['Section 1', 'Subsection 1.1', 'FAQ']
|
|
512
|
+
|
|
513
|
+
# Works on a single Section / Block too — treated as a one-element collection:
|
|
514
|
+
s.mapped("title") # ['Section 1']
|
|
515
|
+
s.mapped("children.blocks") # BlockList — every block under every child
|
|
516
|
+
s.blocks[1].mapped("children.inlines") # same dotted-path rules from a single Block
|
|
517
|
+
```
|
|
518
|
+
|
|
519
|
+
Behaviour notes:
|
|
520
|
+
|
|
521
|
+
- List-valued attributes (e.g. `.blocks`, `.inlines`, `.children`)
|
|
522
|
+
are **flattened** into the result.
|
|
523
|
+
- Scalar attributes (e.g. `.title`, `.kind`, `.text`) are appended;
|
|
524
|
+
the return is a plain `list` (no `.filtered()` chaining).
|
|
525
|
+
- The concrete subclass is preserved when every step yields the same
|
|
526
|
+
`FilteredList` subclass, so the typed-return chain
|
|
527
|
+
`… .mapped("blocks").filtered(kind="paragraph")` keeps working.
|
|
528
|
+
- Items missing the attribute are skipped silently — same convention
|
|
529
|
+
as `.filtered()`.
|
|
530
|
+
- `coll.mapped("")` returns a shallow copy of the same subclass.
|
|
531
|
+
- On a single `Section` / `Block`, `record.mapped(path)` is equivalent
|
|
532
|
+
to wrapping it in a one-element list and mapping — so a scalar path
|
|
533
|
+
returns a list of one (e.g. `section.mapped("title") == [section.title]`).
|
|
534
|
+
|
|
535
|
+
|
|
420
536
|
### `to_list()` — flatten body to strings
|
|
421
537
|
|
|
422
538
|
One entry per top-level block. Lists expand to one entry per top-level
|
|
@@ -176,6 +176,122 @@ bool(s.block(99)) # False — null sentinel
|
|
|
176
176
|
out-of-range — strict access stays strict. Use `block()` / `.get()` only
|
|
177
177
|
when you want soft fall-through.
|
|
178
178
|
|
|
179
|
+
### `.filtered(**kwargs)` — narrow any collection in place
|
|
180
|
+
|
|
181
|
+
Collection accessors return `BlockList` / `SectionList`, both `list`
|
|
182
|
+
subclasses that add a `.filtered(**kwargs)` method for chainable,
|
|
183
|
+
attribute-based narrowing. The kwargs form keeps the API usable from
|
|
184
|
+
Jinja2 / Django templates (which can't define lambdas). Existing list
|
|
185
|
+
operations (indexing, iteration, `len()`, `isinstance(x, list)`) keep
|
|
186
|
+
working unchanged.
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
s = e["Section 1"]
|
|
190
|
+
|
|
191
|
+
# Equality (most common):
|
|
192
|
+
s.blocks.filtered(kind="paragraph")
|
|
193
|
+
e["Section 1"].children.filtered(level=2)
|
|
194
|
+
|
|
195
|
+
# Multiple kwargs AND together:
|
|
196
|
+
s.blocks.filtered(kind="code", info="python")
|
|
197
|
+
|
|
198
|
+
# Operator suffixes:
|
|
199
|
+
e.headers().filtered(level__gte=2)
|
|
200
|
+
e.headers().filtered(level__in=[2, 3])
|
|
201
|
+
e.headers().filtered(title__startswith="Sub")
|
|
202
|
+
|
|
203
|
+
# Chains keep the typed return — both results are SectionList:
|
|
204
|
+
e.headers() \
|
|
205
|
+
.filtered(level=2) \
|
|
206
|
+
.filtered(title__startswith="A")
|
|
207
|
+
|
|
208
|
+
# No kwargs → a shallow copy of the same subclass.
|
|
209
|
+
e.headers().filtered()
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
Supported operator suffixes:
|
|
213
|
+
|
|
214
|
+
| Suffix | Meaning |
|
|
215
|
+
|--------|---------|
|
|
216
|
+
| *(none)* | `==` |
|
|
217
|
+
| `__ne` | `!=` |
|
|
218
|
+
| `__lt`, `__lte`, `__gt`, `__gte` | comparison |
|
|
219
|
+
| `__in` | membership in iterable |
|
|
220
|
+
| `__contains` | substring (`b in a`) or container-`in` |
|
|
221
|
+
| `__startswith`, `__endswith` | string prefix / suffix |
|
|
222
|
+
|
|
223
|
+
Items missing the requested attribute are treated as non-matches — no
|
|
224
|
+
exception. Slices preserve the subclass too, so
|
|
225
|
+
`section.children[1:].filtered(level=2)` works.
|
|
226
|
+
|
|
227
|
+
The method is available on:
|
|
228
|
+
|
|
229
|
+
| Returns `BlockList` | Returns `SectionList` |
|
|
230
|
+
|---------------------|------------------------|
|
|
231
|
+
| `Section.blocks` | `Section.children` |
|
|
232
|
+
| `Block.children` | `Section.find(title)` |
|
|
233
|
+
| `Block.walk()` | `Section.walk()` |
|
|
234
|
+
| | `MDExtractor.find(title)` |
|
|
235
|
+
| | `MDExtractor.walk()` |
|
|
236
|
+
| | `MDExtractor.headers()` |
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
### `.mapped(path)` — extract / flatten across a collection
|
|
240
|
+
|
|
241
|
+
`BlockList`, `SectionList`, `Section`, and `Block` all expose a
|
|
242
|
+
`.mapped(path)` method — dotted-path attribute traversal that flattens
|
|
243
|
+
list-valued attributes and keeps the typed return so you can keep
|
|
244
|
+
chaining `.filtered(...)`. Calling it on a single `Section` / `Block`
|
|
245
|
+
behaves like a one-element collection, so the same path rules apply.
|
|
246
|
+
|
|
247
|
+
```python
|
|
248
|
+
s = e["Section 1"]
|
|
249
|
+
|
|
250
|
+
# Pull every body block from every direct subsection (flat BlockList):
|
|
251
|
+
s.children.mapped("blocks")
|
|
252
|
+
|
|
253
|
+
# Dotted paths walk further — every list_item inside every top-level
|
|
254
|
+
# block of Section 1, fully flattened:
|
|
255
|
+
s.blocks.mapped("children") # → 3 list_items (the bullets)
|
|
256
|
+
|
|
257
|
+
# Two-level dotted path — every inline token under those bullets:
|
|
258
|
+
s.blocks.mapped("children.inlines")
|
|
259
|
+
# → [bold 'Lightweight', text ' — …', em 'Flexible', text ' — …',
|
|
260
|
+
# code 'Tested', text ' — …']
|
|
261
|
+
|
|
262
|
+
# Equivalent — chained calls produce the same result as the dotted path:
|
|
263
|
+
s.blocks.mapped("children.inlines") == s.blocks.mapped("children").mapped("inlines")
|
|
264
|
+
|
|
265
|
+
# Chain with .filtered() — still a BlockList:
|
|
266
|
+
s.blocks.mapped("children").filtered(kind="list_item")
|
|
267
|
+
|
|
268
|
+
# Scalar attributes return a plain list:
|
|
269
|
+
s.children.mapped("title") # ['Subsection 1.1', 'FAQ']
|
|
270
|
+
e.headers().mapped("title") # ['Section 1', 'Subsection 1.1', 'FAQ']
|
|
271
|
+
|
|
272
|
+
# Works on a single Section / Block too — treated as a one-element collection:
|
|
273
|
+
s.mapped("title") # ['Section 1']
|
|
274
|
+
s.mapped("children.blocks") # BlockList — every block under every child
|
|
275
|
+
s.blocks[1].mapped("children.inlines") # same dotted-path rules from a single Block
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
Behaviour notes:
|
|
279
|
+
|
|
280
|
+
- List-valued attributes (e.g. `.blocks`, `.inlines`, `.children`)
|
|
281
|
+
are **flattened** into the result.
|
|
282
|
+
- Scalar attributes (e.g. `.title`, `.kind`, `.text`) are appended;
|
|
283
|
+
the return is a plain `list` (no `.filtered()` chaining).
|
|
284
|
+
- The concrete subclass is preserved when every step yields the same
|
|
285
|
+
`FilteredList` subclass, so the typed-return chain
|
|
286
|
+
`… .mapped("blocks").filtered(kind="paragraph")` keeps working.
|
|
287
|
+
- Items missing the attribute are skipped silently — same convention
|
|
288
|
+
as `.filtered()`.
|
|
289
|
+
- `coll.mapped("")` returns a shallow copy of the same subclass.
|
|
290
|
+
- On a single `Section` / `Block`, `record.mapped(path)` is equivalent
|
|
291
|
+
to wrapping it in a one-element list and mapping — so a scalar path
|
|
292
|
+
returns a list of one (e.g. `section.mapped("title") == [section.title]`).
|
|
293
|
+
|
|
294
|
+
|
|
179
295
|
### `to_list()` — flatten body to strings
|
|
180
296
|
|
|
181
297
|
One entry per top-level block. Lists expand to one entry per top-level
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""markdown-extractor — extract structured sections from Markdown.
|
|
2
|
+
|
|
3
|
+
Public API:
|
|
4
|
+
MDExtractor — entry point for parsing a Markdown document.
|
|
5
|
+
Section — a node in the parsed header tree.
|
|
6
|
+
Block — a node in a section's parsed body block tree.
|
|
7
|
+
SectionList — the filterable list returned by ``.children`` / ``walk()`` / ``find()`` / ``headers()``.
|
|
8
|
+
BlockList — the filterable list returned by ``.blocks`` and ``Block.children`` / ``walk()``.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from markdown_extractor._collections import BlockList, FilteredList, SectionList
|
|
12
|
+
from markdown_extractor.blocks import Block
|
|
13
|
+
from markdown_extractor.extractor import MDExtractor
|
|
14
|
+
from markdown_extractor.section import Section
|
|
15
|
+
|
|
16
|
+
__version__ = "0.2.0"
|
|
17
|
+
__all__ = [
|
|
18
|
+
"MDExtractor",
|
|
19
|
+
"Section",
|
|
20
|
+
"Block",
|
|
21
|
+
"SectionList",
|
|
22
|
+
"BlockList",
|
|
23
|
+
"FilteredList",
|
|
24
|
+
"__version__",
|
|
25
|
+
]
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"""Collection types that add ``.filtered(**kwargs)`` on top of ``list``.
|
|
2
|
+
|
|
3
|
+
:class:`FilteredList` is a thin ``list`` subclass with one extra method —
|
|
4
|
+
``filtered`` — for chainable, attribute-based narrowing using keyword
|
|
5
|
+
arguments (so it works inside Jinja2 / Django templates, which cannot
|
|
6
|
+
define lambdas). The two named subclasses :class:`BlockList` and
|
|
7
|
+
:class:`SectionList` are what the public API actually returns; they exist
|
|
8
|
+
mostly to give nicer reprs and let ``isinstance(x, BlockList)`` discriminate
|
|
9
|
+
without poking at generic introspection.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from typing import Any, Generic, List, Optional, Tuple, TypeVar, Union
|
|
15
|
+
|
|
16
|
+
T = TypeVar("T")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
_MISSING = object()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# Operator table — ``attr__<op>=value`` maps to ``_OPS[op](attr_value, value)``.
|
|
23
|
+
# Adding a new operator is a one-line entry here.
|
|
24
|
+
_OPS = {
|
|
25
|
+
"eq": lambda a, b: a == b,
|
|
26
|
+
"ne": lambda a, b: a != b,
|
|
27
|
+
"lt": lambda a, b: a < b,
|
|
28
|
+
"lte": lambda a, b: a <= b,
|
|
29
|
+
"gt": lambda a, b: a > b,
|
|
30
|
+
"gte": lambda a, b: a >= b,
|
|
31
|
+
"in": lambda a, b: a in b,
|
|
32
|
+
"contains": lambda a, b: b in a,
|
|
33
|
+
"startswith": lambda a, b: a.startswith(b),
|
|
34
|
+
"endswith": lambda a, b: a.endswith(b),
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _resolve_key(key: str) -> Tuple[str, str]:
|
|
39
|
+
"""Split ``"level__gte"`` into ``("level", "gte")``.
|
|
40
|
+
|
|
41
|
+
Falls back to ``(key, "eq")`` if the suffix after the last ``__`` is
|
|
42
|
+
not a known operator, so an attribute literally named ``foo__bar`` is
|
|
43
|
+
still addressable via ``foo__bar=value``.
|
|
44
|
+
"""
|
|
45
|
+
if "__" in key:
|
|
46
|
+
attr, _, op = key.rpartition("__")
|
|
47
|
+
if op in _OPS and attr:
|
|
48
|
+
return attr, op
|
|
49
|
+
return key, "eq"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class FilteredList(list, Generic[T]):
|
|
53
|
+
"""A ``list`` subclass with chainable, kwargs-based filtering.
|
|
54
|
+
|
|
55
|
+
Examples::
|
|
56
|
+
|
|
57
|
+
section.blocks.filtered(kind="paragraph")
|
|
58
|
+
section.blocks.filtered(kind="code", info="python")
|
|
59
|
+
e.headers().filtered(level__gte=2)
|
|
60
|
+
e.headers().filtered(level__in=[2, 3])
|
|
61
|
+
e.headers().filtered(title__startswith="Sub")
|
|
62
|
+
|
|
63
|
+
Multiple kwargs are AND-combined. Each key is ``attr`` (implicit
|
|
64
|
+
equality) or ``attr__<op>``. Supported operators: ``eq`` (default),
|
|
65
|
+
``ne``, ``lt``, ``lte``, ``gt``, ``gte``, ``in``, ``contains``,
|
|
66
|
+
``startswith``, ``endswith``.
|
|
67
|
+
|
|
68
|
+
Items missing the requested attribute are treated as non-matches
|
|
69
|
+
(no exception). Filtering and slicing both preserve the concrete
|
|
70
|
+
subclass, so chains stay typed end-to-end.
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
def filtered(self, **conditions: Any) -> "FilteredList[T]":
|
|
74
|
+
"""Return a new collection of the same type containing only items
|
|
75
|
+
matching every keyword condition.
|
|
76
|
+
|
|
77
|
+
With no kwargs, returns a shallow copy.
|
|
78
|
+
"""
|
|
79
|
+
if not conditions:
|
|
80
|
+
return type(self)(self)
|
|
81
|
+
checks = [(*_resolve_key(k), v) for k, v in conditions.items()]
|
|
82
|
+
out: "FilteredList[T]" = type(self)()
|
|
83
|
+
for item in self:
|
|
84
|
+
keep = True
|
|
85
|
+
for attr, op, expected in checks:
|
|
86
|
+
value = getattr(item, attr, _MISSING)
|
|
87
|
+
if value is _MISSING:
|
|
88
|
+
keep = False
|
|
89
|
+
break
|
|
90
|
+
try:
|
|
91
|
+
matched = _OPS[op](value, expected)
|
|
92
|
+
except (TypeError, AttributeError):
|
|
93
|
+
matched = False
|
|
94
|
+
if not matched:
|
|
95
|
+
keep = False
|
|
96
|
+
break
|
|
97
|
+
if keep:
|
|
98
|
+
out.append(item)
|
|
99
|
+
return out
|
|
100
|
+
|
|
101
|
+
def mapped(self, path: str) -> Union["FilteredList[Any]", List[Any]]:
|
|
102
|
+
"""Gather values from a dotted attribute path, flattening lists.
|
|
103
|
+
|
|
104
|
+
Examples::
|
|
105
|
+
|
|
106
|
+
section.children.mapped("title") # list[str]
|
|
107
|
+
section.children.mapped("blocks") # BlockList
|
|
108
|
+
section.children.mapped("blocks.inlines") # BlockList
|
|
109
|
+
|
|
110
|
+
Equivalent: ``c.mapped("x.y") == c.mapped("x").mapped("y")``.
|
|
111
|
+
List-typed attributes (e.g. ``.blocks``) flatten into the result;
|
|
112
|
+
scalar attributes append. The concrete subclass is preserved
|
|
113
|
+
when every value came from the same :class:`FilteredList`
|
|
114
|
+
subclass, so ``.mapped("blocks").filtered(kind="paragraph")``
|
|
115
|
+
keeps working. Items missing the attribute are skipped (same
|
|
116
|
+
as :meth:`filtered`). An empty path returns a shallow copy.
|
|
117
|
+
"""
|
|
118
|
+
if not path:
|
|
119
|
+
return type(self)(self)
|
|
120
|
+
attr, _, rest = path.partition(".")
|
|
121
|
+
|
|
122
|
+
gathered: List[Any] = []
|
|
123
|
+
list_kind: Optional[type] = None
|
|
124
|
+
mixed = False
|
|
125
|
+
saw_scalar = False
|
|
126
|
+
|
|
127
|
+
for item in self:
|
|
128
|
+
value = getattr(item, attr, _MISSING)
|
|
129
|
+
if value is _MISSING:
|
|
130
|
+
continue
|
|
131
|
+
if isinstance(value, list):
|
|
132
|
+
if isinstance(value, FilteredList):
|
|
133
|
+
if list_kind is None:
|
|
134
|
+
list_kind = type(value)
|
|
135
|
+
elif list_kind is not type(value):
|
|
136
|
+
mixed = True
|
|
137
|
+
gathered.extend(value)
|
|
138
|
+
else:
|
|
139
|
+
saw_scalar = True
|
|
140
|
+
gathered.append(value)
|
|
141
|
+
|
|
142
|
+
result: Union[FilteredList[Any], List[Any]]
|
|
143
|
+
if list_kind is not None and not saw_scalar:
|
|
144
|
+
result = (FilteredList if mixed else list_kind)(gathered)
|
|
145
|
+
else:
|
|
146
|
+
result = gathered # plain list (scalars, or nothing matched)
|
|
147
|
+
|
|
148
|
+
if rest:
|
|
149
|
+
if isinstance(result, FilteredList):
|
|
150
|
+
return result.mapped(rest)
|
|
151
|
+
return [] # can't descend into scalars / missing attrs
|
|
152
|
+
return result
|
|
153
|
+
|
|
154
|
+
def __getitem__(self, key):
|
|
155
|
+
result = super().__getitem__(key)
|
|
156
|
+
if isinstance(key, slice):
|
|
157
|
+
return type(self)(result)
|
|
158
|
+
return result
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class BlockList(FilteredList["Block"]): # type: ignore[type-arg]
|
|
162
|
+
"""A :class:`FilteredList` of :class:`~markdown_extractor.blocks.Block`."""
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
class SectionList(FilteredList["Section"]): # type: ignore[type-arg]
|
|
166
|
+
"""A :class:`FilteredList` of :class:`~markdown_extractor.section.Section`."""
|
|
@@ -17,8 +17,10 @@ from __future__ import annotations
|
|
|
17
17
|
|
|
18
18
|
import re
|
|
19
19
|
from dataclasses import dataclass, field
|
|
20
|
-
from typing import
|
|
20
|
+
from typing import List, Optional
|
|
21
21
|
|
|
22
|
+
from markdown_extractor._collections import BlockList
|
|
23
|
+
from markdown_extractor.inline import parse_inlines
|
|
22
24
|
from markdown_extractor.text_renderer import strip_inline
|
|
23
25
|
|
|
24
26
|
|
|
@@ -33,13 +35,36 @@ from markdown_extractor.text_renderer import strip_inline
|
|
|
33
35
|
class Block:
|
|
34
36
|
kind: str
|
|
35
37
|
text: str = ""
|
|
36
|
-
children:
|
|
38
|
+
children: "BlockList" = field(default_factory=BlockList)
|
|
37
39
|
info: str = "" # code language, list marker style, etc.
|
|
40
|
+
inlines: "BlockList" = field(default_factory=BlockList) # structured inline tokens (paragraph / list_item)
|
|
38
41
|
|
|
39
|
-
def walk(self) ->
|
|
40
|
-
|
|
42
|
+
def walk(self) -> "BlockList":
|
|
43
|
+
"""Return this block and every descendant in depth-first order.
|
|
44
|
+
|
|
45
|
+
The result is a :class:`BlockList`, so you can chain
|
|
46
|
+
``block.walk().filtered(kind="code")``.
|
|
47
|
+
"""
|
|
48
|
+
out: BlockList = BlockList()
|
|
49
|
+
out.append(self)
|
|
41
50
|
for child in self.children:
|
|
42
|
-
|
|
51
|
+
out.extend(child.walk())
|
|
52
|
+
return out
|
|
53
|
+
|
|
54
|
+
def mapped(self, path: str):
|
|
55
|
+
"""Dotted-path attribute traversal on a single block.
|
|
56
|
+
|
|
57
|
+
Equivalent to ``BlockList([self]).mapped(path)`` — treats this
|
|
58
|
+
block as a one-element collection so the same dotted-path /
|
|
59
|
+
flattening rules apply::
|
|
60
|
+
|
|
61
|
+
block.mapped("children") # BlockList of children
|
|
62
|
+
block.mapped("children.inlines") # BlockList — flat
|
|
63
|
+
block.mapped("text") # [text] (scalar → list)
|
|
64
|
+
|
|
65
|
+
See :meth:`FilteredList.mapped` for full semantics.
|
|
66
|
+
"""
|
|
67
|
+
return BlockList([self]).mapped(path)
|
|
43
68
|
|
|
44
69
|
def to_dict(self) -> dict:
|
|
45
70
|
out: dict = {"kind": self.kind, "text": self.text}
|
|
@@ -47,6 +72,8 @@ class Block:
|
|
|
47
72
|
out["info"] = self.info
|
|
48
73
|
if self.children:
|
|
49
74
|
out["children"] = [c.to_dict() for c in self.children]
|
|
75
|
+
if self.inlines:
|
|
76
|
+
out["inlines"] = [t.to_dict() for t in self.inlines]
|
|
50
77
|
return out
|
|
51
78
|
|
|
52
79
|
@property
|
|
@@ -105,12 +132,12 @@ def _expand_indent(s: str) -> int:
|
|
|
105
132
|
return col
|
|
106
133
|
|
|
107
134
|
|
|
108
|
-
def parse_blocks(text: str) ->
|
|
135
|
+
def parse_blocks(text: str) -> BlockList:
|
|
109
136
|
"""Parse a section's body text into a list of top-level blocks."""
|
|
110
137
|
if not text or not text.strip():
|
|
111
|
-
return
|
|
138
|
+
return BlockList()
|
|
112
139
|
lines = text.split("\n")
|
|
113
|
-
return _parse(lines, 0, len(lines), base_indent=0)
|
|
140
|
+
return BlockList(_parse(lines, 0, len(lines), base_indent=0))
|
|
114
141
|
|
|
115
142
|
|
|
116
143
|
def _parse(lines: List[str], start: int, end: int, base_indent: int) -> List[Block]:
|
|
@@ -202,7 +229,7 @@ def _consume_list(lines, i, end, base_indent, ordered: bool):
|
|
|
202
229
|
"""
|
|
203
230
|
list_indent = _expand_indent(lines[i])
|
|
204
231
|
kind = "ordered_list" if ordered else "list"
|
|
205
|
-
items:
|
|
232
|
+
items: BlockList = BlockList()
|
|
206
233
|
j = i
|
|
207
234
|
while j < end:
|
|
208
235
|
line = lines[j]
|
|
@@ -252,8 +279,8 @@ def _consume_list(lines, i, end, base_indent, ordered: bool):
|
|
|
252
279
|
m = m_o if ordered else m_b
|
|
253
280
|
if m is None:
|
|
254
281
|
break
|
|
255
|
-
rest = m.group("rest")
|
|
256
|
-
item = Block(kind="list_item", text=rest
|
|
282
|
+
rest = m.group("rest").strip()
|
|
283
|
+
item = Block(kind="list_item", text=rest, inlines=parse_inlines(rest))
|
|
257
284
|
items.append(item)
|
|
258
285
|
j += 1
|
|
259
286
|
continue
|
|
@@ -318,7 +345,8 @@ def _consume_paragraph(lines, i, end, base_indent):
|
|
|
318
345
|
break
|
|
319
346
|
body.append(stripped)
|
|
320
347
|
j += 1
|
|
321
|
-
|
|
348
|
+
text = " ".join(body)
|
|
349
|
+
return Block(kind="paragraph", text=text, inlines=parse_inlines(text)), j
|
|
322
350
|
|
|
323
351
|
|
|
324
352
|
def flatten(blocks: List[Block]) -> List[str]:
|
|
@@ -350,10 +378,10 @@ def _null_block() -> Block:
|
|
|
350
378
|
Behaviour:
|
|
351
379
|
- ``bool(b)`` is ``False``
|
|
352
380
|
- ``b.text_plain`` → ``""``
|
|
353
|
-
- ``b.text`` → ``""``, ``b.children`` → ``[]``
|
|
381
|
+
- ``b.text`` → ``""``, ``b.children`` → ``[]``, ``b.inlines`` → ``[]``
|
|
354
382
|
- ``b.get(*more)`` → keeps returning this sentinel
|
|
355
383
|
"""
|
|
356
384
|
global _NULL_BLOCK
|
|
357
385
|
if _NULL_BLOCK is None:
|
|
358
|
-
_NULL_BLOCK = Block(kind="", text="", children=
|
|
386
|
+
_NULL_BLOCK = Block(kind="", text="", children=BlockList(), info="")
|
|
359
387
|
return _NULL_BLOCK
|
|
@@ -6,6 +6,7 @@ import json
|
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from typing import Iterator, List, Optional, Union
|
|
8
8
|
|
|
9
|
+
from markdown_extractor._collections import SectionList
|
|
9
10
|
from markdown_extractor.blocks import Block
|
|
10
11
|
from markdown_extractor.parser import parse
|
|
11
12
|
from markdown_extractor.section import Section
|
|
@@ -103,19 +104,21 @@ class MDExtractor:
|
|
|
103
104
|
"""Navigate by a sequence of titles (root → leaf)."""
|
|
104
105
|
return self._root.get_section(*path)
|
|
105
106
|
|
|
106
|
-
def find(self, title: str) ->
|
|
107
|
+
def find(self, title: str) -> SectionList:
|
|
107
108
|
"""Find every section whose title equals ``title`` (any depth)."""
|
|
108
109
|
return self._root.find(title)
|
|
109
110
|
|
|
110
|
-
def walk(self) ->
|
|
111
|
-
"""
|
|
112
|
-
for section in self._root.walk():
|
|
113
|
-
if section.level > 0:
|
|
114
|
-
yield section
|
|
111
|
+
def walk(self) -> SectionList:
|
|
112
|
+
"""Every header section in the document, depth-first.
|
|
115
113
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
114
|
+
Returns a :class:`SectionList` (the synthetic root is excluded),
|
|
115
|
+
so the result can be chained with ``.filtered(...)``.
|
|
116
|
+
"""
|
|
117
|
+
return self._root.walk().filtered(level__gt=0)
|
|
118
|
+
|
|
119
|
+
def headers(self) -> SectionList:
|
|
120
|
+
"""All header sections as a flat :class:`SectionList`."""
|
|
121
|
+
return self.walk()
|
|
119
122
|
|
|
120
123
|
def to_dict(self) -> dict:
|
|
121
124
|
"""JSON-friendly dict of the whole tree."""
|
{markdown_extractor-0.1.1 → markdown_extractor-0.2.0}/src/markdown_extractor/html_renderer.py
RENAMED
|
@@ -17,6 +17,14 @@ from html import escape
|
|
|
17
17
|
from typing import Iterable, List
|
|
18
18
|
|
|
19
19
|
from markdown_extractor.blocks import Block
|
|
20
|
+
from markdown_extractor.inline import (
|
|
21
|
+
_BOLD_RE,
|
|
22
|
+
_EM_STAR_RE,
|
|
23
|
+
_EM_UNDER_RE,
|
|
24
|
+
_IMG_RE,
|
|
25
|
+
_INLINE_CODE_RE,
|
|
26
|
+
_LINK_RE,
|
|
27
|
+
)
|
|
20
28
|
|
|
21
29
|
|
|
22
30
|
def render(blocks: Iterable[Block]) -> str:
|
|
@@ -49,13 +57,9 @@ def _render_block(block: Block) -> str:
|
|
|
49
57
|
# ---------------------------------------------------------------- inline
|
|
50
58
|
|
|
51
59
|
# Order matters: replace inline code first (so its contents are not further
|
|
52
|
-
# transformed), then images, links, bold, em.
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)\s]+)\)")
|
|
56
|
-
_BOLD_RE = re.compile(r"\*\*([^*]+)\*\*")
|
|
57
|
-
_EM_RE = re.compile(r"(?<![*\w])\*([^*\n]+?)\*(?!\w)")
|
|
58
|
-
_EM_UNDER_RE = re.compile(r"(?<![\w_])_([^_\n]+?)_(?!\w)")
|
|
60
|
+
# transformed), then images, links, bold, em. Regex patterns live in
|
|
61
|
+
# :mod:`markdown_extractor.inline` so the parser and both renderers
|
|
62
|
+
# share one source of truth.
|
|
59
63
|
|
|
60
64
|
|
|
61
65
|
def _inline(text: str) -> str:
|
|
@@ -76,7 +80,7 @@ def _inline(text: str) -> str:
|
|
|
76
80
|
def repl_code(m: re.Match) -> str:
|
|
77
81
|
return stash(f"<code>{escape(m.group(1))}</code>")
|
|
78
82
|
|
|
79
|
-
out =
|
|
83
|
+
out = _INLINE_CODE_RE.sub(repl_code, text)
|
|
80
84
|
out = escape(out, quote=False)
|
|
81
85
|
|
|
82
86
|
def repl_img(m: re.Match) -> str:
|
|
@@ -94,7 +98,7 @@ def _inline(text: str) -> str:
|
|
|
94
98
|
out = _IMG_RE.sub(repl_img, out)
|
|
95
99
|
out = _LINK_RE.sub(repl_link, out)
|
|
96
100
|
out = _BOLD_RE.sub(lambda m: f"<strong>{m.group(1)}</strong>", out)
|
|
97
|
-
out =
|
|
101
|
+
out = _EM_STAR_RE.sub(lambda m: f"<em>{m.group(1)}</em>", out)
|
|
98
102
|
out = _EM_UNDER_RE.sub(lambda m: f"<em>{m.group(1)}</em>", out)
|
|
99
103
|
|
|
100
104
|
# Restore stashed HTML.
|