primarysource 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- primarysource/__init__.py +5 -0
- primarysource/bbf/__init__.py +56 -0
- primarysource/bbf/_xml.py +63 -0
- primarysource/bbf/catalogue.json +777 -0
- primarysource/bbf/catalogue.py +137 -0
- primarysource/bbf/cli.py +79 -0
- primarysource/bbf/datamodel.py +281 -0
- primarysource/bbf/importer.py +151 -0
- primarysource/bbf/library.py +380 -0
- primarysource/bbf/server.py +72 -0
- primarysource/budget.py +63 -0
- primarysource/c2sp/__init__.py +53 -0
- primarysource/c2sp/_fetch.py +5 -0
- primarysource/c2sp/catalogue.json +231 -0
- primarysource/c2sp/catalogue.py +95 -0
- primarysource/c2sp/cli.py +62 -0
- primarysource/c2sp/importer.py +90 -0
- primarysource/c2sp/library.py +309 -0
- primarysource/c2sp/markdown.py +31 -0
- primarysource/c2sp/server.py +68 -0
- primarysource/cavp/__init__.py +42 -0
- primarysource/cavp/cli.py +84 -0
- primarysource/cavp/data/vectors.toml +47 -0
- primarysource/cavp/importer.py +156 -0
- primarysource/cavp/model.py +279 -0
- primarysource/cavp/parse.py +269 -0
- primarysource/cavp/paths.py +17 -0
- primarysource/cavp/server.py +67 -0
- primarysource/cavp/tools.py +169 -0
- primarysource/cli.py +107 -0
- primarysource/cursor.py +82 -0
- primarysource/cve/__init__.py +5 -0
- primarysource/cve/__main__.py +6 -0
- primarysource/cve/cli.py +144 -0
- primarysource/cve/collection.py +20 -0
- primarysource/cve/config.py +58 -0
- primarysource/cve/errors.py +16 -0
- primarysource/cve/kernel.py +232 -0
- primarysource/cve/mcp.py +139 -0
- primarysource/cve/parse.py +318 -0
- primarysource/cve/py.typed +0 -0
- primarysource/cve/service.py +760 -0
- primarysource/cve/store.py +214 -0
- primarysource/cve/sync.py +438 -0
- primarysource/cve/views.py +304 -0
- primarysource/debpolicy/__init__.py +16 -0
- primarysource/debpolicy/catalogue.py +263 -0
- primarysource/debpolicy/cli.py +12 -0
- primarysource/debpolicy/server.py +11 -0
- primarysource/draft/__init__.py +11 -0
- primarysource/draft/catalogue.py +117 -0
- primarysource/draft/cli.py +149 -0
- primarysource/draft/collection.py +21 -0
- primarysource/draft/datatracker.py +158 -0
- primarysource/draft/importer.py +163 -0
- primarysource/draft/licence.py +89 -0
- primarysource/draft/names.py +64 -0
- primarysource/draft/net.py +133 -0
- primarysource/draft/segment.py +192 -0
- primarysource/draft/server.py +91 -0
- primarysource/draft/status.py +146 -0
- primarysource/draft/store.py +199 -0
- primarysource/draft/tracked.json +104 -0
- primarysource/draft/views.py +430 -0
- primarysource/draft/xmlmeta.py +74 -0
- primarysource/ecma/__init__.py +52 -0
- primarysource/ecma/_pdf.py +143 -0
- primarysource/ecma/catalogue.json +22 -0
- primarysource/ecma/catalogue.py +100 -0
- primarysource/ecma/cli.py +67 -0
- primarysource/ecma/identify.py +25 -0
- primarysource/ecma/importer.py +108 -0
- primarysource/ecma/library.py +240 -0
- primarysource/ecma/server.py +67 -0
- primarysource/errors.py +29 -0
- primarysource/fetch.py +360 -0
- primarysource/fsutil.py +150 -0
- primarysource/gitdocs/__init__.py +15 -0
- primarysource/gitdocs/catalogue.py +116 -0
- primarysource/gitdocs/cli.py +12 -0
- primarysource/gitdocs/server.py +8 -0
- primarysource/home.py +65 -0
- primarysource/iana/__init__.py +32 -0
- primarysource/iana/catalogue.json +244 -0
- primarysource/iana/catalogue.py +69 -0
- primarysource/iana/cli.py +69 -0
- primarysource/iana/importer.py +157 -0
- primarysource/iana/model.py +325 -0
- primarysource/iana/reader.py +315 -0
- primarysource/iana/server.py +67 -0
- primarysource/iana/store.py +83 -0
- primarysource/iana/xmlsafe.py +47 -0
- primarysource/identify.py +186 -0
- primarysource/ieee/__init__.py +58 -0
- primarysource/ieee/_html.py +49 -0
- primarysource/ieee/catalogue.json +74 -0
- primarysource/ieee/catalogue.py +123 -0
- primarysource/ieee/cli.py +79 -0
- primarysource/ieee/identify.py +24 -0
- primarysource/ieee/ieee802.py +47 -0
- primarysource/ieee/importer.py +305 -0
- primarysource/ieee/posix.py +233 -0
- primarysource/ieee/server.py +79 -0
- primarysource/index.py +446 -0
- primarysource/iso/__init__.py +51 -0
- primarysource/iso/cli.py +98 -0
- primarysource/iso/designation.py +95 -0
- primarysource/iso/identify.py +100 -0
- primarysource/iso/importer.py +224 -0
- primarysource/iso/library.py +249 -0
- primarysource/iso/server.py +69 -0
- primarysource/itu/__init__.py +41 -0
- primarysource/itu/__main__.py +8 -0
- primarysource/itu/asn1mod.py +478 -0
- primarysource/itu/catalog.py +472 -0
- primarysource/itu/cli.py +445 -0
- primarysource/itu/collection.py +20 -0
- primarysource/itu/corpus.py +118 -0
- primarysource/itu/errata.py +16 -0
- primarysource/itu/formal.py +565 -0
- primarysource/itu/identify.py +26 -0
- primarysource/itu/index.py +490 -0
- primarysource/itu/mcp.py +474 -0
- primarysource/itu/py.typed +0 -0
- primarysource/itu/sources.py +600 -0
- primarysource/itu/text.py +609 -0
- primarysource/kconfig/__init__.py +16 -0
- primarysource/kconfig/catalogue.py +98 -0
- primarysource/kconfig/cli.py +12 -0
- primarysource/kconfig/server.py +8 -0
- primarysource/model.py +63 -0
- primarysource/nist/__init__.py +5 -0
- primarysource/nist/collection.py +46 -0
- primarysource/nist/data/acquired-nist.json +1657 -0
- primarysource/nist/data/nist-catalog.json +12090 -0
- primarysource/nist/data/sources.toml +1996 -0
- primarysource/nist/identify.py +41 -0
- primarysource/nist/ingest/__init__.py +1 -0
- primarysource/nist/ingest/acquire_nist.py +128 -0
- primarysource/nist/ingest/controls.py +82 -0
- primarysource/nist/ingest/discover_nist.py +218 -0
- primarysource/nist/ingest/extract.py +48 -0
- primarysource/nist/ingest/fetch.py +211 -0
- primarysource/nist/ingest/index.py +14 -0
- primarysource/nist/ingest/oscal.py +480 -0
- primarysource/nist/ingest/prose_layouts.py +236 -0
- primarysource/nist/ingest/sources.py +94 -0
- primarysource/nist/ingest/status.py +255 -0
- primarysource/nist/models.py +7 -0
- primarysource/nist/paths.py +33 -0
- primarysource/nist/server/__init__.py +3 -0
- primarysource/nist/server/corpus.py +38 -0
- primarysource/nist/server/mcp.py +19 -0
- primarysource/nist/server/tools.py +35 -0
- primarysource/nist/spec.py +38 -0
- primarysource/pdfcorpus/__init__.py +16 -0
- primarysource/pdfcorpus/corpus.py +159 -0
- primarysource/pdfcorpus/fts.py +175 -0
- primarysource/pdfcorpus/index.py +70 -0
- primarysource/pdfcorpus/models.py +30 -0
- primarysource/pdfcorpus/server.py +96 -0
- primarysource/pdfcorpus/spec.py +37 -0
- primarysource/pdfcorpus/split.py +391 -0
- primarysource/pdfcorpus/tools.py +244 -0
- primarysource/pep/__init__.py +15 -0
- primarysource/pep/catalogue.py +191 -0
- primarysource/pep/cli.py +12 -0
- primarysource/pep/server.py +8 -0
- primarysource/pkgdocs/__init__.py +19 -0
- primarysource/pkgdocs/fetch.py +356 -0
- primarysource/pkgdocs/index.py +89 -0
- primarysource/pkgdocs/model.py +76 -0
- primarysource/pkgdocs/readers/__init__.py +1 -0
- primarysource/pkgdocs/readers/_common.py +134 -0
- primarysource/pkgdocs/readers/asciidoc.py +160 -0
- primarysource/pkgdocs/readers/latex.py +315 -0
- primarysource/pkgdocs/readers/markdown.py +173 -0
- primarysource/pkgdocs/readers/pod.py +132 -0
- primarysource/pkgdocs/readers/rst.py +239 -0
- primarysource/pkgdocs/readers/scdoc.py +108 -0
- primarysource/pkgdocs/runtime.py +552 -0
- primarysource/pkgdocs/serving.py +158 -0
- primarysource/pkgdocs/store.py +167 -0
- primarysource/pms/__init__.py +16 -0
- primarysource/pms/catalogue.py +88 -0
- primarysource/pms/cli.py +12 -0
- primarysource/pms/server.py +8 -0
- primarysource/py.typed +0 -0
- primarysource/readers/__init__.py +10 -0
- primarysource/readers/bookmarks.py +192 -0
- primarysource/readers/html.py +1312 -0
- primarysource/readers/markdown.py +189 -0
- primarysource/readers/pdf.py +187 -0
- primarysource/readers/xml.py +230 -0
- primarysource/refs.py +87 -0
- primarysource/registry.py +161 -0
- primarysource/render.py +49 -0
- primarysource/rfc/__init__.py +3 -0
- primarysource/rfc/__main__.py +6 -0
- primarysource/rfc/cite.py +204 -0
- primarysource/rfc/cli.py +903 -0
- primarysource/rfc/collection.py +19 -0
- primarysource/rfc/config.py +138 -0
- primarysource/rfc/errata.py +203 -0
- primarysource/rfc/fsutil.py +82 -0
- primarysource/rfc/graph.py +213 -0
- primarysource/rfc/index/__init__.py +1 -0
- primarysource/rfc/index/chunker.py +281 -0
- primarysource/rfc/index/embed.py +166 -0
- primarysource/rfc/index/hybrid.py +305 -0
- primarysource/rfc/index/lexical.py +223 -0
- primarysource/rfc/index/rfcxml.py +673 -0
- primarysource/rfc/index/search.py +508 -0
- primarysource/rfc/index/txtseg.py +585 -0
- primarysource/rfc/library.py +294 -0
- primarysource/rfc/mcp/__init__.py +1 -0
- primarysource/rfc/mcp/server.py +381 -0
- primarysource/rfc/packs/__init__.py +1 -0
- primarysource/rfc/packs/builder.py +228 -0
- primarysource/rfc/packs/collections/all-current-txt.json +10 -0
- primarysource/rfc/packs/collections/essentials.json +19 -0
- primarysource/rfc/packs/collections/fyi-current.json +10 -0
- primarysource/rfc/packs/collections/standards-current.json +10 -0
- primarysource/rfc/packs/collections/year-2025.json +10 -0
- primarysource/rfc/packs/collections/year-2026.json +10 -0
- primarysource/rfc/packs/collections.py +163 -0
- primarysource/rfc/packs/docsetexport.py +163 -0
- primarysource/rfc/packs/epub.py +159 -0
- primarysource/rfc/packs/export.py +306 -0
- primarysource/rfc/packs/htmlexport.py +82 -0
- primarysource/rfc/packs/serve.py +143 -0
- primarysource/rfc/packs/zimexport.py +171 -0
- primarysource/rfc/query.py +466 -0
- primarysource/rfc/subjects.py +457 -0
- primarysource/rfc/sync/__init__.py +1 -0
- primarysource/rfc/sync/downloader.py +132 -0
- primarysource/rfc/sync/manifest.py +162 -0
- primarysource/rfc/sync/scanner.py +49 -0
- primarysource/rfc/sync/state.py +464 -0
- primarysource/rfc/views.py +1067 -0
- primarysource/rpmdocs/__init__.py +15 -0
- primarysource/rpmdocs/catalogue.py +117 -0
- primarysource/rpmdocs/cli.py +12 -0
- primarysource/rpmdocs/server.py +8 -0
- primarysource/scaffold.py +223 -0
- primarysource/serve.py +75 -0
- primarysource/settings.py +152 -0
- primarysource/usb/__init__.py +5 -0
- primarysource/usb/collection.py +37 -0
- primarysource/usb/data/acquired.json +2025 -0
- primarysource/usb/data/sources-usb.toml +852 -0
- primarysource/usb/data/sources.toml +10 -0
- primarysource/usb/data/usb-catalog.json +1663 -0
- primarysource/usb/data/usb-classification.json +1986 -0
- primarysource/usb/data/usb-corpus-build.json +608 -0
- primarysource/usb/data/usb-pdf-choice.json +839 -0
- primarysource/usb/data/usb-split-measure.json +1803 -0
- primarysource/usb/ingest/__init__.py +1 -0
- primarysource/usb/ingest/acquire.py +130 -0
- primarysource/usb/ingest/discover_usb.py +88 -0
- primarysource/usb/ingest/extract.py +48 -0
- primarysource/usb/ingest/fetch.py +197 -0
- primarysource/usb/ingest/index.py +14 -0
- primarysource/usb/ingest/sources.py +85 -0
- primarysource/usb/models.py +7 -0
- primarysource/usb/paths.py +33 -0
- primarysource/usb/server/__init__.py +3 -0
- primarysource/usb/server/corpus.py +38 -0
- primarysource/usb/server/mcp.py +19 -0
- primarysource/usb/server/tools.py +35 -0
- primarysource/usb/spec.py +20 -0
- primarysource/usbids/__init__.py +56 -0
- primarysource/usbids/cli.py +65 -0
- primarysource/usbids/importer.py +272 -0
- primarysource/usbids/parse.py +283 -0
- primarysource/usbids/server.py +55 -0
- primarysource/usbids/store.py +80 -0
- primarysource/usbids/tools.py +174 -0
- primarysource/zim/__init__.py +5 -0
- primarysource/zim/__main__.py +8 -0
- primarysource/zim/_libzim.py +37 -0
- primarysource/zim/catalog.py +594 -0
- primarysource/zim/cli.py +555 -0
- primarysource/zim/collection.py +41 -0
- primarysource/zim/errors.py +100 -0
- primarysource/zim/groups.py +30 -0
- primarysource/zim/library.py +668 -0
- primarysource/zim/manage.py +636 -0
- primarysource/zim/markdown.py +1247 -0
- primarysource/zim/mcp.py +201 -0
- primarysource/zim/purl.py +130 -0
- primarysource/zim/py.typed +0 -0
- primarysource/zim/refs.py +90 -0
- primarysource/zim/service.py +1182 -0
- primarysource/zim/snapshot.py +197 -0
- primarysource/zim/sync.py +269 -0
- primarysource/zim/views.py +230 -0
- primarysource-0.1.0.dist-info/METADATA +118 -0
- primarysource-0.1.0.dist-info/RECORD +303 -0
- primarysource-0.1.0.dist-info/WHEEL +4 -0
- primarysource-0.1.0.dist-info/entry_points.txt +2 -0
- primarysource-0.1.0.dist-info/licenses/LICENSE +203 -0
- primarysource-0.1.0.dist-info/licenses/NOTICE +51 -0
|
@@ -0,0 +1,552 @@
|
|
|
1
|
+
"""What a pkgdocs collection does: import a pinned edition, then docs, outline, read, search, cite, status.
|
|
2
|
+
|
|
3
|
+
A collection is a `Spec`: its trees (each a pinned source repository, with the files to take from it and a
|
|
4
|
+
`build` that turns them into documents), its licence and placement, and the spelling of its document ids. This
|
|
5
|
+
module holds the behaviour every one of them shares, so a collection directory is data plus a `build`.
|
|
6
|
+
|
|
7
|
+
Refs: `[<collection>:]<doc>[@<commit>][§<section>][#<anchor>][¶<n>]`. `§` takes a section id (`5.6.1`), an anchor
|
|
8
|
+
(`s-f-Source`), a heading, or a unique part of one; `@<commit>` (four or more hex digits, or a pin's label) picks
|
|
9
|
+
an imported edition other than the newest; `¶n` is the nth block of the section.
|
|
10
|
+
|
|
11
|
+
A read of `§s¶n` starts at block n and runs on through the subsections of `s`, so a count past the section's own
|
|
12
|
+
blocks goes on into them, one unit per subsection heading and per block, in reading order. That is what makes a
|
|
13
|
+
read's `next:` footer a ref: `§s¶<n>` where the next page starts. Lists (`docs`, `outline`, `search`) take a
|
|
14
|
+
0-based `start` and print `next: start=<n>`.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import re
|
|
20
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
21
|
+
from dataclasses import dataclass, field
|
|
22
|
+
from datetime import UTC, datetime
|
|
23
|
+
from difflib import get_close_matches
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
from primarysource import cursor
|
|
27
|
+
from primarysource.budget import page
|
|
28
|
+
from primarysource.errors import LibError
|
|
29
|
+
from primarysource.home import collection_home
|
|
30
|
+
from primarysource.model import Unit, span_hash
|
|
31
|
+
from primarysource.pkgdocs import fetch, index, store
|
|
32
|
+
from primarysource.pkgdocs.model import Doc, Sec
|
|
33
|
+
from primarysource.refs import Ref, RefGrammar
|
|
34
|
+
from primarysource.registry import Collection
|
|
35
|
+
from primarysource.render import next_start, render
|
|
36
|
+
|
|
37
|
+
DOC_PATTERN = r"[A-Za-z0-9][A-Za-z0-9._/+-]*(?:@[0-9A-Za-z][0-9A-Za-z._-]{1,39})?"
|
|
38
|
+
SECTION_PATTERN = r"[^#¶\n]+?"
|
|
39
|
+
_HASH_ESCAPE = "%23"
|
|
40
|
+
_BARE = re.compile(r"(?P<section>\d+(?:\.\d+)+)(?:@(?P<commit>[0-9A-Za-z][\w.-]{1,39}))?(?P<para>¶\d+)?")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True, slots=True)
|
|
44
|
+
class Licence:
|
|
45
|
+
"""The licence of a collection's text and where it may be placed. `redistribute` is `layer` (may be baked
|
|
46
|
+
into an image), `volume` (user-supplied, never in an image) or `none`. `evidence` names the file in the
|
|
47
|
+
source repository the licence was read from. `attribution` is the credit shown by `status`; `credit` is the
|
|
48
|
+
shorter form a licence that requires attribution (CC-BY-SA) needs in every citation."""
|
|
49
|
+
|
|
50
|
+
spdx: str
|
|
51
|
+
redistribute: str
|
|
52
|
+
attribution: str = ""
|
|
53
|
+
evidence: str = ""
|
|
54
|
+
credit: str = ""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass(frozen=True, slots=True)
|
|
58
|
+
class Pin:
|
|
59
|
+
"""One edition of a tree: a full commit, a human label (a tag), its date, the `fetch.manifest_digest` of the
|
|
60
|
+
files taken from it, and where to fetch it (the sources are mirrors of each other and must give that digest)."""
|
|
61
|
+
|
|
62
|
+
commit: str
|
|
63
|
+
label: str
|
|
64
|
+
date: str
|
|
65
|
+
files_sha256: str | None
|
|
66
|
+
sources: tuple[fetch.Source, ...]
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(frozen=True, slots=True)
|
|
70
|
+
class Build:
|
|
71
|
+
"""What a tree's `build` returns: the documents, chapter aliases (`5` -> `ch-controlfields`) and counts of the
|
|
72
|
+
files it left out, by reason."""
|
|
73
|
+
|
|
74
|
+
docs: tuple[Doc, ...]
|
|
75
|
+
aliases: Mapping[str, str] = field(default_factory=dict)
|
|
76
|
+
skipped: Mapping[str, int] = field(default_factory=dict)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass(frozen=True, slots=True)
|
|
80
|
+
class TreeSpec:
|
|
81
|
+
"""One pinned repository (or one subtree of it). `pins[0]` is the current edition."""
|
|
82
|
+
|
|
83
|
+
name: str
|
|
84
|
+
title: str
|
|
85
|
+
pins: tuple[Pin, ...]
|
|
86
|
+
include: Callable[[str], bool]
|
|
87
|
+
build: Callable[[Mapping[str, bytes], Pin], Build]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass(frozen=True, slots=True)
|
|
91
|
+
class Spec:
|
|
92
|
+
"""Everything that is particular to one collection."""
|
|
93
|
+
|
|
94
|
+
name: str
|
|
95
|
+
title: str
|
|
96
|
+
licence: Licence
|
|
97
|
+
trees: tuple[TreeSpec, ...]
|
|
98
|
+
instructions: str
|
|
99
|
+
example_refs: tuple[str, ...] = ()
|
|
100
|
+
|
|
101
|
+
@property
|
|
102
|
+
def prefix(self) -> str:
|
|
103
|
+
return f"{self.name}_"
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def grammar(self) -> RefGrammar:
|
|
107
|
+
return RefGrammar(DOC_PATTERN, SECTION_PATTERN)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@dataclass(frozen=True, slots=True)
|
|
111
|
+
class Report:
|
|
112
|
+
"""What one tree's import did."""
|
|
113
|
+
|
|
114
|
+
tree: str
|
|
115
|
+
commit: str
|
|
116
|
+
label: str
|
|
117
|
+
path: Path
|
|
118
|
+
docs: int
|
|
119
|
+
sections: int
|
|
120
|
+
units: int
|
|
121
|
+
files_sha256: str
|
|
122
|
+
pinned: bool
|
|
123
|
+
skipped: dict[str, int]
|
|
124
|
+
warnings: int
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _norm(text: str) -> str:
|
|
128
|
+
return " ".join(text.casefold().split()).rstrip(":.;,")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _now() -> str:
|
|
132
|
+
return datetime.now(UTC).isoformat(timespec="seconds")
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class Library:
|
|
136
|
+
"""A collection's installed editions, read through a `Spec`. `home` replaces `$PRIMARYSOURCE_HOME` (tests)."""
|
|
137
|
+
|
|
138
|
+
def __init__(self, spec: Spec, home: Path | None = None) -> None:
|
|
139
|
+
self.spec = spec
|
|
140
|
+
self._home = home
|
|
141
|
+
|
|
142
|
+
# -- locations -------------------------------------------------------------------------------------------
|
|
143
|
+
|
|
144
|
+
def base(self, tree: str) -> Path:
|
|
145
|
+
root = self._home / self.spec.name if self._home is not None else collection_home(self.spec.name)
|
|
146
|
+
return root / tree
|
|
147
|
+
|
|
148
|
+
def _import(self) -> str:
|
|
149
|
+
return f"primarysource {self.spec.name} import"
|
|
150
|
+
|
|
151
|
+
def _editions(self) -> list[tuple[TreeSpec, store.Edition]]:
|
|
152
|
+
"""The newest installed edition of each tree that has one."""
|
|
153
|
+
found: list[tuple[TreeSpec, store.Edition]] = []
|
|
154
|
+
for tree in self.spec.trees:
|
|
155
|
+
editions = store.installed(self.base(tree.name))
|
|
156
|
+
if editions:
|
|
157
|
+
found.append((tree, editions[0]))
|
|
158
|
+
if not found:
|
|
159
|
+
raise LibError(
|
|
160
|
+
f"{self.spec.title} is not imported (nothing under {self.base('')})",
|
|
161
|
+
f"{self.spec.prefix}status says what is pinned",
|
|
162
|
+
self._import(),
|
|
163
|
+
)
|
|
164
|
+
return found
|
|
165
|
+
|
|
166
|
+
def _by_commit(self, token: str) -> list[tuple[TreeSpec, store.Edition]]:
|
|
167
|
+
want = token.lower()
|
|
168
|
+
matches: list[tuple[TreeSpec, store.Edition]] = []
|
|
169
|
+
for tree in self.spec.trees:
|
|
170
|
+
labels = {p.label.lower(): p.commit for p in tree.pins}
|
|
171
|
+
for edition in store.installed(self.base(tree.name)):
|
|
172
|
+
if edition.commit.startswith(want) or labels.get(want) == edition.commit:
|
|
173
|
+
matches.append((tree, edition))
|
|
174
|
+
return matches
|
|
175
|
+
|
|
176
|
+
# -- import ----------------------------------------------------------------------------------------------
|
|
177
|
+
|
|
178
|
+
def import_edition(
|
|
179
|
+
self,
|
|
180
|
+
tree: str | None = None,
|
|
181
|
+
commit: str | None = None,
|
|
182
|
+
archive: str | None = None,
|
|
183
|
+
*,
|
|
184
|
+
opener: fetch.Opener | None = None,
|
|
185
|
+
throttle: fetch.Throttle | None = None,
|
|
186
|
+
limits: fetch.Limits | None = None,
|
|
187
|
+
allow_unpinned: bool = False,
|
|
188
|
+
) -> list[Report]:
|
|
189
|
+
"""Fetch (or read `archive`, a downloaded copy of) each selected tree at its pinned commit and install it."""
|
|
190
|
+
chosen = [t for t in self.spec.trees if tree in (None, t.name)]
|
|
191
|
+
if not chosen:
|
|
192
|
+
names = ", ".join(t.name for t in self.spec.trees)
|
|
193
|
+
raise LibError(f"no tree named {tree!r} in {self.spec.name} (trees: {names})", "")
|
|
194
|
+
if archive is not None and len(chosen) != 1:
|
|
195
|
+
names = ", ".join(t.name for t in self.spec.trees)
|
|
196
|
+
raise LibError("--archive needs --tree, because each tree has its own archive", f"trees: {names}")
|
|
197
|
+
return [self._import_tree(t, commit, archive, opener, throttle, limits, allow_unpinned) for t in chosen]
|
|
198
|
+
|
|
199
|
+
def _import_tree(
|
|
200
|
+
self,
|
|
201
|
+
tree: TreeSpec,
|
|
202
|
+
commit: str | None,
|
|
203
|
+
archive: str | None,
|
|
204
|
+
opener: fetch.Opener | None,
|
|
205
|
+
throttle: fetch.Throttle | None,
|
|
206
|
+
limits: fetch.Limits | None,
|
|
207
|
+
allow_unpinned: bool,
|
|
208
|
+
) -> Report:
|
|
209
|
+
pin = tree.pins[0]
|
|
210
|
+
if commit is not None:
|
|
211
|
+
matches = [p for p in tree.pins if p.commit.startswith(commit.lower()) or p.label == commit]
|
|
212
|
+
if len(matches) != 1:
|
|
213
|
+
have = ", ".join(f"{p.label} ({p.commit[:12]})" for p in tree.pins)
|
|
214
|
+
raise LibError(f"{commit!r} is not one pinned edition of {tree.name} (pinned: {have})", "")
|
|
215
|
+
pin = matches[0]
|
|
216
|
+
expect = pin.files_sha256
|
|
217
|
+
if expect is None and not allow_unpinned:
|
|
218
|
+
raise LibError(
|
|
219
|
+
f"{tree.name} {pin.label} has no recorded digest",
|
|
220
|
+
"record the digest in the catalogue, or pass --allow-unpinned to trust this one fetch",
|
|
221
|
+
)
|
|
222
|
+
if allow_unpinned:
|
|
223
|
+
expect = None
|
|
224
|
+
if archive is not None:
|
|
225
|
+
first = pin.sources[0]
|
|
226
|
+
source = fetch.Source(archive, first.strip, subdir=first.subdir)
|
|
227
|
+
fetched = fetch.read_local(archive, source, include=tree.include, expect=expect, limits=limits)
|
|
228
|
+
else:
|
|
229
|
+
fetched = fetch.fetch_tree(
|
|
230
|
+
pin.sources, include=tree.include, expect=expect, limits=limits, opener=opener, throttle=throttle
|
|
231
|
+
)
|
|
232
|
+
built = tree.build(fetched.files, pin)
|
|
233
|
+
ids = [d.id for d in built.docs]
|
|
234
|
+
if not ids:
|
|
235
|
+
raise LibError(f"nothing in the {tree.name} archive was selected", "the source layout may have changed")
|
|
236
|
+
if len(set(ids)) != len(ids):
|
|
237
|
+
repeated = sorted({i for i in ids if ids.count(i) > 1})[:5]
|
|
238
|
+
raise LibError(
|
|
239
|
+
f"the build produced duplicate document ids: {', '.join(repeated)}", "report the collection"
|
|
240
|
+
)
|
|
241
|
+
record = {
|
|
242
|
+
"collection": self.spec.name,
|
|
243
|
+
"tree": tree.name,
|
|
244
|
+
"label": pin.label,
|
|
245
|
+
"date": pin.date,
|
|
246
|
+
"url": fetched.url,
|
|
247
|
+
"files_sha256": fetched.files_sha256,
|
|
248
|
+
"tar_sha256": fetched.tar_sha256,
|
|
249
|
+
"gz_sha256": fetched.gz_sha256,
|
|
250
|
+
"pinned": fetched.pinned,
|
|
251
|
+
"archive_members": fetched.members,
|
|
252
|
+
"selected_files": len(fetched.files),
|
|
253
|
+
"selected_bytes": fetched.kept_bytes,
|
|
254
|
+
"skipped": {**fetched.skipped, **dict(built.skipped)},
|
|
255
|
+
"licence": {
|
|
256
|
+
"spdx": self.spec.licence.spdx,
|
|
257
|
+
"redistribute": self.spec.licence.redistribute,
|
|
258
|
+
"attribution": self.spec.licence.attribution,
|
|
259
|
+
"evidence": self.spec.licence.evidence,
|
|
260
|
+
},
|
|
261
|
+
"imported": _now(),
|
|
262
|
+
}
|
|
263
|
+
path = store.write_edition(self.base(tree.name), pin.commit, record, built.docs, built.aliases)
|
|
264
|
+
sections = sum(len(d.sections) for d in built.docs)
|
|
265
|
+
return Report(
|
|
266
|
+
tree.name,
|
|
267
|
+
pin.commit,
|
|
268
|
+
pin.label,
|
|
269
|
+
path,
|
|
270
|
+
len(built.docs),
|
|
271
|
+
sections,
|
|
272
|
+
sum(len(s.units) for d in built.docs for s in d.sections),
|
|
273
|
+
fetched.files_sha256,
|
|
274
|
+
fetched.pinned,
|
|
275
|
+
{**fetched.skipped, **dict(built.skipped)},
|
|
276
|
+
sum(len(d.warnings) for d in built.docs),
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
# -- ref resolution --------------------------------------------------------------------------------------
|
|
280
|
+
|
|
281
|
+
def _parse(self, text: str) -> tuple[Ref, str | None]:
|
|
282
|
+
"""(parsed ref, commit token or None). A leading `<collection>:` is accepted and ignored."""
|
|
283
|
+
cleaned = text.strip()
|
|
284
|
+
head = f"{self.spec.name}:"
|
|
285
|
+
if cleaned.lower().startswith(head):
|
|
286
|
+
cleaned = cleaned[len(head) :]
|
|
287
|
+
bare = _BARE.fullmatch(cleaned)
|
|
288
|
+
if bare: # `5.6.1`, `5.6.1@aceb1daf`, `5.6.1¶2`: a section number with no document
|
|
289
|
+
ref = self.spec.grammar.parse(f"§{bare['section']}{bare['para'] or ''}")
|
|
290
|
+
return ref, bare["commit"]
|
|
291
|
+
ref = self.spec.grammar.parse(cleaned)
|
|
292
|
+
doc, commit = ref.doc, None
|
|
293
|
+
if doc is not None and "@" in doc:
|
|
294
|
+
doc, _, commit = doc.partition("@")
|
|
295
|
+
section = ref.section.replace(_HASH_ESCAPE, "#") if ref.section else ref.section
|
|
296
|
+
return Ref(doc or None, section, ref.para, ref.anchor, ref.lines), commit
|
|
297
|
+
|
|
298
|
+
def _find_doc(self, token: str, commit: str | None) -> tuple[TreeSpec, store.Edition, str]:
|
|
299
|
+
pool = self._by_commit(commit) if commit else self._editions()
|
|
300
|
+
if commit and not pool:
|
|
301
|
+
raise LibError(f"no imported edition matches {commit!r}", f"{self.spec.prefix}status lists them")
|
|
302
|
+
for tree, edition in pool:
|
|
303
|
+
if token in edition.entries:
|
|
304
|
+
return tree, edition, token
|
|
305
|
+
if token in edition.aliases:
|
|
306
|
+
return tree, edition, edition.aliases[token]
|
|
307
|
+
lowered = {k.lower(): k for _, e in pool for k in e.entries}
|
|
308
|
+
if token.lower() in lowered:
|
|
309
|
+
for tree, edition in pool:
|
|
310
|
+
if lowered[token.lower()] in edition.entries:
|
|
311
|
+
return tree, edition, lowered[token.lower()]
|
|
312
|
+
known = sorted({k for _, e in pool for k in e.entries})
|
|
313
|
+
close = get_close_matches(token, known, n=5, cutoff=0.6)
|
|
314
|
+
hint = f"did you mean: {', '.join(close)}" if close else f"{self.spec.prefix}docs lists them"
|
|
315
|
+
raise LibError(f"no document {token!r} in {self.spec.title}", hint)
|
|
316
|
+
|
|
317
|
+
def _section(self, doc: Doc, token: str | None, anchor: str | None) -> tuple[Sec, str]:
|
|
318
|
+
"""The section a ref names and a warning for the footer (empty when there is none)."""
|
|
319
|
+
if token is None and anchor is None:
|
|
320
|
+
if not doc.sections:
|
|
321
|
+
raise LibError(f"{doc.id} has no sections", "")
|
|
322
|
+
return doc.sections[0], ""
|
|
323
|
+
for key in (token, anchor):
|
|
324
|
+
if not key:
|
|
325
|
+
continue
|
|
326
|
+
found = [s for s in doc.sections if s.id == key] or [s for s in doc.sections if s.anchor == key]
|
|
327
|
+
if not found:
|
|
328
|
+
found = [s for s in doc.sections if s.anchor.lower() == key.lower()]
|
|
329
|
+
if found:
|
|
330
|
+
return found[0], ""
|
|
331
|
+
want = _norm(token or anchor or "")
|
|
332
|
+
exact = [s for s in doc.sections if _norm(s.title) == want]
|
|
333
|
+
if exact:
|
|
334
|
+
return exact[0], (
|
|
335
|
+
f"{len(exact)} sections are titled {token!r}; the first was read" if len(exact) > 1 else ""
|
|
336
|
+
)
|
|
337
|
+
partial = [s for s in doc.sections if want and want in _norm(s.title)]
|
|
338
|
+
if len(partial) > 1:
|
|
339
|
+
starts = [s for s in partial if _norm(s.title).startswith(want)]
|
|
340
|
+
partial = starts if len(starts) == 1 else partial
|
|
341
|
+
if len(partial) == 1:
|
|
342
|
+
return partial[0], ""
|
|
343
|
+
if partial:
|
|
344
|
+
listing = "; ".join(f"{s.id} {s.title}" for s in partial[:8])
|
|
345
|
+
raise LibError(f"{token!r} matches {len(partial)} sections of {doc.id}", f"name one by id: {listing}")
|
|
346
|
+
titles = [s.title for s in doc.sections]
|
|
347
|
+
close = get_close_matches(token or "", titles, n=3, cutoff=0.5)
|
|
348
|
+
hint = f"did you mean: {'; '.join(close)}" if close else f"{self.spec.prefix}outline('{doc.id}') lists them"
|
|
349
|
+
raise LibError(f"no section {token or anchor!r} in {doc.id}", hint)
|
|
350
|
+
|
|
351
|
+
def _target(self, text: str, *, subtree: bool = False) -> tuple[store.Edition, Doc, Sec, str, int | None]:
|
|
352
|
+
"""The edition, document, section, warning and `¶n` a ref names. `subtree` lets `¶n` count on past the
|
|
353
|
+
section's own blocks into its subsections, as a read does."""
|
|
354
|
+
ref, commit = self._parse(text)
|
|
355
|
+
if ref.lines is not None:
|
|
356
|
+
raise LibError("a line range is not a ref in this collection", "name a section: <doc>§<section>")
|
|
357
|
+
doc_token, section = ref.doc, ref.section
|
|
358
|
+
if doc_token is None:
|
|
359
|
+
if section is None:
|
|
360
|
+
raise LibError(f"not a ref: {text!r}", "expected <doc>§<section>[¶n]")
|
|
361
|
+
# a bare section number: its first component names a chapter alias, as in `5.6.1`
|
|
362
|
+
doc_token = section.split(".")[0]
|
|
363
|
+
elif section is None and ref.anchor is None and doc_token not in self._known_ids(commit) and "." in doc_token:
|
|
364
|
+
doc_token, section = doc_token.split(".")[0], doc_token
|
|
365
|
+
_, edition, doc_id = self._find_doc(doc_token, commit)
|
|
366
|
+
doc = edition.load(doc_id)
|
|
367
|
+
sec, warning = self._section(doc, section, ref.anchor)
|
|
368
|
+
count = len(self._ordered(doc, sec)) - 1 if subtree else len(sec.units)
|
|
369
|
+
if ref.para is not None and not 1 <= ref.para <= count:
|
|
370
|
+
what = "blocks" if count == len(sec.units) else "blocks and headings with its subsections"
|
|
371
|
+
raise LibError(
|
|
372
|
+
f"{doc.id}§{sec.id} has {count} {what}; there is no ¶{ref.para}",
|
|
373
|
+
f"{self.spec.prefix}read('{self._ref(edition, doc, sec)}')",
|
|
374
|
+
)
|
|
375
|
+
return edition, doc, sec, warning, ref.para
|
|
376
|
+
|
|
377
|
+
def _known_ids(self, commit: str | None) -> set[str]:
|
|
378
|
+
pool = self._by_commit(commit) if commit else self._editions()
|
|
379
|
+
return {k for _, e in pool for k in (*e.entries, *e.aliases)}
|
|
380
|
+
|
|
381
|
+
def _ref(self, edition: store.Edition, doc: Doc, sec: Sec, para: int | None = None) -> str:
|
|
382
|
+
text = f"{self.spec.name}:{doc.id}@{edition.commit[:12]}§{sec.id}"
|
|
383
|
+
return text + (f"¶{para}" if para else "")
|
|
384
|
+
|
|
385
|
+
def _citation(self, edition: store.Edition, doc: Doc, sec: Sec, para: int | None) -> str:
|
|
386
|
+
source = doc.meta.get("path", "")
|
|
387
|
+
label = f" {edition.label}" if edition.label else ""
|
|
388
|
+
tree = str(edition.record.get("tree", ""))
|
|
389
|
+
lic = self.spec.licence
|
|
390
|
+
parts = [
|
|
391
|
+
f'{doc.title}, "{sec.title}" §{sec.id}' + (f" ¶{para}" if para else ""),
|
|
392
|
+
f"{self.spec.title} ({tree}{label}, commit {edition.commit[:12]})",
|
|
393
|
+
]
|
|
394
|
+
if source:
|
|
395
|
+
parts.append(source)
|
|
396
|
+
parts.append(f"licence {lic.spdx}" + (f" ({lic.credit})" if lic.credit else ""))
|
|
397
|
+
return self._ref(edition, doc, sec, para) + " -- " + "; ".join(parts)
|
|
398
|
+
|
|
399
|
+
# -- the tools -------------------------------------------------------------------------------------------
|
|
400
|
+
|
|
401
|
+
@staticmethod
|
|
402
|
+
def _ordered(doc: Doc, sec: Sec) -> list[str]:
|
|
403
|
+
"""The units a read of `sec` pages through: its heading, its blocks, then each subsection's heading and
|
|
404
|
+
blocks. `¶n` is `ordered[n]`, so the section's own `¶n` keeps its meaning."""
|
|
405
|
+
ordered: list[str] = []
|
|
406
|
+
inside = False
|
|
407
|
+
for candidate in doc.sections:
|
|
408
|
+
if candidate is sec:
|
|
409
|
+
inside = True
|
|
410
|
+
elif inside and candidate.depth <= sec.depth:
|
|
411
|
+
break
|
|
412
|
+
if inside:
|
|
413
|
+
ordered.append("#" * min(candidate.depth, 6) + f" {candidate.id} {candidate.title}")
|
|
414
|
+
ordered.extend(candidate.units)
|
|
415
|
+
return ordered
|
|
416
|
+
|
|
417
|
+
def docs(self, prefix: str = "", limit: int = 200, start: int = 0) -> str:
|
|
418
|
+
"""One line per document: `<id> <title>`, optionally those whose id starts with `prefix`."""
|
|
419
|
+
rows: list[str] = []
|
|
420
|
+
for _, edition in self._editions():
|
|
421
|
+
rows.extend(f"{e.id} {e.title}" for e in edition.entries.values() if e.id.startswith(prefix))
|
|
422
|
+
if not rows:
|
|
423
|
+
return f"no documents with an id starting {prefix!r}\n" if prefix else "no documents\n"
|
|
424
|
+
chosen, nxt = cursor.window(rows, start, limit)
|
|
425
|
+
return render("\n".join(chosen), {"unit": f"{len(rows)} documents", "next": nxt})
|
|
426
|
+
|
|
427
|
+
def outline(self, ref: str, start: int = 0, max_tokens: int = 4000) -> str:
|
|
428
|
+
"""A document's sections, each with the ref that reads it. `ref` names a document (`gitformat-pack`)."""
|
|
429
|
+
parsed, commit = self._parse(ref)
|
|
430
|
+
token = parsed.doc or (parsed.section or "").split(".")[0]
|
|
431
|
+
if not token:
|
|
432
|
+
raise LibError(f"not a document: {ref!r}", f"primarysource {self.spec.name} docs")
|
|
433
|
+
_, edition, doc_id = self._find_doc(token, commit)
|
|
434
|
+
doc = edition.load(doc_id)
|
|
435
|
+
lines = [
|
|
436
|
+
f"{' ' * (s.depth - 1)}{s.id} {s.title} [{len(s.units)}] {self._ref(edition, doc, s)}"
|
|
437
|
+
for s in doc.sections
|
|
438
|
+
]
|
|
439
|
+
if start < 0:
|
|
440
|
+
raise LibError(f"start is a 0-based offset, not {start}", "pass the start= value of the last next: line")
|
|
441
|
+
taken = page(lines, start + 1, max_tokens)
|
|
442
|
+
footer: dict[str, str] = {"cite": self._citation(edition, doc, doc.sections[0], None)} if doc.sections else {}
|
|
443
|
+
footer["unit"] = "block (the [n] after a title counts them)"
|
|
444
|
+
footer["next"] = next_start(None if taken.next_start is None else taken.next_start - 1)
|
|
445
|
+
if doc.warnings:
|
|
446
|
+
footer["warning"] = "; ".join(doc.warnings)
|
|
447
|
+
return render(f"{doc.title}\n" + "\n".join(taken.units), footer)
|
|
448
|
+
|
|
449
|
+
def read(self, ref: str, max_tokens: int = 2000) -> str:
|
|
450
|
+
"""The text of a section, its subsections included, cut at block boundaries: from its heading, or from block
|
|
451
|
+
n with `¶n`. The `next:` footer is the ref that reads the next page."""
|
|
452
|
+
edition, doc, sec, warning, para = self._target(ref, subtree=True)
|
|
453
|
+
if max_tokens < 1:
|
|
454
|
+
raise LibError("max_tokens must be positive", "")
|
|
455
|
+
taken = page(self._ordered(doc, sec), (para or 0) + 1, max_tokens)
|
|
456
|
+
body = "\n\n".join(taken.units)
|
|
457
|
+
one = para if para is not None and len(taken.units) == 1 and para <= len(sec.units) else None
|
|
458
|
+
footer = {
|
|
459
|
+
"cite": self._citation(edition, doc, sec, one),
|
|
460
|
+
"unit": "block (¶n counts the blocks of one section, then on through its subsections)",
|
|
461
|
+
"span": span_hash(body),
|
|
462
|
+
"next": "" if taken.next_start is None else self._ref(edition, doc, sec, taken.next_start - 1),
|
|
463
|
+
}
|
|
464
|
+
notes = [w for w in (warning, *doc.warnings) if w]
|
|
465
|
+
if notes:
|
|
466
|
+
footer["warning"] = "; ".join(notes)
|
|
467
|
+
return render(body, footer)
|
|
468
|
+
|
|
469
|
+
def cite(self, ref: str) -> str:
|
|
470
|
+
"""The citation for a ref, with the hash of the exact text cited, and nothing else."""
|
|
471
|
+
edition, doc, sec, _, para = self._target(ref)
|
|
472
|
+
text = sec.units[para - 1] if para else "\n\n".join(sec.units)
|
|
473
|
+
return render(self._citation(edition, doc, sec, para), {"span": span_hash(text)})
|
|
474
|
+
|
|
475
|
+
def search(self, query: str, limit: int = 10, start: int = 0, doc_prefix: str = "") -> str:
|
|
476
|
+
"""Sections matching the words of `query`, best first, one line each with the ref that reads it."""
|
|
477
|
+
if start < 0 or limit < 1:
|
|
478
|
+
raise LibError(f"start is a 0-based offset and limit positive (got {start}, {limit})", "")
|
|
479
|
+
scored: list[tuple[float, str]] = []
|
|
480
|
+
relaxed = False
|
|
481
|
+
for _, edition in self._editions():
|
|
482
|
+
hits, was_relaxed = index.search(
|
|
483
|
+
edition.index_path, query, limit=start + limit + 1, doc_prefix=doc_prefix
|
|
484
|
+
)
|
|
485
|
+
relaxed = relaxed or was_relaxed
|
|
486
|
+
for hit in hits:
|
|
487
|
+
doc = edition.entries[hit.doc]
|
|
488
|
+
ref = f"{self.spec.name}:{hit.doc}@{edition.commit[:12]}§{hit.section}"
|
|
489
|
+
scored.append((hit.score, f"{ref}: {doc.title} / {hit.title} -- {hit.snippet}"))
|
|
490
|
+
scored.sort(key=lambda row: row[0])
|
|
491
|
+
window, nxt = cursor.window([line for _, line in scored], start, limit)
|
|
492
|
+
if not window:
|
|
493
|
+
return render("no sections matched", {})
|
|
494
|
+
footer: dict[str, str] = {"next": nxt}
|
|
495
|
+
if relaxed:
|
|
496
|
+
footer["relaxed"] = "no section had every word; these have some of them"
|
|
497
|
+
return render("\n".join(window), footer)
|
|
498
|
+
|
|
499
|
+
def status(self) -> str:
|
|
500
|
+
"""What is pinned, what is imported, and the licence."""
|
|
501
|
+
lic = self.spec.licence
|
|
502
|
+
lines = [
|
|
503
|
+
f"{self.spec.title}",
|
|
504
|
+
f"licence: {lic.spdx} ({lic.redistribute}){'; ' + lic.attribution if lic.attribution else ''}",
|
|
505
|
+
]
|
|
506
|
+
for tree in self.spec.trees:
|
|
507
|
+
have = {e.commit: e for e in store.installed(self.base(tree.name))}
|
|
508
|
+
lines.append(f"tree {tree.name}: {tree.title}")
|
|
509
|
+
for pin in tree.pins:
|
|
510
|
+
got = have.get(pin.commit)
|
|
511
|
+
state = (
|
|
512
|
+
f"imported {got.record.get('imported', '')} ({got.record.get('docs', '?')} documents)"
|
|
513
|
+
if got
|
|
514
|
+
else "not imported"
|
|
515
|
+
)
|
|
516
|
+
lines.append(f" {pin.label} {pin.commit[:12]} {pin.date}: {state}")
|
|
517
|
+
others = [c[:12] for c in have if c not in {p.commit for p in tree.pins}]
|
|
518
|
+
if others:
|
|
519
|
+
lines.append(f" other imported commits: {', '.join(others)}")
|
|
520
|
+
return "\n".join(lines) + "\n"
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def pin_files(*names: str) -> Callable[[str], bool]:
|
|
524
|
+
"""An `include` that accepts exactly these paths."""
|
|
525
|
+
wanted = frozenset(names)
|
|
526
|
+
return lambda path: path in wanted
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def matching(pattern: str) -> Callable[[str], bool]:
|
|
530
|
+
"""An `include` that accepts paths matching the regular expression `pattern` (fully)."""
|
|
531
|
+
compiled = re.compile(pattern)
|
|
532
|
+
return lambda path: compiled.fullmatch(path) is not None
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def parse_docs(
|
|
536
|
+
files: Mapping[str, bytes], reader: Callable[[bytes, str], Doc], ids: Sequence[tuple[str, str]]
|
|
537
|
+
) -> list[Doc]:
|
|
538
|
+
"""Read `(path, doc id)` pairs with `reader`, recording the source path in each document's meta."""
|
|
539
|
+
return [reader(files[path], doc_id).with_meta(path=path) for path, doc_id in ids if path in files]
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def make_collection(spec: Spec, package: str) -> Collection:
|
|
543
|
+
"""The registry entry for `spec`, whose `server` and `cli` targets live in `package` (`server.py`, `cli.py`)."""
|
|
544
|
+
return Collection(
|
|
545
|
+
name=spec.name,
|
|
546
|
+
title=spec.title,
|
|
547
|
+
prefix=spec.prefix,
|
|
548
|
+
unit=Unit.BLOCK,
|
|
549
|
+
grammar=spec.grammar,
|
|
550
|
+
server=f"{package}.server:server",
|
|
551
|
+
cli=f"{package}.cli:main",
|
|
552
|
+
)
|