moca-reader 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- moca_reader/__init__.py +84 -0
- moca_reader/_js.py +291 -0
- moca_reader/attestations.py +271 -0
- moca_reader/audience.py +49 -0
- moca_reader/backends/__init__.py +1 -0
- moca_reader/backends/bm25.py +118 -0
- moca_reader/backends/dense.py +146 -0
- moca_reader/backends/hybrid.py +73 -0
- moca_reader/backends/lexical.py +112 -0
- moca_reader/backends/memory_store.py +122 -0
- moca_reader/citation.py +213 -0
- moca_reader/codes.py +110 -0
- moca_reader/conformance.py +249 -0
- moca_reader/content.py +312 -0
- moca_reader/diagnostics.py +45 -0
- moca_reader/digest.py +92 -0
- moca_reader/dsse.py +97 -0
- moca_reader/ed25519.py +83 -0
- moca_reader/embedders.py +224 -0
- moca_reader/evidence_text.py +187 -0
- moca_reader/frontmatter.py +151 -0
- moca_reader/library.py +176 -0
- moca_reader/manifest.py +186 -0
- moca_reader/metadata.py +58 -0
- moca_reader/profiles.py +5 -0
- moca_reader/py.typed +0 -0
- moca_reader/reader.py +468 -0
- moca_reader/relations.py +46 -0
- moca_reader/resolver.py +33 -0
- moca_reader/reviews.py +18 -0
- moca_reader/schemas/ag-ui-citations.schema.json +36 -0
- moca_reader/schemas/citation-record.schema.json +326 -0
- moca_reader/schemas/manifest.schema.json +171 -0
- moca_reader/schemas/node.schema.json +372 -0
- moca_reader/schemas/review-predicate.schema.json +42 -0
- moca_reader/schemas/search-hit.schema.json +62 -0
- moca_reader/schemas/sidecar-index.schema.json +155 -0
- moca_reader/schemas/structure.schema.json +93 -0
- moca_reader/schemas/trust-root.schema.json +130 -0
- moca_reader/schemas.py +44 -0
- moca_reader/search.py +346 -0
- moca_reader/sidecar.py +228 -0
- moca_reader/source.py +285 -0
- moca_reader/state.py +119 -0
- moca_reader/strategies/__init__.py +1 -0
- moca_reader/strategies/diversify.py +28 -0
- moca_reader/strategies/ontology_guided.py +63 -0
- moca_reader/structure.py +385 -0
- moca_reader/structure_ops.py +160 -0
- moca_reader/unicode.py +33 -0
- moca_reader-0.6.0.dist-info/METADATA +209 -0
- moca_reader-0.6.0.dist-info/RECORD +55 -0
- moca_reader-0.6.0.dist-info/WHEEL +4 -0
- moca_reader-0.6.0.dist-info/licenses/LICENSE +201 -0
- moca_reader-0.6.0.dist-info/licenses/NOTICE +15 -0
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://w3id.org/moca/schemas/v1/structure.schema.json",
|
|
4
|
+
"title": "MOCA structure view",
|
|
5
|
+
"description": "The sorted JSON view of structure.ttl that moca-lint structure prints and Readers expose (structureView). It is not a package file: Readers read structure from structure.ttl only (docs/adr/0019-park-structure-json.md). See spec/moca-package-spec.md section 5.6.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": [
|
|
8
|
+
"structureVersion",
|
|
9
|
+
"concepts",
|
|
10
|
+
"sequences"
|
|
11
|
+
],
|
|
12
|
+
"properties": {
|
|
13
|
+
"structureVersion": {
|
|
14
|
+
"const": 1
|
|
15
|
+
},
|
|
16
|
+
"concepts": {
|
|
17
|
+
"type": "array",
|
|
18
|
+
"items": {
|
|
19
|
+
"type": "object",
|
|
20
|
+
"required": [
|
|
21
|
+
"iri",
|
|
22
|
+
"broader",
|
|
23
|
+
"related",
|
|
24
|
+
"parts",
|
|
25
|
+
"requires",
|
|
26
|
+
"replaces"
|
|
27
|
+
],
|
|
28
|
+
"properties": {
|
|
29
|
+
"iri": {
|
|
30
|
+
"$ref": "#/$defs/iri"
|
|
31
|
+
},
|
|
32
|
+
"label": {
|
|
33
|
+
"type": "string"
|
|
34
|
+
},
|
|
35
|
+
"definition": {
|
|
36
|
+
"type": "string"
|
|
37
|
+
},
|
|
38
|
+
"deprecated": {
|
|
39
|
+
"const": true
|
|
40
|
+
},
|
|
41
|
+
"broader": {
|
|
42
|
+
"$ref": "#/$defs/iris"
|
|
43
|
+
},
|
|
44
|
+
"related": {
|
|
45
|
+
"$ref": "#/$defs/iris"
|
|
46
|
+
},
|
|
47
|
+
"parts": {
|
|
48
|
+
"$ref": "#/$defs/iris"
|
|
49
|
+
},
|
|
50
|
+
"requires": {
|
|
51
|
+
"$ref": "#/$defs/iris"
|
|
52
|
+
},
|
|
53
|
+
"replaces": {
|
|
54
|
+
"$ref": "#/$defs/iris"
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
"additionalProperties": false
|
|
58
|
+
}
|
|
59
|
+
},
|
|
60
|
+
"sequences": {
|
|
61
|
+
"type": "array",
|
|
62
|
+
"items": {
|
|
63
|
+
"type": "object",
|
|
64
|
+
"required": [
|
|
65
|
+
"iri",
|
|
66
|
+
"members"
|
|
67
|
+
],
|
|
68
|
+
"properties": {
|
|
69
|
+
"iri": {
|
|
70
|
+
"$ref": "#/$defs/iri"
|
|
71
|
+
},
|
|
72
|
+
"members": {
|
|
73
|
+
"$ref": "#/$defs/iris"
|
|
74
|
+
}
|
|
75
|
+
},
|
|
76
|
+
"additionalProperties": false
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
},
|
|
80
|
+
"additionalProperties": false,
|
|
81
|
+
"$defs": {
|
|
82
|
+
"iri": {
|
|
83
|
+
"type": "string",
|
|
84
|
+
"pattern": "^[A-Za-z][A-Za-z0-9+.-]*:[^\\t\\n\\v\\f\\r \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+(?![\\s\\S])"
|
|
85
|
+
},
|
|
86
|
+
"iris": {
|
|
87
|
+
"type": "array",
|
|
88
|
+
"items": {
|
|
89
|
+
"$ref": "#/$defs/iri"
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://w3id.org/moca/schemas/v1/trust-root.schema.json",
|
|
4
|
+
"title": "MOCA trust root",
|
|
5
|
+
"description": "Host-supplied: which keys and Sigstore identities a host trusts, and for what. See spec/moca-attestations.md section 6.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"properties": {
|
|
8
|
+
"keys": {
|
|
9
|
+
"type": "array",
|
|
10
|
+
"items": {
|
|
11
|
+
"type": "object",
|
|
12
|
+
"required": [
|
|
13
|
+
"keyid",
|
|
14
|
+
"publicKey",
|
|
15
|
+
"roles"
|
|
16
|
+
],
|
|
17
|
+
"properties": {
|
|
18
|
+
"keyid": {
|
|
19
|
+
"type": "string",
|
|
20
|
+
"minLength": 1,
|
|
21
|
+
"description": "Unique within the trust root."
|
|
22
|
+
},
|
|
23
|
+
"publicKey": {
|
|
24
|
+
"type": "string",
|
|
25
|
+
"minLength": 1
|
|
26
|
+
},
|
|
27
|
+
"identity": {
|
|
28
|
+
"type": "string"
|
|
29
|
+
},
|
|
30
|
+
"expires": {
|
|
31
|
+
"type": "string",
|
|
32
|
+
"format": "date-time"
|
|
33
|
+
},
|
|
34
|
+
"roles": {
|
|
35
|
+
"type": "array",
|
|
36
|
+
"items": {
|
|
37
|
+
"enum": [
|
|
38
|
+
"package",
|
|
39
|
+
"review"
|
|
40
|
+
]
|
|
41
|
+
},
|
|
42
|
+
"uniqueItems": true,
|
|
43
|
+
"description": "What the key may attest. Required; an empty list trusts the key for nothing."
|
|
44
|
+
},
|
|
45
|
+
"ids": {
|
|
46
|
+
"type": "array",
|
|
47
|
+
"items": {
|
|
48
|
+
"type": "string",
|
|
49
|
+
"minLength": 1
|
|
50
|
+
},
|
|
51
|
+
"uniqueItems": true,
|
|
52
|
+
"description": "Package ids this entry may speak for in supersedes and conflictsWith relations: exact ids, or prefixes ending in /, : or # (package spec §7.2)."
|
|
53
|
+
},
|
|
54
|
+
"reviewers": {
|
|
55
|
+
"type": "array",
|
|
56
|
+
"items": {
|
|
57
|
+
"type": "string",
|
|
58
|
+
"minLength": 1
|
|
59
|
+
},
|
|
60
|
+
"uniqueItems": true,
|
|
61
|
+
"description": "The reviewers (OKF actors) this entry may sign reviews as. Without it, any (attestations §4)."
|
|
62
|
+
}
|
|
63
|
+
},
|
|
64
|
+
"additionalProperties": false
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
"sigstore": {
|
|
68
|
+
"type": "object",
|
|
69
|
+
"properties": {
|
|
70
|
+
"trustedRoot": {
|
|
71
|
+
"type": "object",
|
|
72
|
+
"description": "A Sigstore trusted root (trusted_root.json from sigstore/protobuf-specs): the Fulcio, Rekor, CT log and timestamp keys to verify against, offline. Attestations section 6."
|
|
73
|
+
},
|
|
74
|
+
"identities": {
|
|
75
|
+
"type": "array",
|
|
76
|
+
"items": {
|
|
77
|
+
"type": "object",
|
|
78
|
+
"required": [
|
|
79
|
+
"issuer",
|
|
80
|
+
"subject",
|
|
81
|
+
"roles"
|
|
82
|
+
],
|
|
83
|
+
"properties": {
|
|
84
|
+
"issuer": {
|
|
85
|
+
"type": "string",
|
|
86
|
+
"description": "Matched exactly against the certificate's OIDC issuer."
|
|
87
|
+
},
|
|
88
|
+
"subject": {
|
|
89
|
+
"type": "string",
|
|
90
|
+
"description": "Matched exactly against the certificate's subject alternative name (email or URI)."
|
|
91
|
+
},
|
|
92
|
+
"roles": {
|
|
93
|
+
"type": "array",
|
|
94
|
+
"items": {
|
|
95
|
+
"enum": [
|
|
96
|
+
"package",
|
|
97
|
+
"review"
|
|
98
|
+
]
|
|
99
|
+
},
|
|
100
|
+
"uniqueItems": true,
|
|
101
|
+
"description": "What the identity may attest. Required; an empty list trusts it for nothing."
|
|
102
|
+
},
|
|
103
|
+
"ids": {
|
|
104
|
+
"type": "array",
|
|
105
|
+
"items": {
|
|
106
|
+
"type": "string",
|
|
107
|
+
"minLength": 1
|
|
108
|
+
},
|
|
109
|
+
"uniqueItems": true,
|
|
110
|
+
"description": "Package ids this entry may speak for in supersedes and conflictsWith relations: exact ids, or prefixes ending in /, : or # (package spec §7.2)."
|
|
111
|
+
},
|
|
112
|
+
"reviewers": {
|
|
113
|
+
"type": "array",
|
|
114
|
+
"items": {
|
|
115
|
+
"type": "string",
|
|
116
|
+
"minLength": 1
|
|
117
|
+
},
|
|
118
|
+
"uniqueItems": true,
|
|
119
|
+
"description": "The reviewers (OKF actors) this entry may sign reviews as. Without it, any (attestations §4)."
|
|
120
|
+
}
|
|
121
|
+
},
|
|
122
|
+
"additionalProperties": false
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
},
|
|
126
|
+
"additionalProperties": false
|
|
127
|
+
}
|
|
128
|
+
},
|
|
129
|
+
"additionalProperties": false
|
|
130
|
+
}
|
moca_reader/schemas.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Vendored copies of moca-spec's schemas/v1. tests/test_schemas.py keeps them byte-identical to the spec's."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from functools import cache
|
|
7
|
+
from importlib.resources import files
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from jsonschema import Draft202012Validator, FormatChecker
|
|
11
|
+
|
|
12
|
+
_FILES = {
|
|
13
|
+
"manifest": "manifest",
|
|
14
|
+
"node": "node",
|
|
15
|
+
"citationRecord": "citation-record",
|
|
16
|
+
"sidecarIndex": "sidecar-index",
|
|
17
|
+
"reviewPredicate": "review-predicate",
|
|
18
|
+
"trustRoot": "trust-root",
|
|
19
|
+
"searchHit": "search-hit",
|
|
20
|
+
"structure": "structure",
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@cache
|
|
25
|
+
def schema(name: str) -> dict[str, Any]:
|
|
26
|
+
text = files("moca_reader").joinpath("schemas", f"{_FILES[name]}.schema.json").read_text("utf-8")
|
|
27
|
+
loaded: dict[str, Any] = json.loads(text)
|
|
28
|
+
return loaded
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@cache
|
|
32
|
+
def _validator(name: str) -> Draft202012Validator:
|
|
33
|
+
return Draft202012Validator(schema(name), format_checker=FormatChecker(["date", "date-time"]))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def validate_against(name: str, value: Any) -> list[str]:
|
|
37
|
+
"""Human-readable errors; empty when valid."""
|
|
38
|
+
out: list[str] = []
|
|
39
|
+
for err in _validator(name).iter_errors(value):
|
|
40
|
+
where = "/" + "/".join(str(p) for p in err.absolute_path) if err.absolute_path else "(root)"
|
|
41
|
+
msg = f"{where} {err.message}"
|
|
42
|
+
if msg not in out:
|
|
43
|
+
out.append(msg)
|
|
44
|
+
return out
|
moca_reader/search.py
ADDED
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
"""The Reader's search (Reader contract §9 and §10).
|
|
2
|
+
|
|
3
|
+
Pipeline: ingress hook -> backend -> resolve hits into citation records -> egress hook -> the gate. Backends and
|
|
4
|
+
hooks only propose candidates; the gate (digest allowlist, default retrieval policy, audience, locale, concepts and
|
|
5
|
+
scope) always runs last, so nothing either of them does can return content the policy excludes.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Callable, Sequence
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from datetime import UTC, datetime
|
|
13
|
+
from types import MappingProxyType
|
|
14
|
+
from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
|
|
15
|
+
|
|
16
|
+
from ._js import js_string, timestamp, truthy
|
|
17
|
+
from .audience import record_allowed
|
|
18
|
+
from .citation import CitationRecord, RelationIndex, citation_for, passes_default_policy, pick_representation
|
|
19
|
+
from .digest import utf8
|
|
20
|
+
from .state import refresh_record, state_index
|
|
21
|
+
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
from .library import Library
|
|
24
|
+
from .reader import PackageResult
|
|
25
|
+
from .structure_ops import StructureIndex
|
|
26
|
+
|
|
27
|
+
MAX_FETCH = 200
|
|
28
|
+
SearchHit = dict[str, Any] # search-hit.schema.json, plus "record" for store backends
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@runtime_checkable
|
|
32
|
+
class SearchBackend(Protocol):
|
|
33
|
+
"""A search backend the host chooses (Reader contract §9.1). A package never chooses one."""
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def features(self) -> Sequence[str]:
|
|
37
|
+
""" "lexical" or "dense", plus "filterPushdown"."""
|
|
38
|
+
...
|
|
39
|
+
|
|
40
|
+
def search(self, query: str, limit: int, filters: dict[str, Any]) -> list[SearchHit]: ...
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@runtime_checkable
|
|
44
|
+
class Embedder(Protocol):
|
|
45
|
+
"""A host-supplied embedder (Reader interface §5.3). A Reader never chooses one, and never adds text to what it embeds.
|
|
46
|
+
|
|
47
|
+
``query_prefix`` and ``document_prefix`` are declared for comparison with an index; the embedder applies its
|
|
48
|
+
model's convention for each ``role`` ("query" or "document") itself.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
name: str
|
|
52
|
+
version: str | None
|
|
53
|
+
dimensions: int
|
|
54
|
+
query_prefix: str | None
|
|
55
|
+
document_prefix: str | None
|
|
56
|
+
|
|
57
|
+
def embed(self, texts: list[str], role: str) -> list[list[float]]: ...
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True)
|
|
61
|
+
class HookContext:
|
|
62
|
+
"""Read-only context for retrieval hooks (Reader interface §5.4)."""
|
|
63
|
+
|
|
64
|
+
request: MappingProxyType[str, Any]
|
|
65
|
+
library: Library | None
|
|
66
|
+
structure: StructureIndex | None
|
|
67
|
+
audiences: list[str] | None
|
|
68
|
+
now: datetime
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
Ingress = Callable[[str, HookContext], Any]
|
|
72
|
+
Egress = Callable[[list[CitationRecord], HookContext], list[CitationRecord]]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass(frozen=True)
|
|
76
|
+
class SearchHooks:
|
|
77
|
+
"""Retrieval hooks: host code, never package content (moca-spec ADR-0010)."""
|
|
78
|
+
|
|
79
|
+
ingress: Ingress | None = None
|
|
80
|
+
egress: Egress | None = None
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass(frozen=True)
|
|
84
|
+
class StoreScope:
|
|
85
|
+
"""For a store backend: the states of the packages the host has loaded (``package_state``), and a clock.
|
|
86
|
+
|
|
87
|
+
Contract §9.4: every library- and trust-dependent field of a stored record is recomputed from these states.
|
|
88
|
+
"""
|
|
89
|
+
|
|
90
|
+
packages: Sequence[dict[str, Any]]
|
|
91
|
+
clock: Callable[[], datetime] = field(default=lambda: datetime.now(UTC))
|
|
92
|
+
allow_unsigned_relations: bool = False
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@dataclass
|
|
96
|
+
class _Context:
|
|
97
|
+
now: datetime
|
|
98
|
+
index: RelationIndex | None
|
|
99
|
+
request: dict[str, Any]
|
|
100
|
+
caller: MappingProxyType[str, Any]
|
|
101
|
+
loaded: dict[str, PackageResult]
|
|
102
|
+
hook_context: HookContext
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class Search:
|
|
106
|
+
"""One entry point over a backend: ``search(query, limit=, include_all=, locale=, concepts=, scope=)``."""
|
|
107
|
+
|
|
108
|
+
def __init__(
|
|
109
|
+
self,
|
|
110
|
+
source: Library | StoreScope,
|
|
111
|
+
backend: SearchBackend,
|
|
112
|
+
audiences: Sequence[str] | None = None,
|
|
113
|
+
overfetch: int = 3,
|
|
114
|
+
hooks: SearchHooks | None = None,
|
|
115
|
+
allow_unlabelled: bool = False,
|
|
116
|
+
) -> None:
|
|
117
|
+
from .library import Library
|
|
118
|
+
|
|
119
|
+
if backend is None or not callable(getattr(backend, "search", None)):
|
|
120
|
+
raise TypeError("Search needs a backend")
|
|
121
|
+
self.library: Library | None = source if isinstance(source, Library) else None
|
|
122
|
+
self.states: dict[str, dict[str, Any]] | None = (
|
|
123
|
+
None if isinstance(source, Library) else {s["digest"]: s for s in source.packages}
|
|
124
|
+
)
|
|
125
|
+
self.allowed: set[str] | None = None if self.states is None else set(self.states)
|
|
126
|
+
self.store_index = (
|
|
127
|
+
None
|
|
128
|
+
if self.states is None or isinstance(source, Library)
|
|
129
|
+
else state_index(self.states, source.allow_unsigned_relations)
|
|
130
|
+
)
|
|
131
|
+
self.clock: Callable[[], datetime] = source.clock
|
|
132
|
+
self.backend = backend
|
|
133
|
+
self.audiences = list(audiences) if audiences is not None else None
|
|
134
|
+
# Contract §8 (moca-spec ADR-0016): unlabelled content is outside an audience set unless the host allows it.
|
|
135
|
+
self.allow_unlabelled = allow_unlabelled is True
|
|
136
|
+
self.overfetch = overfetch
|
|
137
|
+
self.hooks = hooks or SearchHooks()
|
|
138
|
+
|
|
139
|
+
def search(
|
|
140
|
+
self,
|
|
141
|
+
query: str,
|
|
142
|
+
*,
|
|
143
|
+
limit: int = 5,
|
|
144
|
+
include_all: bool = False,
|
|
145
|
+
locale: str | None = None,
|
|
146
|
+
concepts: Sequence[str] | None = None,
|
|
147
|
+
scope: str | None = None,
|
|
148
|
+
) -> list[CitationRecord]:
|
|
149
|
+
ctx = self._context(limit, include_all, locale, concepts, scope)
|
|
150
|
+
if self.hooks.ingress is not None:
|
|
151
|
+
_apply_ingress(ctx, self.hooks.ingress(query, ctx.hook_context))
|
|
152
|
+
limit_, filters = self._plan(ctx)
|
|
153
|
+
effective = ctx.request.get("query")
|
|
154
|
+
hits = self.backend.search(effective if effective is not None else query, limit_, filters)
|
|
155
|
+
candidates = self._resolve(hits, ctx)
|
|
156
|
+
if self.hooks.egress is not None:
|
|
157
|
+
candidates = self.hooks.egress(
|
|
158
|
+
[{**r, "retrieval": {"via": "search"}} for r in candidates], ctx.hook_context
|
|
159
|
+
)
|
|
160
|
+
return self._gate(candidates, ctx)
|
|
161
|
+
|
|
162
|
+
def _context(
|
|
163
|
+
self, limit: int, include_all: bool, locale: str | None, concepts: Sequence[str] | None, scope: str | None
|
|
164
|
+
) -> _Context:
|
|
165
|
+
now = self.clock()
|
|
166
|
+
index = self.library.relation_index() if self.library else None
|
|
167
|
+
# The caller's request binds the gate. Ingress may change the query and what the backend searches
|
|
168
|
+
# (contract §9.2), never the limit, the opt-in, the locale or what the gate keeps.
|
|
169
|
+
caller = MappingProxyType(
|
|
170
|
+
{
|
|
171
|
+
"limit": limit,
|
|
172
|
+
"includeAll": include_all,
|
|
173
|
+
"locale": locale,
|
|
174
|
+
"concepts": tuple(concepts) if concepts else None,
|
|
175
|
+
"scope": scope,
|
|
176
|
+
}
|
|
177
|
+
)
|
|
178
|
+
request = dict(caller)
|
|
179
|
+
loaded = {p.result.digest: p.result for p in self.library.packages} if self.library else {}
|
|
180
|
+
hook_context = HookContext(
|
|
181
|
+
MappingProxyType(request),
|
|
182
|
+
self.library,
|
|
183
|
+
self.library.structure if self.library else None,
|
|
184
|
+
self.audiences,
|
|
185
|
+
now,
|
|
186
|
+
)
|
|
187
|
+
return _Context(now, index, request, caller, loaded, hook_context) # type: ignore[arg-type]
|
|
188
|
+
|
|
189
|
+
def _plan(self, ctx: _Context) -> tuple[int, dict[str, Any]]:
|
|
190
|
+
request = ctx.request
|
|
191
|
+
exclude: list[dict[str, str]] = []
|
|
192
|
+
if not request["includeAll"] and self.library:
|
|
193
|
+
for p in self.library.packages:
|
|
194
|
+
for node in p.result.nodes:
|
|
195
|
+
if all(
|
|
196
|
+
not passes_default_policy(citation_for(p.result, node, rep, ctx.now, ctx.index))
|
|
197
|
+
for rep in node.representations
|
|
198
|
+
):
|
|
199
|
+
exclude.append(
|
|
200
|
+
{
|
|
201
|
+
"digest": p.result.digest or "",
|
|
202
|
+
"node": f"{js_string(p.result.manifest.get('id'))}#{node.path}",
|
|
203
|
+
}
|
|
204
|
+
)
|
|
205
|
+
concepts = self._concepts(request)
|
|
206
|
+
filters: dict[str, Any] = {
|
|
207
|
+
"digests": [p.result.digest for p in self.library.packages] if self.library else list(self.allowed or []),
|
|
208
|
+
"exclude": exclude,
|
|
209
|
+
}
|
|
210
|
+
if self.audiences is not None:
|
|
211
|
+
filters["audiences"] = self.audiences
|
|
212
|
+
filters["allowUnlabelled"] = self.allow_unlabelled
|
|
213
|
+
if request.get("locale"):
|
|
214
|
+
filters["locale"] = request["locale"]
|
|
215
|
+
if concepts:
|
|
216
|
+
filters["concepts"] = concepts
|
|
217
|
+
# Fetch more than the limit when the gate may drop hits (no pushdown), or when an egress hook reorders
|
|
218
|
+
# them and needs candidates to choose from.
|
|
219
|
+
pushdown = "filterPushdown" in (getattr(self.backend, "features", None) or [])
|
|
220
|
+
fetch = (
|
|
221
|
+
request["limit"]
|
|
222
|
+
if pushdown and self.hooks.egress is None
|
|
223
|
+
else min(request["limit"] * self.overfetch, MAX_FETCH)
|
|
224
|
+
)
|
|
225
|
+
return fetch, filters
|
|
226
|
+
|
|
227
|
+
def _concepts(self, request: Any) -> list[str] | None:
|
|
228
|
+
"""The concepts a record must be bound to: the caller's, plus a scope and its descendants."""
|
|
229
|
+
found = dict.fromkeys(request.get("concepts") or [])
|
|
230
|
+
if request.get("scope"):
|
|
231
|
+
found.setdefault(request["scope"])
|
|
232
|
+
for c in self.library.structure.descendants(request["scope"]) if self.library else []:
|
|
233
|
+
found.setdefault(c)
|
|
234
|
+
return list(found) if found else None
|
|
235
|
+
|
|
236
|
+
def _resolve(self, hits: Any, ctx: _Context) -> list[CitationRecord]:
|
|
237
|
+
"""Backend hits as citation records; hits that do not resolve are dropped."""
|
|
238
|
+
out: list[CitationRecord] = []
|
|
239
|
+
for hit in hits if isinstance(hits, list) else []:
|
|
240
|
+
if isinstance(hit, dict) and truthy(hit.get("record")):
|
|
241
|
+
refreshed = (
|
|
242
|
+
_refresh_clock(hit["record"], ctx.now)
|
|
243
|
+
if self.library
|
|
244
|
+
else refresh_record(hit["record"], ctx.now, self.states or {}, self.store_index or ({}, {}))
|
|
245
|
+
)
|
|
246
|
+
if refreshed is not None:
|
|
247
|
+
out.append({**refreshed, "score": hit.get("score")})
|
|
248
|
+
continue
|
|
249
|
+
result = ctx.loaded.get(str(hit.get("digest"))) if isinstance(hit, dict) else None
|
|
250
|
+
if result is None or hit.get("node") != f"{js_string(result.manifest.get('id'))}#{hit.get('path')}":
|
|
251
|
+
continue
|
|
252
|
+
node = next((n for n in result.nodes if n.path == hit.get("path")), None)
|
|
253
|
+
rep = (
|
|
254
|
+
next((r for r in node.representations if (r.locale or None) == (hit.get("locale") or None)), None)
|
|
255
|
+
if node
|
|
256
|
+
else None
|
|
257
|
+
)
|
|
258
|
+
if node is None or rep is None:
|
|
259
|
+
continue
|
|
260
|
+
span = hit.get("span")
|
|
261
|
+
text = _slice_body(rep, span) if span else rep.body
|
|
262
|
+
if text is None:
|
|
263
|
+
continue
|
|
264
|
+
record = citation_for(result, node, rep, ctx.now, ctx.index, text)
|
|
265
|
+
if span:
|
|
266
|
+
record["span"] = {"start": span["start"], "end": span["end"]}
|
|
267
|
+
out.append({**record, "score": hit.get("score")})
|
|
268
|
+
return out
|
|
269
|
+
|
|
270
|
+
def _gate(self, candidates: Any, ctx: _Context) -> list[CitationRecord]:
|
|
271
|
+
"""The gate: always last, whatever the backend and the hooks did."""
|
|
272
|
+
request = ctx.caller
|
|
273
|
+
concepts = self._concepts(request)
|
|
274
|
+
allowed = set(ctx.loaded) if self.library else (self.allowed or set())
|
|
275
|
+
seen: set[str] = set()
|
|
276
|
+
out: list[CitationRecord] = []
|
|
277
|
+
for record in candidates if isinstance(candidates, list) else []:
|
|
278
|
+
if not isinstance(record, dict) or not all(truthy(record.get(k)) for k in ("package", "node", "trust")):
|
|
279
|
+
continue
|
|
280
|
+
if record["package"].get("digest") not in allowed:
|
|
281
|
+
continue
|
|
282
|
+
if not request["includeAll"] and not passes_default_policy(record):
|
|
283
|
+
continue
|
|
284
|
+
if not record_allowed(record, self.audiences, self.allow_unlabelled):
|
|
285
|
+
continue
|
|
286
|
+
if request["locale"] and not self._locale_ok(record, request["locale"], ctx.loaded):
|
|
287
|
+
continue
|
|
288
|
+
if concepts and not any(c in concepts for c in record.get("concepts") or []):
|
|
289
|
+
continue
|
|
290
|
+
span = record.get("span")
|
|
291
|
+
# Contract §9.2: one result per node representation and span.
|
|
292
|
+
locale = record["node"].get("locale") or ""
|
|
293
|
+
key = f"{record['node'].get('ref')}\0{locale}\0{span.get('start', '') if isinstance(span, dict) else ''}"
|
|
294
|
+
if key in seen:
|
|
295
|
+
continue
|
|
296
|
+
seen.add(key)
|
|
297
|
+
out.append(record)
|
|
298
|
+
if self.hooks.egress is None:
|
|
299
|
+
out.sort(key=lambda r: -(r.get("score") or 0))
|
|
300
|
+
return out[: request["limit"]]
|
|
301
|
+
|
|
302
|
+
def _locale_ok(self, record: CitationRecord, locale: str, loaded: dict[str, PackageResult]) -> bool:
|
|
303
|
+
result = loaded.get(record["package"].get("digest"))
|
|
304
|
+
node = next((n for n in result.nodes if n.path == record["node"].get("path")), None) if result else None
|
|
305
|
+
if node is None or result is None:
|
|
306
|
+
want = locale.lower()
|
|
307
|
+
have = (record["node"].get("locale") or "").lower()
|
|
308
|
+
return have == want or want.startswith(f"{have}-") or have.startswith(f"{want}-")
|
|
309
|
+
rep = pick_representation(node, locale)
|
|
310
|
+
return bool((rep.locale or result.manifest.get("language")) == record["node"].get("locale"))
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _apply_ingress(ctx: _Context, result: Any) -> None:
|
|
314
|
+
"""What an ingress hook may change: the query, and the concepts or scope the backend searches."""
|
|
315
|
+
if not isinstance(result, dict):
|
|
316
|
+
return
|
|
317
|
+
if isinstance(result.get("query"), str):
|
|
318
|
+
ctx.request["query"] = result["query"]
|
|
319
|
+
if isinstance(result.get("concepts"), list):
|
|
320
|
+
ctx.request["concepts"] = list(result["concepts"])
|
|
321
|
+
if isinstance(result.get("scope"), str):
|
|
322
|
+
ctx.request["scope"] = result["scope"]
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _refresh_clock(record: CitationRecord, now: datetime) -> CitationRecord:
|
|
326
|
+
"""A stored citation record with its clock-dependent fields recomputed for now."""
|
|
327
|
+
t = now.timestamp() * 1000
|
|
328
|
+
trust = dict(record["trust"])
|
|
329
|
+
trust["stale"] = t >= timestamp(trust["staleAfter"]) if truthy(trust.get("staleAfter")) else False
|
|
330
|
+
trust["inForce"] = (not truthy(trust.get("validFrom")) or t >= timestamp(trust["validFrom"])) and (
|
|
331
|
+
not truthy(trust.get("validUntil")) or t < timestamp(trust["validUntil"])
|
|
332
|
+
)
|
|
333
|
+
return {**record, "trust": trust}
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _slice_body(rep: Any, span: dict[str, Any]) -> str | None:
|
|
337
|
+
"""The text of a byte span inside a representation's body, or None."""
|
|
338
|
+
body = utf8(rep.body)
|
|
339
|
+
start: Any = span.get("start")
|
|
340
|
+
end: Any = span.get("end")
|
|
341
|
+
if not all(isinstance(v, (int, float)) and not isinstance(v, bool) and float(v).is_integer() for v in (start, end)):
|
|
342
|
+
return None
|
|
343
|
+
lo, hi = int(start) - rep.body_offset, int(end) - rep.body_offset
|
|
344
|
+
if lo < 0 or hi < lo or hi > len(body):
|
|
345
|
+
return None
|
|
346
|
+
return body[lo:hi].decode("utf-8", errors="replace")
|