pubba 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pubba/__init__.py +24 -0
- pubba/core.py +546 -0
- pubba/events.py +61 -0
- pubba/guide.py +223 -0
- pubba/py.typed +0 -0
- pubba/text.py +80 -0
- pubba/trail.py +78 -0
- pubba-0.4.0.dist-info/METADATA +82 -0
- pubba-0.4.0.dist-info/RECORD +11 -0
- pubba-0.4.0.dist-info/WHEEL +4 -0
- pubba-0.4.0.dist-info/licenses/LICENSE +21 -0
pubba/__init__.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""pubba: a small brain for showroom sites, in Python.
|
|
2
|
+
|
|
3
|
+
Load a bundle compiled by `pubba compile` and answer: related items with
|
|
4
|
+
reasons, what next from a visitor's trail, typo-tolerant matching, a headless
|
|
5
|
+
guide, and privacy-safe measurement events. Same answers as every other pubba
|
|
6
|
+
runtime: the conformance suite holds them to it.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from .core import BUNDLE_FORMAT, Entity, Graph, Link, Match, MatchResult, Pubba, Related, Suggestion, graph_from_bundle, load_bundle, reason_from_link, render_reason
|
|
10
|
+
from .events import GUIDE_VIAS, Events, create_events, summarise
|
|
11
|
+
from .guide import DEFAULT_GUIDE_TEMPLATES, EntityChip, Guide, NudgePolicy, QuestionChip, Reply, create_guide
|
|
12
|
+
from .text import close_to, contains_name, edit_distance, has_phrase, normalize
|
|
13
|
+
from .trail import Trail, create_trail, section_step
|
|
14
|
+
|
|
15
|
+
__version__ = "0.4.0"
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"BUNDLE_FORMAT", "Entity", "Graph", "Link", "Match", "MatchResult", "Pubba", "Related", "Suggestion",
|
|
19
|
+
"graph_from_bundle", "load_bundle", "reason_from_link", "render_reason",
|
|
20
|
+
"GUIDE_VIAS", "Events", "create_events", "summarise",
|
|
21
|
+
"DEFAULT_GUIDE_TEMPLATES", "EntityChip", "Guide", "NudgePolicy", "QuestionChip", "Reply", "create_guide",
|
|
22
|
+
"close_to", "contains_name", "edit_distance", "has_phrase", "normalize",
|
|
23
|
+
"Trail", "create_trail", "section_step", "__version__",
|
|
24
|
+
]
|
pubba/core.py
ADDED
|
@@ -0,0 +1,546 @@
|
|
|
1
|
+
"""The pubba runtime: load a compiled bundle and answer from it (SPEC sections 1 to 6).
|
|
2
|
+
|
|
3
|
+
A port of the TypeScript reference; the conformance suite holds them to the
|
|
4
|
+
same answers, scores to 1e-9.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import math
|
|
11
|
+
import re
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple, Union
|
|
14
|
+
|
|
15
|
+
from .events import GUIDE_VIAS, Events
|
|
16
|
+
from .text import contains_name, has_phrase, normalize
|
|
17
|
+
|
|
18
|
+
BUNDLE_FORMAT = 1
|
|
19
|
+
INF = float("inf")
|
|
20
|
+
DEFAULT_STOP_WORDS = ["the", "app", "and", "for", "web", "application", "mobile", "portal", "site", "website"]
|
|
21
|
+
DEFAULT_SIMILAR = r"\b(like|similar|alike|same stack|related to|anything else)\b"
|
|
22
|
+
DEFAULT_TEMPLATES: Dict[str, Any] = {
|
|
23
|
+
"section": "{item} is worth a look.",
|
|
24
|
+
"typed": "{item} matches what you typed.",
|
|
25
|
+
"match": "{item} matches “{name}”.",
|
|
26
|
+
}
|
|
27
|
+
STRONG = 50
|
|
28
|
+
|
|
29
|
+
Reason = Dict[str, Any]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class Entity:
|
|
34
|
+
id: str
|
|
35
|
+
type: str
|
|
36
|
+
label: str
|
|
37
|
+
short: Optional[str] = None
|
|
38
|
+
aliases: Tuple[str, ...] = ()
|
|
39
|
+
url: Optional[str] = None
|
|
40
|
+
text: Optional[str] = None
|
|
41
|
+
data: Optional[Dict[str, Any]] = None
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def short_label(self) -> str:
|
|
45
|
+
"""How it reads inside a sentence."""
|
|
46
|
+
return self.short if self.short is not None else self.label
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True)
|
|
50
|
+
class Link:
|
|
51
|
+
from_: str
|
|
52
|
+
kind: str
|
|
53
|
+
to: str
|
|
54
|
+
weight: float
|
|
55
|
+
via: Optional[Tuple[str, ...]] = None
|
|
56
|
+
rule: Optional[str] = None
|
|
57
|
+
|
|
58
|
+
def other_end(self, id: str) -> str:
|
|
59
|
+
return self.to if self.from_ == id else self.from_
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class Suggestion:
|
|
64
|
+
entity: Entity
|
|
65
|
+
score: float
|
|
66
|
+
reason: Reason
|
|
67
|
+
text: str
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass
|
|
71
|
+
class Related:
|
|
72
|
+
entity: Entity
|
|
73
|
+
weight: float
|
|
74
|
+
link: Link
|
|
75
|
+
reason: Reason
|
|
76
|
+
text: str
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass
|
|
80
|
+
class Match:
|
|
81
|
+
entity: Entity
|
|
82
|
+
score: float
|
|
83
|
+
tier: int
|
|
84
|
+
reason: Reason
|
|
85
|
+
text: str
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass
|
|
89
|
+
class MatchResult:
|
|
90
|
+
query: str
|
|
91
|
+
intent: str
|
|
92
|
+
entity: Optional[Entity] = None
|
|
93
|
+
match: Optional[Match] = None
|
|
94
|
+
similar: List[Related] = field(default_factory=list)
|
|
95
|
+
not_here: Optional[str] = None
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass
|
|
99
|
+
class Graph:
|
|
100
|
+
entities: Dict[str, Entity]
|
|
101
|
+
unlinked: Dict[str, Entity]
|
|
102
|
+
links: List[Link]
|
|
103
|
+
adjacent: Dict[str, List[Link]]
|
|
104
|
+
config: Dict[str, Any]
|
|
105
|
+
distinct_words: Set[str]
|
|
106
|
+
|
|
107
|
+
def all_entities(self) -> List[Entity]:
|
|
108
|
+
return list(self.entities.values()) + list(self.unlinked.values())
|
|
109
|
+
|
|
110
|
+
def find(self, id: str) -> Optional[Entity]:
|
|
111
|
+
return self.entities.get(id) or self.unlinked.get(id)
|
|
112
|
+
|
|
113
|
+
def degree(self, id: str) -> int:
|
|
114
|
+
return len(self.adjacent.get(id, []))
|
|
115
|
+
|
|
116
|
+
def type_config(self, type: str) -> Dict[str, Any]:
|
|
117
|
+
return (self.config.get("types") or {}).get(type) or {}
|
|
118
|
+
|
|
119
|
+
def is_destination(self, e: Entity) -> bool:
|
|
120
|
+
return self.type_config(e.type).get("destination", True) is not False
|
|
121
|
+
|
|
122
|
+
def pull(self, kind: str) -> float:
|
|
123
|
+
return ((self.config.get("kinds") or {}).get(kind) or {}).get("pull", 0.5)
|
|
124
|
+
|
|
125
|
+
def kind_label(self, kind: str) -> str:
|
|
126
|
+
label = ((self.config.get("kinds") or {}).get(kind) or {}).get("label")
|
|
127
|
+
return label if label is not None else kind.replace("-", " ")
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _entity(raw: Dict[str, Any]) -> Entity:
|
|
131
|
+
return Entity(
|
|
132
|
+
id=raw["id"],
|
|
133
|
+
type=raw["type"],
|
|
134
|
+
label=raw["label"],
|
|
135
|
+
short=raw.get("short"),
|
|
136
|
+
aliases=tuple(raw.get("aliases") or ()),
|
|
137
|
+
url=raw.get("url"),
|
|
138
|
+
text=raw.get("text"),
|
|
139
|
+
data=raw.get("data"),
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def graph_from_bundle(bundle: Union[str, bytes, Dict[str, Any]]) -> Graph:
|
|
144
|
+
"""The graph inside a bundle, after checking it is one this runtime can read."""
|
|
145
|
+
b = json.loads(bundle) if isinstance(bundle, (str, bytes)) else bundle
|
|
146
|
+
if not isinstance(b, dict) or not isinstance(b.get("format"), (int, float)) or not isinstance(b.get("entities"), list) or not isinstance(b.get("links"), list):
|
|
147
|
+
raise ValueError("pubba: not a pubba bundle")
|
|
148
|
+
if math.floor(b["format"]) > BUNDLE_FORMAT:
|
|
149
|
+
raise ValueError(f"pubba: bundle format {b['format']} is newer than this runtime reads ({BUNDLE_FORMAT}); upgrade pubba")
|
|
150
|
+
links = [Link(l["from"], l["kind"], l["to"], l["weight"], tuple(l["via"]) if l.get("via") is not None else None, l.get("rule")) for l in b["links"]]
|
|
151
|
+
adjacent: Dict[str, List[Link]] = {}
|
|
152
|
+
for l in links:
|
|
153
|
+
adjacent.setdefault(l.from_, []).append(l)
|
|
154
|
+
adjacent.setdefault(l.to, []).append(l)
|
|
155
|
+
linked: Dict[str, Entity] = {}
|
|
156
|
+
unlinked: Dict[str, Entity] = {}
|
|
157
|
+
for raw in list(b["entities"]) + list(b.get("unlinked") or []):
|
|
158
|
+
e = _entity(raw)
|
|
159
|
+
(linked if e.id in adjacent else unlinked)[e.id] = e
|
|
160
|
+
index = b.get("index") or {}
|
|
161
|
+
return Graph(linked, unlinked, links, adjacent, b.get("config") or {}, set(index.get("distinctWords") or []))
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
# Sentences (SPEC section 6).
|
|
165
|
+
|
|
166
|
+
_TOKEN = re.compile(r"\{(\w+)\}")
|
|
167
|
+
_OPTIONAL = re.compile(r"\[([^\]]*)\]")
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def reason_from_link(link: Link, item_id: str, seed: Optional[str] = None) -> Reason:
|
|
171
|
+
"""The reason a link gives for the entity at one of its ends."""
|
|
172
|
+
r: Reason = {"kind": link.kind, "to": link.other_end(item_id), "direction": "out" if link.from_ == item_id else "in"}
|
|
173
|
+
if link.via:
|
|
174
|
+
r["via"] = list(link.via)
|
|
175
|
+
if seed and seed != r["to"]:
|
|
176
|
+
r["seed"] = seed
|
|
177
|
+
return r
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _fill(text: str, values: Dict[str, str]) -> str:
|
|
181
|
+
def keep(m: "re.Match[str]") -> str:
|
|
182
|
+
inner = m.group(1)
|
|
183
|
+
return inner if all(values.get(t) for t in _TOKEN.findall(inner)) else ""
|
|
184
|
+
|
|
185
|
+
return _TOKEN.sub(lambda m: values.get(m.group(1)) or "", _OPTIONAL.sub(keep, text))
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _pick(t: Any, reason: Reason, item: Entity, other: Optional[Entity]) -> Optional[str]:
|
|
189
|
+
if isinstance(t, str):
|
|
190
|
+
return t
|
|
191
|
+
if callable(t):
|
|
192
|
+
return t(reason, item, other)
|
|
193
|
+
for c in t.get("cases") or []:
|
|
194
|
+
if (not c.get("item") or c["item"] == item.type) and (not c.get("other") or (other is not None and c["other"] == other.type)) and (not c.get("direction") or c["direction"] == reason.get("direction")):
|
|
195
|
+
return c["text"]
|
|
196
|
+
if reason.get("direction") == "out" and t.get("out"):
|
|
197
|
+
return t["out"]
|
|
198
|
+
if reason.get("direction") == "in" and t.get("in"):
|
|
199
|
+
return t["in"]
|
|
200
|
+
for k in ("default", "out", "in"):
|
|
201
|
+
if t.get(k) is not None:
|
|
202
|
+
return t[k]
|
|
203
|
+
return None
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def render_reason(g: Graph, reason: Reason, item: Entity, templates: Optional[Dict[str, Any]] = None) -> str:
|
|
207
|
+
"""A reason as one sentence."""
|
|
208
|
+
every = {**DEFAULT_TEMPLATES, **(g.config.get("templates") or {}), **(templates or {})}
|
|
209
|
+
other = g.entities.get(reason["to"]) if reason.get("to") else None
|
|
210
|
+
t = every.get(f"{reason['kind']}.{item.type}")
|
|
211
|
+
if t is None:
|
|
212
|
+
t = every.get(reason["kind"])
|
|
213
|
+
text = _pick(t, reason, item, other) if t else None
|
|
214
|
+
if text is None:
|
|
215
|
+
if other is not None:
|
|
216
|
+
text = "{other}: {kind} {item}[ ({via})]." if reason.get("direction") == "in" else "{item}: {kind} {other}[ ({via})]."
|
|
217
|
+
else:
|
|
218
|
+
text = "{item} is worth a look."
|
|
219
|
+
if callable(t):
|
|
220
|
+
return text
|
|
221
|
+
other_label = other.short_label if other is not None else ""
|
|
222
|
+
return _fill(
|
|
223
|
+
text,
|
|
224
|
+
{
|
|
225
|
+
"item": item.short_label,
|
|
226
|
+
"other": other_label,
|
|
227
|
+
"otherLower": other_label.lower(),
|
|
228
|
+
"via": ", ".join((reason.get("via") or [])[:3]),
|
|
229
|
+
"kind": g.kind_label(reason["kind"]),
|
|
230
|
+
"name": reason.get("name") or "",
|
|
231
|
+
},
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
# Matching (SPEC section 4).
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _significant(name: str, stop: Set[str]) -> List[str]:
|
|
239
|
+
return [w for w in name.split(" ") if len(w) > 2 and w not in stop]
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _coverage(q: str, name: str, typo: float) -> float:
|
|
243
|
+
name_tokens = name.split(" ")
|
|
244
|
+
query_tokens = q.split(" ")
|
|
245
|
+
in_query = sum(1 for w in name_tokens if has_phrase(q, w, typo)) / len(name_tokens)
|
|
246
|
+
explained = sum(1 for t in query_tokens if any(t == w or has_phrase(t, w, typo) for w in name_tokens)) / len(query_tokens)
|
|
247
|
+
return 0.5 * in_query + 0.4 * explained
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _score_name(q: str, name: str, is_alias: bool, m: Dict[str, Any], s: Dict[str, Any]) -> Optional[Tuple[float, str, str]]:
|
|
251
|
+
if not name or len(name) < m.get("minLength", 0):
|
|
252
|
+
return None
|
|
253
|
+
typo = s["typo"]
|
|
254
|
+
has = lambda w: has_phrase(q, w, typo) # noqa: E731
|
|
255
|
+
if m.get("mode", "title") == "phrase":
|
|
256
|
+
if len(name) < 2 or not has(name):
|
|
257
|
+
return None
|
|
258
|
+
exact = not s["precise"] or contains_name(q, name)
|
|
259
|
+
return ((100 if exact else 95) + len(name), "alias" if is_alias else "phrase", name)
|
|
260
|
+
if contains_name(q, name):
|
|
261
|
+
return (100 + len(name), "alias" if is_alias else "name", name)
|
|
262
|
+
if s["precise"] and " " in name and has(name):
|
|
263
|
+
return (95 + len(name), "alias" if is_alias else "name", name)
|
|
264
|
+
words = _significant(name, s["stop"])
|
|
265
|
+
hits = sum(1 for w in words if has(w))
|
|
266
|
+
if hits and hits == len(words) and len("".join(words)) > m.get("minWordsLength", 0):
|
|
267
|
+
return (50 + hits + (_coverage(q, name, typo) if s["precise"] else 0), "words", name)
|
|
268
|
+
if s["partial"] and len(words) >= 3 and hits >= 2 and hits == len(words) - 1:
|
|
269
|
+
return (40 + hits, "words", name)
|
|
270
|
+
if s["distinct"] is not None:
|
|
271
|
+
for w in words:
|
|
272
|
+
if len(w) >= 5 and w in s["distinct"] and has_phrase(q, w, INF):
|
|
273
|
+
return (25, "distinct-word", w)
|
|
274
|
+
first = words[0] if words else None
|
|
275
|
+
if m.get("firstWord") and first and len(first) > 3 and has(first):
|
|
276
|
+
return (20, "first-word", first)
|
|
277
|
+
return None
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
class Track:
|
|
281
|
+
"""The moments a site reports to its measurement, checked so no visitor text gets in."""
|
|
282
|
+
|
|
283
|
+
def __init__(self, graph: Graph, events: Optional[Events]) -> None:
|
|
284
|
+
self._graph = graph
|
|
285
|
+
self._events = events
|
|
286
|
+
|
|
287
|
+
def guide_opened(self) -> None:
|
|
288
|
+
if self._events:
|
|
289
|
+
self._events.emit({"type": "guide-opened"})
|
|
290
|
+
|
|
291
|
+
def guided_to(self, id: str, via: str) -> bool:
|
|
292
|
+
e = self._graph.find(id)
|
|
293
|
+
if not self._events or e is None or via not in GUIDE_VIAS:
|
|
294
|
+
return False
|
|
295
|
+
self._events.emit({"type": "guided-to", "to": e.id, "entityType": e.type, "via": via})
|
|
296
|
+
return True
|
|
297
|
+
|
|
298
|
+
def reached(self, goal: str) -> bool:
|
|
299
|
+
if not self._events or goal not in self._events.goals:
|
|
300
|
+
return False
|
|
301
|
+
self._events.emit({"type": "reached", "goal": goal})
|
|
302
|
+
return True
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
class Pubba:
|
|
306
|
+
"""Answers from one graph: related items, what next, matching, sentences, measurement."""
|
|
307
|
+
|
|
308
|
+
def __init__(self, graph: Graph, events: Optional[Events] = None) -> None:
|
|
309
|
+
self.graph = graph
|
|
310
|
+
self._events = events
|
|
311
|
+
self.track = Track(graph, events)
|
|
312
|
+
|
|
313
|
+
def get(self, id: str) -> Optional[Entity]:
|
|
314
|
+
return self.graph.entities.get(id)
|
|
315
|
+
|
|
316
|
+
def with_events(self, events: Optional[Events]) -> "Pubba":
|
|
317
|
+
"""The same graph, reporting to these events."""
|
|
318
|
+
return Pubba(self.graph, events)
|
|
319
|
+
|
|
320
|
+
def _suggested(self, source: str, out: Sequence[Union[Suggestion, Related]]) -> None:
|
|
321
|
+
if self._events and out:
|
|
322
|
+
kinds: List[str] = []
|
|
323
|
+
for x in out:
|
|
324
|
+
if x.reason["kind"] not in kinds:
|
|
325
|
+
kinds.append(x.reason["kind"])
|
|
326
|
+
self._events.emit({"type": "suggested", "source": source, "ids": [x.entity.id for x in out], "reasonKinds": kinds})
|
|
327
|
+
|
|
328
|
+
# --- matching
|
|
329
|
+
|
|
330
|
+
def search(self, query: str, limit: int = 10, types: Optional[Sequence[str]] = None, templates: Optional[Dict[str, Any]] = None) -> List[Match]:
|
|
331
|
+
"""Every entity a query could mean, best first."""
|
|
332
|
+
g = self.graph
|
|
333
|
+
q = normalize(query)
|
|
334
|
+
if not q:
|
|
335
|
+
return []
|
|
336
|
+
mc = g.config.get("match") or {}
|
|
337
|
+
stop = set(mc.get("stopWords", DEFAULT_STOP_WORDS))
|
|
338
|
+
settings = {
|
|
339
|
+
"stop": stop,
|
|
340
|
+
"typo": mc.get("typoMinLength", 4),
|
|
341
|
+
"partial": bool(mc.get("partialWords", False)),
|
|
342
|
+
"precise": bool(mc.get("precise", False)),
|
|
343
|
+
"distinct": g.distinct_words if mc.get("distinctWords") else None,
|
|
344
|
+
}
|
|
345
|
+
ignore = [w for w in (normalize(x) for x in mc.get("ignore", [])) if w]
|
|
346
|
+
out: List[Match] = []
|
|
347
|
+
for entity in g.all_entities():
|
|
348
|
+
if types is not None and entity.type not in types:
|
|
349
|
+
continue
|
|
350
|
+
m = g.type_config(entity.type).get("match") or {}
|
|
351
|
+
if m.get("skip"):
|
|
352
|
+
continue
|
|
353
|
+
best: Optional[Tuple[float, str, str]] = None
|
|
354
|
+
for raw, is_alias in [(entity.label, False)] + [(a, True) for a in entity.aliases]:
|
|
355
|
+
name = normalize(raw)
|
|
356
|
+
if any(w in name for w in ignore):
|
|
357
|
+
continue
|
|
358
|
+
s = _score_name(q, name, is_alias, m, settings)
|
|
359
|
+
if s and (best is None or s[0] > best[0]):
|
|
360
|
+
best = s
|
|
361
|
+
if best is None:
|
|
362
|
+
continue
|
|
363
|
+
reason: Reason = {"kind": "match", "by": best[1], "name": best[2]}
|
|
364
|
+
out.append(Match(entity, best[0], m.get("tier", 1), reason, render_reason(g, reason, entity, templates)))
|
|
365
|
+
precise = settings["precise"]
|
|
366
|
+
out.sort(key=lambda x: ((1 if precise and x.score < STRONG else 0), x.tier, -x.score))
|
|
367
|
+
return out[:limit]
|
|
368
|
+
|
|
369
|
+
def match(self, query: str, similar_limit: int = 3, templates: Optional[Dict[str, Any]] = None) -> MatchResult:
|
|
370
|
+
"""What a query means: one entity, things like one, or nothing here."""
|
|
371
|
+
result = self._match(query, similar_limit, templates)
|
|
372
|
+
if self._events:
|
|
373
|
+
event: Dict[str, Any] = {"type": "matched", "intent": result.intent}
|
|
374
|
+
if result.entity is not None:
|
|
375
|
+
event["entityType"] = result.entity.type
|
|
376
|
+
if result.match is not None and result.match.reason.get("by"):
|
|
377
|
+
event["by"] = result.match.reason["by"]
|
|
378
|
+
self._events.emit(event)
|
|
379
|
+
return result
|
|
380
|
+
|
|
381
|
+
def _match(self, query: str, similar_limit: int, templates: Optional[Dict[str, Any]]) -> MatchResult:
|
|
382
|
+
g = self.graph
|
|
383
|
+
found = self.search(query, 1, templates=templates)
|
|
384
|
+
best = found[0] if found else None
|
|
385
|
+
q = normalize(query)
|
|
386
|
+
mc = g.config.get("match") or {}
|
|
387
|
+
typo = mc.get("typoMinLength", 4)
|
|
388
|
+
missing = None
|
|
389
|
+
for raw in mc.get("notHere", []):
|
|
390
|
+
name = normalize(raw)
|
|
391
|
+
if name and has_phrase(q, name, typo) and (missing is None or len(name) > len(missing[1])):
|
|
392
|
+
missing = (raw, name)
|
|
393
|
+
if missing and (best is None or len(missing[1]) > len(best.reason.get("name") or normalize(best.entity.label))):
|
|
394
|
+
return MatchResult(query, "none", not_here=missing[0])
|
|
395
|
+
if best is None:
|
|
396
|
+
return MatchResult(query, "none")
|
|
397
|
+
kinds = g.type_config(best.entity.type).get("similar")
|
|
398
|
+
if kinds is not False and re.search(mc.get("similarPattern", DEFAULT_SIMILAR), q):
|
|
399
|
+
similar = [
|
|
400
|
+
Related(e, l.weight, l, reason_from_link(l, e.id), render_reason(g, reason_from_link(l, e.id), e, templates))
|
|
401
|
+
for e, l in self._neighbours(best.entity.id, kinds)
|
|
402
|
+
if g.is_destination(e)
|
|
403
|
+
][:similar_limit]
|
|
404
|
+
if similar:
|
|
405
|
+
return MatchResult(query, "similar", best.entity, best, similar)
|
|
406
|
+
return MatchResult(query, "entity", best.entity, best)
|
|
407
|
+
|
|
408
|
+
# --- related and next
|
|
409
|
+
|
|
410
|
+
def _neighbours(self, id: str, kinds: Optional[Sequence[str]] = None) -> List[Tuple[Entity, Link]]:
|
|
411
|
+
out = []
|
|
412
|
+
for l in self.graph.adjacent.get(id, []):
|
|
413
|
+
if kinds is not None and l.kind not in kinds:
|
|
414
|
+
continue
|
|
415
|
+
e = self.graph.entities.get(l.other_end(id))
|
|
416
|
+
if e is not None:
|
|
417
|
+
out.append((e, l))
|
|
418
|
+
return sorted(out, key=lambda x: -x[1].weight)
|
|
419
|
+
|
|
420
|
+
def related(self, id: str, kinds: Optional[Sequence[str]] = None, limit: Optional[int] = None, destinations_only: bool = True, accept: Optional[Callable[[Entity], bool]] = None, templates: Optional[Dict[str, Any]] = None) -> List[Related]:
|
|
421
|
+
"""What is related to one entity, strongest link first, each with why."""
|
|
422
|
+
g = self.graph
|
|
423
|
+
pairs = sorted(self._neighbours(id, kinds), key=lambda x: (-x[1].weight, -g.pull(x[1].kind)))
|
|
424
|
+
pairs = [(e, l) for e, l in pairs if (not destinations_only or g.is_destination(e)) and (accept is None or accept(e))]
|
|
425
|
+
if limit is not None:
|
|
426
|
+
pairs = pairs[:limit]
|
|
427
|
+
out = []
|
|
428
|
+
for e, l in pairs:
|
|
429
|
+
reason = reason_from_link(l, e.id)
|
|
430
|
+
out.append(Related(e, l.weight, l, reason, render_reason(g, reason, e, templates)))
|
|
431
|
+
self._suggested("related", out)
|
|
432
|
+
return out
|
|
433
|
+
|
|
434
|
+
def spread(self, trail: Sequence[str]) -> Tuple[Dict[str, float], Dict[str, Tuple[Link, str, str, float]], Dict[str, float], Set[str]]:
|
|
435
|
+
"""How brightly each entity lights up for this trail, with the strongest reason for each."""
|
|
436
|
+
g = self.graph
|
|
437
|
+
cfg = g.config.get("next") or {}
|
|
438
|
+
recency = cfg.get("recency", 0.6)
|
|
439
|
+
second = cfg.get("secondStep", 0.4)
|
|
440
|
+
hub = cfg.get("hubDamping", 3)
|
|
441
|
+
boost = cfg.get("sectionBoost", 0.08)
|
|
442
|
+
sections = cfg.get("sections") or {}
|
|
443
|
+
score: Dict[str, float] = {}
|
|
444
|
+
why: Dict[str, Tuple[Link, str, str, float]] = {}
|
|
445
|
+
section_only: Set[str] = set()
|
|
446
|
+
|
|
447
|
+
def give(id: str, part: float, link: Link, src: str, seed: str) -> None:
|
|
448
|
+
score[id] = score.get(id, 0) + part
|
|
449
|
+
if part > (why[id][3] if id in why else 0):
|
|
450
|
+
why[id] = (link, src, seed, part)
|
|
451
|
+
|
|
452
|
+
seeds: Dict[str, float] = {}
|
|
453
|
+
for id in reversed(list(trail)):
|
|
454
|
+
if id not in seeds:
|
|
455
|
+
seeds[id] = recency ** len(seeds)
|
|
456
|
+
for seed, w in seeds.items():
|
|
457
|
+
if seed.startswith("section:"):
|
|
458
|
+
type = sections.get(seed[8:])
|
|
459
|
+
if not type:
|
|
460
|
+
continue
|
|
461
|
+
for e in g.entities.values():
|
|
462
|
+
if e.type != type:
|
|
463
|
+
continue
|
|
464
|
+
score[e.id] = score.get(e.id, 0) + w * boost * min(1, g.degree(e.id) / 8)
|
|
465
|
+
section_only.add(e.id)
|
|
466
|
+
continue
|
|
467
|
+
for l1 in g.adjacent.get(seed, []):
|
|
468
|
+
mid = l1.other_end(seed)
|
|
469
|
+
a = w * l1.weight * g.pull(l1.kind)
|
|
470
|
+
give(mid, a, l1, seed, seed)
|
|
471
|
+
carry = (a * second) / math.sqrt(1 + g.degree(mid) / hub)
|
|
472
|
+
for l2 in g.adjacent.get(mid, []):
|
|
473
|
+
far = l2.other_end(mid)
|
|
474
|
+
if far == seed:
|
|
475
|
+
continue
|
|
476
|
+
give(far, carry * l2.weight * g.pull(l2.kind), l2, mid, seed)
|
|
477
|
+
return score, why, seeds, section_only
|
|
478
|
+
|
|
479
|
+
def next(self, trail: Sequence[str], limit: int = 3, exclude: Sequence[str] = (), accept: Optional[Callable[[Entity], bool]] = None, templates: Optional[Dict[str, Any]] = None) -> List[Suggestion]:
|
|
480
|
+
"""The next things to suggest after this trail: unseen, worth going to, and mixed."""
|
|
481
|
+
g = self.graph
|
|
482
|
+
score, why, seeds, section_only = self.spread(trail)
|
|
483
|
+
skip = set(seeds) | set(exclude)
|
|
484
|
+
ranked = []
|
|
485
|
+
for id, s in score.items():
|
|
486
|
+
e = g.entities.get(id)
|
|
487
|
+
if e is None or not (id in why or id in section_only) or not g.is_destination(e) or id in skip or (accept is not None and not accept(e)):
|
|
488
|
+
continue
|
|
489
|
+
ranked.append((e, s, why.get(id)))
|
|
490
|
+
ranked.sort(key=lambda r: -r[1])
|
|
491
|
+
cap = max(1, limit - 1)
|
|
492
|
+
taken: List[int] = []
|
|
493
|
+
per_type: Dict[str, int] = {}
|
|
494
|
+
for i, r in enumerate(ranked):
|
|
495
|
+
if len(taken) == limit:
|
|
496
|
+
break
|
|
497
|
+
if per_type.get(r[0].type, 0) >= cap:
|
|
498
|
+
continue
|
|
499
|
+
per_type[r[0].type] = per_type.get(r[0].type, 0) + 1
|
|
500
|
+
taken.append(i)
|
|
501
|
+
for i in range(len(ranked)):
|
|
502
|
+
if len(taken) < limit and i not in taken:
|
|
503
|
+
taken.append(i)
|
|
504
|
+
chosen = sorted((ranked[i] for i in taken), key=lambda r: -r[1])
|
|
505
|
+
out = []
|
|
506
|
+
for e, s, w in chosen:
|
|
507
|
+
reason = reason_from_link(w[0], e.id, w[2]) if w else {"kind": "section"}
|
|
508
|
+
out.append(Suggestion(e, s, reason, render_reason(g, reason, e, templates)))
|
|
509
|
+
self._suggested("next", out)
|
|
510
|
+
return out
|
|
511
|
+
|
|
512
|
+
def autocomplete(self, typed: str, trail: Sequence[str] = (), limit: int = 3, accept: Optional[Callable[[Entity], bool]] = None, templates: Optional[Dict[str, Any]] = None) -> List[Suggestion]:
|
|
513
|
+
"""Type-ahead: names that start with (or contain) what's typed, nearest the trail first."""
|
|
514
|
+
g = self.graph
|
|
515
|
+
q = normalize(typed)
|
|
516
|
+
if len(q) < 2:
|
|
517
|
+
return []
|
|
518
|
+
near = self.spread(trail)[0]
|
|
519
|
+
hits = []
|
|
520
|
+
for e in g.entities.values():
|
|
521
|
+
if not g.is_destination(e) or (accept is not None and not accept(e)):
|
|
522
|
+
continue
|
|
523
|
+
name = normalize(e.label)
|
|
524
|
+
kind = 3 if name.startswith(q) else 2 if any(w.startswith(q) for w in name.split(" ")) else 1 if len(q) >= 3 and q in name else 0
|
|
525
|
+
if not kind:
|
|
526
|
+
continue
|
|
527
|
+
by = "prefix" if kind == 3 else "word-prefix" if kind == 2 else "contains"
|
|
528
|
+
hits.append((e, by, kind * 10 + near.get(e.id, 0) * 5 + min(g.degree(e.id), 20) / 20))
|
|
529
|
+
hits.sort(key=lambda h: -h[2])
|
|
530
|
+
out = []
|
|
531
|
+
for e, by, rank in hits[:limit]:
|
|
532
|
+
reason: Reason = {"kind": "typed", "by": by}
|
|
533
|
+
out.append(Suggestion(e, rank, reason, render_reason(g, reason, e, templates)))
|
|
534
|
+
return out
|
|
535
|
+
|
|
536
|
+
def explain(self, reason: Reason, item: Union[Entity, str], templates: Optional[Dict[str, Any]] = None) -> str:
|
|
537
|
+
"""A reason as a sentence."""
|
|
538
|
+
e = self.graph.entities.get(item) if isinstance(item, str) else item
|
|
539
|
+
if e is None:
|
|
540
|
+
raise KeyError(f"pubba: no entity {item}")
|
|
541
|
+
return render_reason(self.graph, reason, e, templates)
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def load_bundle(bundle: Union[str, bytes, Dict[str, Any]], events: Optional[Events] = None) -> Pubba:
|
|
545
|
+
"""Answers from a compiled bundle (`pubba compile`), exactly as the content it was compiled from."""
|
|
546
|
+
return Pubba(graph_from_bundle(bundle), events)
|
pubba/events.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Privacy-safe measurement events (SPEC section 8).
|
|
2
|
+
|
|
3
|
+
pubba only emits: no storage, no network, no visitor ids, no timestamps.
|
|
4
|
+
Event fields hold entity ids, types, kinds and declared goals; never a query.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Any, Callable, Dict, Iterable, List, Optional
|
|
10
|
+
|
|
11
|
+
GUIDE_VIAS = ("suggestion", "chip", "typed", "match", "related")
|
|
12
|
+
|
|
13
|
+
Event = Dict[str, Any]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Events:
|
|
17
|
+
def __init__(self, goals: Optional[Iterable[str]] = None) -> None:
|
|
18
|
+
self.goals = tuple(goals if goals is not None else ("contact", "resume"))
|
|
19
|
+
self._listeners: List[Callable[[Event], None]] = []
|
|
20
|
+
|
|
21
|
+
def on(self, listener: Callable[[Event], None]) -> Callable[[], None]:
|
|
22
|
+
"""Listen to every event; returns an unsubscribe function. A listener that raises is ignored."""
|
|
23
|
+
self._listeners.append(listener)
|
|
24
|
+
return lambda: self._listeners.remove(listener) if listener in self._listeners else None
|
|
25
|
+
|
|
26
|
+
def emit(self, event: Event) -> None:
|
|
27
|
+
for listener in list(self._listeners):
|
|
28
|
+
try:
|
|
29
|
+
listener(dict(event))
|
|
30
|
+
except Exception:
|
|
31
|
+
# Measurement must never break the guide.
|
|
32
|
+
pass
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def create_events(goals: Optional[Iterable[str]] = None) -> Events:
|
|
36
|
+
return Events(goals)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def summarise(events: Iterable[Event]) -> Dict[str, Any]:
|
|
40
|
+
"""One visit's funnel: was the guide used, where did it lead, which goals were reached."""
|
|
41
|
+
guided_to: List[str] = []
|
|
42
|
+
guided_types: List[str] = []
|
|
43
|
+
reached: List[str] = []
|
|
44
|
+
used = False
|
|
45
|
+
before_goal = False
|
|
46
|
+
for e in events:
|
|
47
|
+
t = e.get("type")
|
|
48
|
+
if t in ("guide-opened", "matched"):
|
|
49
|
+
used = True
|
|
50
|
+
if t == "guided-to":
|
|
51
|
+
used = True
|
|
52
|
+
if e["to"] not in guided_to:
|
|
53
|
+
guided_to.append(e["to"])
|
|
54
|
+
if e["entityType"] not in guided_types:
|
|
55
|
+
guided_types.append(e["entityType"])
|
|
56
|
+
if t == "reached":
|
|
57
|
+
if e["goal"] not in reached:
|
|
58
|
+
reached.append(e["goal"])
|
|
59
|
+
if guided_to:
|
|
60
|
+
before_goal = True
|
|
61
|
+
return {"usedGuide": used, "guidedTo": guided_to, "guidedToTypes": guided_types, "reached": reached, "guidedBeforeGoal": before_goal}
|
pubba/guide.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"""The headless guide (SPEC section 9): replies, chips, the trail and nudges, with no UI."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import time
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Union
|
|
9
|
+
|
|
10
|
+
from .core import Entity, Match, Pubba, Reason
|
|
11
|
+
from .text import collapse, js_trim
|
|
12
|
+
from .trail import Trail
|
|
13
|
+
|
|
14
|
+
DEFAULT_GUIDE_TEMPLATES: Dict[str, str] = {
|
|
15
|
+
"open": "Here's {item}.",
|
|
16
|
+
"similar": "{reason}",
|
|
17
|
+
"none": "I couldn't find that here. These might help:",
|
|
18
|
+
"notHere": "{name} isn't here. These might help:",
|
|
19
|
+
"start": "Where would you like to start?",
|
|
20
|
+
"nudge": "{reason} Want a look?",
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
_TOKEN = re.compile(r"\{(\w+)\}")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class EntityChip:
|
|
28
|
+
"""A follow-up button: a place to go, with why."""
|
|
29
|
+
|
|
30
|
+
id: str
|
|
31
|
+
label: str
|
|
32
|
+
reason: Reason
|
|
33
|
+
text: str
|
|
34
|
+
url: Optional[str] = None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class QuestionChip:
|
|
39
|
+
"""A follow-up button that asks the guide a question."""
|
|
40
|
+
|
|
41
|
+
label: str
|
|
42
|
+
question: str
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
Chip = Union[EntityChip, QuestionChip]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class Reply:
|
|
50
|
+
kind: str
|
|
51
|
+
text: str
|
|
52
|
+
chips: List[Chip] = field(default_factory=list)
|
|
53
|
+
action: Optional[Dict[str, str]] = None
|
|
54
|
+
match: Optional[Match] = None
|
|
55
|
+
similar: Optional[List[EntityChip]] = None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass
|
|
59
|
+
class NudgePolicy:
|
|
60
|
+
delay_ms: float = 9000
|
|
61
|
+
gap_ms: float = 60000
|
|
62
|
+
min_score: float = 0.3
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _fill(text: str, values: Dict[str, str]) -> str:
|
|
66
|
+
return collapse(_TOKEN.sub(lambda m: values.get(m.group(1)) or "", text))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class Guide:
|
|
70
|
+
"""The navigator: a typed question becomes a reply; steps are recorded; nudges come at the right moments."""
|
|
71
|
+
|
|
72
|
+
def __init__(
|
|
73
|
+
self,
|
|
74
|
+
pubba: Pubba,
|
|
75
|
+
trail: Optional[Trail] = None,
|
|
76
|
+
route: Optional[Callable[[Entity], Optional[str]]] = None,
|
|
77
|
+
chips: int = 3,
|
|
78
|
+
nudge: Optional[NudgePolicy] = None,
|
|
79
|
+
prefer: Optional[Callable[[Entity], bool]] = None,
|
|
80
|
+
accept: Optional[Callable[[Entity], bool]] = None,
|
|
81
|
+
fallback: Optional[Callable[[Dict[str, Any]], Optional[Sequence[Union[str, Chip]]]]] = None,
|
|
82
|
+
line: Optional[Callable[[Entity], Optional[str]]] = None,
|
|
83
|
+
templates: Optional[Dict[str, str]] = None,
|
|
84
|
+
now: Optional[Callable[[], float]] = None,
|
|
85
|
+
) -> None:
|
|
86
|
+
self.pubba = pubba
|
|
87
|
+
self.trail = trail or Trail()
|
|
88
|
+
self._route = route or (lambda e: e.url)
|
|
89
|
+
self._limit = chips
|
|
90
|
+
self.nudge_policy = nudge or NudgePolicy()
|
|
91
|
+
self._prefer = prefer or (lambda e: self._route(e) is not None)
|
|
92
|
+
self._accept = accept
|
|
93
|
+
self._fallback = fallback
|
|
94
|
+
self._line = line
|
|
95
|
+
self._t = {**DEFAULT_GUIDE_TEMPLATES, **(templates or {})}
|
|
96
|
+
self._now = now or (lambda: time.time() * 1000)
|
|
97
|
+
self._last_step_at = float("-inf")
|
|
98
|
+
self._last_step_self = False
|
|
99
|
+
self._last_nudge_at = float("-inf")
|
|
100
|
+
self._nudged_from: Set[str] = set()
|
|
101
|
+
|
|
102
|
+
# --- helpers
|
|
103
|
+
|
|
104
|
+
def _tpl(self, key: str, e: Optional[Entity] = None) -> str:
|
|
105
|
+
if e is not None and f"{key}.{e.type}" in self._t:
|
|
106
|
+
return self._t[f"{key}.{e.type}"]
|
|
107
|
+
return self._t[key]
|
|
108
|
+
|
|
109
|
+
def _vars(self, e: Entity) -> Dict[str, str]:
|
|
110
|
+
line = self._line(e) if self._line else None
|
|
111
|
+
return {"item": e.short_label, "label": e.label, "line": line or ""}
|
|
112
|
+
|
|
113
|
+
def _chip(self, entity: Entity, reason: Reason, text: str) -> EntityChip:
|
|
114
|
+
return EntityChip(entity.id, entity.short_label, reason, text, self._route(entity))
|
|
115
|
+
|
|
116
|
+
def _open(self, e: Entity) -> Dict[str, str]:
|
|
117
|
+
action = {"type": "open", "id": e.id}
|
|
118
|
+
url = self._route(e)
|
|
119
|
+
if url is not None:
|
|
120
|
+
action["url"] = url
|
|
121
|
+
return action
|
|
122
|
+
|
|
123
|
+
def _starters(self, n: int) -> List[Chip]:
|
|
124
|
+
g = self.pubba.graph
|
|
125
|
+
es = [e for e in g.entities.values() if g.is_destination(e) and (self._accept is None or self._accept(e))]
|
|
126
|
+
es = sorted(enumerate(es), key=lambda x: (-g.degree(x[1].id), x[0]))[:n]
|
|
127
|
+
reason: Reason = {"kind": "section"}
|
|
128
|
+
return [self._chip(e, dict(reason), self.pubba.explain(reason, e)) for _, e in es]
|
|
129
|
+
|
|
130
|
+
def _chips(self, n: int, extra: Sequence[str], context: Dict[str, Any]) -> List[Chip]:
|
|
131
|
+
out: List[Chip] = [self._chip(s.entity, s.reason, s.text) for s in self.pubba.next(self.trail.get() + list(extra), limit=n, accept=self._accept)]
|
|
132
|
+
if out:
|
|
133
|
+
return out
|
|
134
|
+
fallback = self._fallback(context) if self._fallback else None
|
|
135
|
+
if fallback is not None:
|
|
136
|
+
return [QuestionChip(c, c) if isinstance(c, str) else c for c in list(fallback)[:n]]
|
|
137
|
+
return self._starters(n)
|
|
138
|
+
|
|
139
|
+
# --- the guide
|
|
140
|
+
|
|
141
|
+
def ask(self, question: str) -> Reply:
|
|
142
|
+
"""Answer a typed question. Doesn't move the visitor: follow the action, then call follow()."""
|
|
143
|
+
if not js_trim(question):
|
|
144
|
+
return Reply("start", _fill(self._tpl("start"), {}), self._chips(self._limit, [], {"kind": "start"}))
|
|
145
|
+
m = self.pubba.match(question)
|
|
146
|
+
if m.intent == "none":
|
|
147
|
+
if m.not_here:
|
|
148
|
+
return Reply("not-here", _fill(self._tpl("notHere"), {"name": m.not_here}), self._chips(self._limit, [], {"kind": "not-here"}))
|
|
149
|
+
return Reply("none", _fill(self._tpl("none"), {}), self._chips(self._limit, [], {"kind": "none"}))
|
|
150
|
+
entity = m.entity
|
|
151
|
+
assert entity is not None
|
|
152
|
+
similar = [r for r in m.similar if self._accept is None or self._accept(r.entity)]
|
|
153
|
+
if m.intent == "similar" and similar:
|
|
154
|
+
first = similar[0]
|
|
155
|
+
listed = ", ".join(r.entity.label + (" (" + ", ".join(r.reason["via"][:2]) + ")" if r.reason.get("via") else "") for r in similar)
|
|
156
|
+
return Reply(
|
|
157
|
+
"similar",
|
|
158
|
+
_fill(self._tpl("similar", entity), {**self._vars(first.entity), "reason": first.text, "of": entity.label, "list": listed}),
|
|
159
|
+
[self._chip(r.entity, r.reason, r.text) for r in similar[1:][: self._limit]],
|
|
160
|
+
self._open(first.entity),
|
|
161
|
+
m.match,
|
|
162
|
+
[self._chip(r.entity, r.reason, r.text) for r in similar],
|
|
163
|
+
)
|
|
164
|
+
context = {"kind": "open", "entity": entity}
|
|
165
|
+
if entity.id in self.pubba.graph.entities:
|
|
166
|
+
chips = [c for c in self._chips(self._limit, [entity.id], context) if not isinstance(c, EntityChip) or c.id != entity.id]
|
|
167
|
+
else:
|
|
168
|
+
chips = self._chips(self._limit, [], context)
|
|
169
|
+
return Reply("open", _fill(self._tpl("open", entity), self._vars(entity)), chips, self._open(entity), m.match)
|
|
170
|
+
|
|
171
|
+
def chips(self, limit: Optional[int] = None) -> List[Chip]:
|
|
172
|
+
"""Follow-ups for where the visitor is now."""
|
|
173
|
+
return self._chips(self._limit if limit is None else limit, [], {"kind": "chips"})
|
|
174
|
+
|
|
175
|
+
def visited(self, id: str) -> None:
|
|
176
|
+
"""The visitor went somewhere on their own."""
|
|
177
|
+
if self.pubba.graph.find(id) is None and not id.startswith("section:"):
|
|
178
|
+
return
|
|
179
|
+
self.trail.record(id)
|
|
180
|
+
self._last_step_at = self._now()
|
|
181
|
+
self._last_step_self = True
|
|
182
|
+
|
|
183
|
+
def follow(self, id: str, via: str) -> None:
|
|
184
|
+
"""The visitor went somewhere because the guide sent them."""
|
|
185
|
+
if self.pubba.graph.find(id) is None:
|
|
186
|
+
return
|
|
187
|
+
self.trail.record(id)
|
|
188
|
+
self._last_step_at = self._now()
|
|
189
|
+
self._last_step_self = False
|
|
190
|
+
self.pubba.track.guided_to(id, via)
|
|
191
|
+
|
|
192
|
+
def nudge(self) -> Optional[Reply]:
|
|
193
|
+
"""A suggestion to show unprompted now, or None. Call it after nudge_policy.delay_ms."""
|
|
194
|
+
steps = self.trail.get()
|
|
195
|
+
last = steps[-1] if steps else None
|
|
196
|
+
at = self._now()
|
|
197
|
+
if not last or not self._last_step_self or last.startswith("section:") or last in self._nudged_from:
|
|
198
|
+
return None
|
|
199
|
+
p = self.nudge_policy
|
|
200
|
+
if at - self._last_step_at < p.delay_ms or at - self._last_nudge_at < p.gap_ms:
|
|
201
|
+
return None
|
|
202
|
+
options = [s for s in self.pubba.next(steps, limit=5, accept=self._accept) if s.reason["kind"] != "section" and s.score >= p.min_score]
|
|
203
|
+
top = next((s for s in options if self._prefer(s.entity)), options[0] if options else None)
|
|
204
|
+
if top is None:
|
|
205
|
+
return None
|
|
206
|
+
self._last_nudge_at = at
|
|
207
|
+
self._nudged_from.add(last)
|
|
208
|
+
return Reply(
|
|
209
|
+
"open",
|
|
210
|
+
_fill(self._tpl("nudge", top.entity), {**self._vars(top.entity), "reason": top.text}),
|
|
211
|
+
[self._chip(top.entity, top.reason, top.text)],
|
|
212
|
+
self._open(top.entity),
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
def opened(self) -> None:
|
|
216
|
+
self.pubba.track.guide_opened()
|
|
217
|
+
|
|
218
|
+
def reached(self, goal: str) -> None:
|
|
219
|
+
self.pubba.track.reached(goal)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def create_guide(pubba: Pubba, **options: Any) -> Guide:
|
|
223
|
+
return Guide(pubba, **options)
|
pubba/py.typed
ADDED
|
File without changes
|
pubba/text.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Text helpers behind matching (SPEC section 3)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import unicodedata
|
|
7
|
+
|
|
8
|
+
# JavaScript's \s, which the reference uses: Unicode white space plus the byte order mark.
|
|
9
|
+
_JS_SPACE = "\t\n\v\f\r
"
|
|
10
|
+
_SPACE_CLASS = "[" + re.escape(_JS_SPACE) + "]"
|
|
11
|
+
_SPACES = re.compile(_SPACE_CLASS + "+")
|
|
12
|
+
_OTHER = re.compile("[^a-z0-9+#." + re.escape(_JS_SPACE) + "]")
|
|
13
|
+
_MARKS = re.compile("[̀-ͯ]")
|
|
14
|
+
_NOT_LETTER = re.compile("[^a-z]")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def js_trim(s: str) -> str:
|
|
18
|
+
"""Trim like JavaScript's String.prototype.trim."""
|
|
19
|
+
return s.strip(_JS_SPACE)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def collapse(s: str) -> str:
|
|
23
|
+
"""Runs of white space become one space, then trimmed."""
|
|
24
|
+
return js_trim(_SPACES.sub(" ", s))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def normalize(s: str) -> str:
|
|
28
|
+
"""Lowercase, accents off, punctuation to spaces (keeping + # . for C#, C++, Next.js)."""
|
|
29
|
+
s = unicodedata.normalize("NFKD", s.lower())
|
|
30
|
+
s = _MARKS.sub("", s)
|
|
31
|
+
s = _OTHER.sub(" ", s)
|
|
32
|
+
return collapse(s)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def edit_distance(a: str, b: str) -> int:
|
|
36
|
+
"""Optimal string alignment: a swap of two neighbouring characters counts once."""
|
|
37
|
+
d = [[0] * (len(b) + 1) for _ in range(len(a) + 1)]
|
|
38
|
+
for i in range(len(a) + 1):
|
|
39
|
+
d[i][0] = i
|
|
40
|
+
for j in range(len(b) + 1):
|
|
41
|
+
d[0][j] = j
|
|
42
|
+
for i in range(1, len(a) + 1):
|
|
43
|
+
for j in range(1, len(b) + 1):
|
|
44
|
+
cost = 0 if a[i - 1] == b[j - 1] else 1
|
|
45
|
+
d[i][j] = min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + cost)
|
|
46
|
+
if i > 1 and j > 1 and a[i - 1] == b[j - 2] and a[i - 2] == b[j - 1]:
|
|
47
|
+
d[i][j] = min(d[i][j], d[i - 2][j - 2] + 1)
|
|
48
|
+
return d[len(a)][len(b)]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def close_to(token: str | None, word: str, typo_min_length: float = 4) -> bool:
|
|
52
|
+
"""Does a typed word mean this word? Exact, plural, prefix, or a near miss."""
|
|
53
|
+
if not token:
|
|
54
|
+
return False
|
|
55
|
+
if token == word or token == word + "s":
|
|
56
|
+
return True
|
|
57
|
+
# Short words and names with symbols (go, js, c#, next.js) must match exactly.
|
|
58
|
+
if len(word) < 4 or _NOT_LETTER.search(word):
|
|
59
|
+
return False
|
|
60
|
+
if token.startswith(word):
|
|
61
|
+
return True
|
|
62
|
+
if len(token) < 3 or len(word) < typo_min_length:
|
|
63
|
+
return False
|
|
64
|
+
most = 2 if len(word) >= 7 else 1
|
|
65
|
+
return edit_distance(token, word) <= most or (len(token) > len(word) and edit_distance(token[: len(word)], word) <= most)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def has_phrase(query: str, phrase: str, typo_min_length: float = 4) -> bool:
|
|
69
|
+
"""Does a normalised query contain this phrase, word by word, typos allowed?"""
|
|
70
|
+
tokens = query.split(" ")
|
|
71
|
+
words = phrase.split(" ")
|
|
72
|
+
for i in range(len(tokens) - len(words) + 1):
|
|
73
|
+
if all(close_to(tokens[i + j], w, typo_min_length) for j, w in enumerate(words)):
|
|
74
|
+
return True
|
|
75
|
+
return False
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def contains_name(query: str, name: str) -> bool:
|
|
79
|
+
"""Does a normalised query contain this normalised name as whole words?"""
|
|
80
|
+
return (" " + name + " ") in (" " + query + " ")
|
pubba/trail.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""The visitor's trail (SPEC section 7)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from typing import Callable, List, Optional, Protocol
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class TrailStorage(Protocol):
|
|
10
|
+
def get_item(self, key: str) -> Optional[str]: ...
|
|
11
|
+
def set_item(self, key: str, value: str) -> None: ...
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Trail:
|
|
15
|
+
"""A visitor's trail through the site, in entity ids, most recent last. Never sent anywhere."""
|
|
16
|
+
|
|
17
|
+
def __init__(self, max: int = 24, storage: Optional[TrailStorage] = None, key: str = "pubba-trail") -> None:
|
|
18
|
+
self._max = max
|
|
19
|
+
self._storage = storage
|
|
20
|
+
self._key = key
|
|
21
|
+
self._steps: List[str] = []
|
|
22
|
+
self._loaded = False
|
|
23
|
+
self._listeners: List[Callable[[], None]] = []
|
|
24
|
+
|
|
25
|
+
def _load(self) -> None:
|
|
26
|
+
if self._loaded:
|
|
27
|
+
return
|
|
28
|
+
self._loaded = True
|
|
29
|
+
try:
|
|
30
|
+
saved = json.loads((self._storage.get_item(self._key) if self._storage else None) or "[]")
|
|
31
|
+
if isinstance(saved, list):
|
|
32
|
+
self._steps = [x for x in saved if isinstance(x, str)][-self._max :]
|
|
33
|
+
except Exception:
|
|
34
|
+
# A broken or missing save starts a fresh trail.
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
def _save(self) -> None:
|
|
38
|
+
try:
|
|
39
|
+
if self._storage:
|
|
40
|
+
self._storage.set_item(self._key, json.dumps(self._steps))
|
|
41
|
+
except Exception:
|
|
42
|
+
# Storage full or blocked: the trail lives on in memory.
|
|
43
|
+
pass
|
|
44
|
+
for listener in list(self._listeners):
|
|
45
|
+
listener()
|
|
46
|
+
|
|
47
|
+
def record(self, id: Optional[str]) -> None:
|
|
48
|
+
"""One step; the same step twice in a row counts once."""
|
|
49
|
+
if not id:
|
|
50
|
+
return
|
|
51
|
+
self._load()
|
|
52
|
+
if self._steps and self._steps[-1] == id:
|
|
53
|
+
return
|
|
54
|
+
self._steps = (self._steps + [id])[-self._max :]
|
|
55
|
+
self._save()
|
|
56
|
+
|
|
57
|
+
def get(self) -> List[str]:
|
|
58
|
+
self._load()
|
|
59
|
+
return list(self._steps)
|
|
60
|
+
|
|
61
|
+
def clear(self) -> None:
|
|
62
|
+
self._loaded = True
|
|
63
|
+
self._steps = []
|
|
64
|
+
self._save()
|
|
65
|
+
|
|
66
|
+
def subscribe(self, listener: Callable[[], None]) -> Callable[[], None]:
|
|
67
|
+
"""Called after every change; returns an unsubscribe function."""
|
|
68
|
+
self._listeners.append(listener)
|
|
69
|
+
return lambda: self._listeners.remove(listener) if listener in self._listeners else None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def create_trail(max: int = 24, storage: Optional[TrailStorage] = None, key: str = "pubba-trail") -> Trail:
|
|
73
|
+
return Trail(max=max, storage=storage, key=key)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def section_step(section: str) -> str:
|
|
77
|
+
"""A trail step for opening a section, which lights every entity of its type a little."""
|
|
78
|
+
return "section:" + section
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pubba
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: A small brain for showroom sites: related items with reasons, what-next from a visitor's trail, typo-tolerant matching and a headless guide, from a compiled pubba bundle.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Tharindu127/pubba
|
|
6
|
+
Project-URL: Repository, https://github.com/Tharindu127/pubba
|
|
7
|
+
Project-URL: Issues, https://github.com/Tharindu127/pubba/issues
|
|
8
|
+
Author: Tharindu Athapaththu
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: explainable,fuzzy-search,guide,knowledge-graph,portfolio,recommendations,related-items,showroom
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
|
|
16
|
+
Classifier: Typing :: Typed
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# pubba for Python
|
|
21
|
+
|
|
22
|
+
The [pubba](https://github.com/Tharindu127/pubba) runtime in Python: load a
|
|
23
|
+
bundle compiled by `pubba compile` and answer related items with reasons, what
|
|
24
|
+
a visitor is likely to want next, typo-tolerant matching, a headless guide and
|
|
25
|
+
privacy-safe measurement events. No dependencies; Python 3.9+.
|
|
26
|
+
|
|
27
|
+
It gives the same answers as every other pubba runtime: it passes the shared
|
|
28
|
+
conformance suite (4,698 cases) exactly.
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install pubba
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from pubba import load_bundle, create_guide, create_trail
|
|
36
|
+
|
|
37
|
+
pubba = load_bundle(open("pubba.bundle.json").read())
|
|
38
|
+
|
|
39
|
+
pubba.match("flutr").entity.id # "tech:Flutter"
|
|
40
|
+
[s.text for s in pubba.next(["project:kiri-wallet"])]
|
|
41
|
+
# ["Kiri Merchant is built on the same stack as Kiri Wallet (Flutter, Dart).", ...]
|
|
42
|
+
|
|
43
|
+
guide = create_guide(pubba, trail=create_trail())
|
|
44
|
+
reply = guide.ask("anything like kiri walet?")
|
|
45
|
+
reply.kind, reply.text, reply.action # "similar", "Kiri Merchant is built on ...", {"type": "open", ...}
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The bundle comes from the content, compiled once with the npm package:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
npx pubba compile pubba.json --out pubba.bundle.json
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## API
|
|
55
|
+
|
|
56
|
+
| | |
|
|
57
|
+
| --- | --- |
|
|
58
|
+
| `load_bundle(json, events=None)` | A `Pubba` from a bundle (a JSON string, bytes or dict) |
|
|
59
|
+
| `pubba.match(query)` | `MatchResult(intent, entity, match, similar, not_here)`; intent is `entity`, `similar` or `none` |
|
|
60
|
+
| `pubba.search(query, limit=10, types=None)` | Every entity a query could mean, best first |
|
|
61
|
+
| `pubba.next(trail, limit=3, exclude=(), accept=None)` | Predictions: `Suggestion(entity, score, reason, text)` |
|
|
62
|
+
| `pubba.related(id, kinds=None, limit=None)` | Neighbours, strongest first: `Related(entity, weight, link, reason, text)` |
|
|
63
|
+
| `pubba.autocomplete(typed, trail=(), limit=3)` | Type-ahead |
|
|
64
|
+
| `pubba.explain(reason, item)` | A reason as a sentence |
|
|
65
|
+
| `pubba.track.guide_opened() / guided_to(id, via) / reached(goal)` | Measurement moments, checked so no visitor text gets in |
|
|
66
|
+
| `create_guide(pubba, trail=, route=, chips=, nudge=NudgePolicy(), prefer=, accept=, fallback=, line=, templates=, now=)` | The headless navigator: `ask`, `chips`, `visited`, `follow`, `nudge`, `opened`, `reached` |
|
|
67
|
+
| `create_trail(max=24, storage=None, key=...)` | The visitor's trail; `storage` has `get_item` / `set_item` |
|
|
68
|
+
| `create_events(goals=None)`, `summarise(events)` | Events (dicts with the spec's field names), and one visit's funnel |
|
|
69
|
+
|
|
70
|
+
Reasons are dicts (`{"kind": "same-stack", "to": "project:kiri-wallet", "direction": "in", "via": ["Flutter", "Dart"]}`),
|
|
71
|
+
the same shape in every runtime. Behaviour is specified in the repository's
|
|
72
|
+
`spec/SPEC.md`.
|
|
73
|
+
|
|
74
|
+
## Tests
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
python -m unittest discover -s tests
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
runs the shared conformance suite from the repository's `conformance/` folder.
|
|
81
|
+
|
|
82
|
+
MIT © Tharindu Athapaththu
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
pubba/__init__.py,sha256=6fbx0t99Ta72lrFrFrMK1KOjf9YMsi4FAPkWGexHNlI,1339
|
|
2
|
+
pubba/core.py,sha256=uWOvOxbCTaDFapYHbZsKrwkqQU07Jiq4PaKkQ_T3Hgg,22425
|
|
3
|
+
pubba/events.py,sha256=PPBe1USkMD1tduhNkypFtTW54S6y93Lj3apYkoEUlf8,2251
|
|
4
|
+
pubba/guide.py,sha256=yALKwguS1Dig3ksYvNfnMvYi0eSreZHkRPDu9v8-CzA,8786
|
|
5
|
+
pubba/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
+
pubba/text.py,sha256=v7cnwpCC6QzrzKt0dPRdtYD8bLDqhI1cDMRu-p5dOhE,2985
|
|
7
|
+
pubba/trail.py,sha256=nm0if6mzS3tYHnDZTBI6wS-4ANyRt-V3GBCOve7zmuo,2630
|
|
8
|
+
pubba-0.4.0.dist-info/METADATA,sha256=DkfmYqIGT7SaD058ERZO-W7bNzcozkSZHIB_PtCayUE,3694
|
|
9
|
+
pubba-0.4.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
10
|
+
pubba-0.4.0.dist-info/licenses/LICENSE,sha256=UR5SXNrb80kxQORINgLxB7Z20tXEGIm7auUD7Bh1jr8,1077
|
|
11
|
+
pubba-0.4.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Tharindu Athapaththu
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|