nomenklatura-mpt 4.1.9__py3-none-any.whl

nomenklatura/tui/app.py

@@ -0,0 +1,294 @@
+import asyncio
+from typing import Dict, Optional, Set, Tuple, cast
+
+from rich.console import RenderableType
+from rich.text import Text
+from textual.app import App, ComposeResult
+from textual.containers import Grid, VerticalScroll
+from textual.screen import ModalScreen
+from textual.widget import Widget
+from textual.widgets import Button, Footer, Label, ListItem, ListView, Static
+
+from followthemoney import DS, SE
+
+from nomenklatura.judgement import Judgement
+from nomenklatura.resolver import Resolver
+from nomenklatura.resolver.edge import Edge
+from nomenklatura.store import Store
+from nomenklatura.tui.comparison import render_comparison
+
+HISTORY_LENGTH = 20
+
+
+class DedupeState(object):
+    def __init__(
+        self,
+        resolver: Resolver[SE],
+        store: Store[DS, SE],
+        url_base: Optional[str] = None,
+    ):
+        self.store = store
+        self.resolver = resolver
+        self.view = store.default_view(external=True)
+        self.url_base = url_base
+        self.latinize = False
+        self.message: Optional[str] = None
+        self.ignore: Set[Tuple[str, str]] = set()
+        self.left: Optional[SE] = None
+        self.right: Optional[SE] = None
+        self.score = 0.0
+        self.recents: Dict[str, SE] = dict()
+
+    def load(self) -> bool:
+        self.left = None
+        self.right = None
+        self.resolver.begin()
+        for left_id, right_id, score in self.resolver.get_candidates():
+            left_id = self.resolver.get_canonical(left_id)
+            right_id = self.resolver.get_canonical(right_id)
+            if (left_id, right_id) in self.ignore:
+                continue
+            if score is None:
+                self.ignore.add((left_id, right_id))
+                continue
+            if not self.resolver.check_candidate(left_id, right_id):
+                self.ignore.add((left_id, right_id))
+                continue
+            self.left = self.view.get_entity(left_id)
+            self.right = self.view.get_entity(right_id)
+            self.score = score
+            if self.left is not None and self.right is not None:
+                if self.left.schema == self.right.schema:
+                    return True
+                if self.left.schema.can_match(self.right.schema):
+                    return True
+            self.ignore.add((left_id, right_id))
+        return False
+
+    def decide(self, judgement: Judgement) -> None:
+        if self.left is not None and self.left.id is not None:
+            if self.right is not None and self.right.id is not None:
+                # Since we don't have an unresolved store as well as the resolved one,
+                # hold on to pre-merge entities to show in history.
+                self.recents[self.left.id] = self.left
+                self.recents[self.right.id] = self.right
+                canonical_id = self.resolver.decide(
+                    self.left.id,
+                    self.right.id,
+                    judgement=judgement,
+                )
+                self.store.update(canonical_id)
+        self.resolver.commit()
+        self.load()
+
+    def edit(self, edge: Edge, judgement: Judgement) -> None:
+        self.resolver.decide(edge.source, edge.target, judgement)
+        self.store.update(edge.source)
+        self.store.update(edge.target)
+        self.resolver.commit()
+        self.load()
+
+
+class DedupeAppWidget(Widget):
+    @property
+    def dedupe(self) -> DedupeState:
+        return cast(DedupeApp, self.app).dedupe
+
+
+class HistoryItem(Static, DedupeAppWidget):
+    def __init__(self, edge: Edge) -> None:
+        self.edge = edge
+        source = self.dedupe.recents.get(edge.source.id, None)
+        target = self.dedupe.recents.get(edge.target.id, None)
+        if target is None:
+            target = self.dedupe.view.get_entity(edge.target.id)
+        source_str = f"src: {edge.source.id}"
+        if source:
+            source_str += f"\n     {source.caption}"
+        target_str = f"tgt: {edge.target.id}"
+        if target:
+            target_str += f"\n     {target.caption}"
+
+        content = (
+            f"{edge.created_at if edge.created_at else 'unknown time'}\n"
+            f"{source_str}\n"
+            f"{target_str}\n"
+            f"{edge.user} decided {edge.judgement.value}"
+        )
+        super().__init__(content)
+
+
+class ConfirmEditModal(ModalScreen[bool]):
+    edge: Optional[Edge] = None
+    judgement: Optional[Judgement] = None
+
+    def compose(self) -> ComposeResult:
+        assert self.edge is not None
+        assert self.judgement is not None
+        message = f"Change {self.edge.source.id} -> {self.edge.target.id} to {self.judgement.value}?"
+        yield Grid(
+            Label(message, id="question"),
+            Button("Yes", variant="error", id="yes"),
+            Button("No", variant="primary", id="no"),
+            id="dialog",
+        )
+
+    def on_button_pressed(self, event: Button.Pressed) -> None:
+        if event.button.id == "yes":
+            self.dismiss(True)
+        else:
+            self.dismiss(False)
+
+
+class HistoryListView(ListView):
+    BINDINGS = [
+        ("x", "positive", "Match"),
+        ("n", "negative", "No match"),
+        ("u", "unsure", "Unsure"),
+        ("d", "delete", "No judgement"),
+    ]
+
+    async def action_positive(self) -> None:
+        await self.trigger_edit(Judgement.POSITIVE)
+
+    async def action_negative(self) -> None:
+        await self.trigger_edit(Judgement.NEGATIVE)
+
+    async def action_unsure(self) -> None:
+        await self.trigger_edit(Judgement.UNSURE)
+
+    async def action_delete(self) -> None:
+        await self.trigger_edit(Judgement.NO_JUDGEMENT)
+
+    async def trigger_edit(self, judgement: Judgement) -> None:
+        selected = self.highlighted_child
+        if selected is None:
+            return
+        edge = selected.query_one(HistoryItem).edge
+        await cast(DedupeApp, self.app).edit(edge, judgement)
+
+
+class HistoryWidget(DedupeAppWidget):
+    list_view: ListView
+    is_visible: bool = False
+
+    def on_mount(self) -> None:
+        self.border_title = "History"
+        self._apply_visibility()
+        self.reload_history()
+
+    def compose(self) -> ComposeResult:
+        self.list_view = HistoryListView()
+        yield Static(
+            (
+                "Tab to toggle between dedupe and history.\n"
+                "Arrow up/down to select history to edit."
+            ),
+            classes="help",
+        )
+        yield self.list_view
+
+    def reload_history(self) -> None:
+        if not self.is_visible:
+            return
+        self.list_view.clear()
+        for edge in self.dedupe.resolver.get_judgements(HISTORY_LENGTH):
+            self.list_view.append(ListItem(HistoryItem(edge)))
+        self.list_view.scroll_home(animate=False)
+
+    def toggle_visible(self) -> None:
+        self.is_visible = not self.is_visible
+        self._apply_visibility()
+        self.reload_history()
+
+    def _apply_visibility(self) -> None:
+        if self.is_visible:
+            self.styles.display = "block"
+        else:
+            self.styles.display = "none"
+
+
+class CompareWidget(DedupeAppWidget, can_focus=True):
+    def render(self) -> RenderableType:
+        if self.dedupe.message is not None:
+            return Text(self.dedupe.message, justify="center")
+        if self.dedupe.left and self.dedupe.right:
+            return render_comparison(
+                self.dedupe.view,
+                self.dedupe.left,
+                self.dedupe.right,
+                self.dedupe.score,
+                latinize=self.dedupe.latinize,
+                url_base=self.dedupe.url_base,
+            )
+        return Text("No candidates.", justify="center")
+
+
+class DedupeWidget(Widget):
+    def compose(self) -> ComposeResult:
+        yield VerticalScroll(CompareWidget())
+        yield HistoryWidget()
+
+
+class DedupeApp(App[int]):
+    CSS_PATH = "app.tcss"
+    dedupe: DedupeState
+
+    BINDINGS = [
+        ("x", "positive", "Match"),
+        ("n", "negative", "No match"),
+        ("u", "unsure", "Unsure"),
+        ("l", "latinize", "Latinize"),
+        ("h", "history", "Toggle History"),
+        ("q", "exit_hard", "Quit"),
+    ]
+
+    async def decide(self, judgement: Judgement) -> None:
+        self.dedupe.decide(judgement)
+        self.force_render()
+
+    async def edit(self, edge: Edge, judgement: Judgement) -> None:
+        async def handle_confirmation(confirmed: bool | None) -> None:
+            if confirmed:
+                self.dedupe.edit(edge, judgement)
+                self.force_render()
+            else:
+                self.dedupe.message = "Canceled edit."
+                self.force_render()
+                await asyncio.sleep(1)
+                self.dedupe.message = None
+                self.force_render()
+
+        screen = ConfirmEditModal()
+        screen.edge = edge
+        screen.judgement = judgement
+        self.app.push_screen(screen, handle_confirmation)
+
+    def force_render(self) -> None:
+        self.query_one(CompareWidget).refresh(layout=True)
+        self.query_one(HistoryWidget).reload_history()
+        self.query_one(HistoryWidget).refresh(layout=True)
+
+    async def action_positive(self) -> None:
+        await self.decide(Judgement.POSITIVE)
+
+    async def action_negative(self) -> None:
+        await self.decide(Judgement.NEGATIVE)
+
+    async def action_unsure(self) -> None:
+        await self.decide(Judgement.UNSURE)
+
+    async def action_latinize(self) -> None:
+        self.dedupe.latinize = not self.dedupe.latinize
+        self.force_render()
+
+    async def action_history(self) -> None:
+        self.query_one(HistoryWidget).toggle_visible()
+
+    async def action_exit_hard(self) -> None:
+        self.exit(0)
+
+    def compose(self) -> ComposeResult:
+        self.dedupe.load()
+        yield DedupeWidget()
+        yield Footer()

nomenklatura/tui/app.tcss

@@ -0,0 +1,52 @@
+ConfirmEditModal {
+    align: center middle;
+}
+
+#dialog {
+    grid-size: 2;
+    grid-gutter: 1 2;
+    grid-rows: 1fr 3;
+    padding: 0 1;
+    width: 60;
+    height: 11;
+    border: thick $background 80%;
+    background: $surface;
+}
+
+#question {
+    column-span: 2;
+    height: 1fr;
+    width: 1fr;
+    content-align: center middle;
+}
+
+
+HistoryItem {
+    border: solid white;
+}
+
+HistoryWidget {
+    width: 50;
+    border: solid white;
+}
+
+HistoryWidget > .help {
+    padding: 1
+}
+
+DedupeWidget {
+    layout: horizontal;
+}
+
+CompareWidget:focus {
+    background: #222222;
+}
+
+CompareWidget {
+    width: 1fr;
+    height: auto;
+}
+
+DedupeApp {
+    layout: vertical;
+}

nomenklatura/tui/comparison.py

@@ -0,0 +1,81 @@
+from typing import Optional, Union
+from normality import latinize_text
+from rich.table import Table
+from rich.text import Text
+from followthemoney import DS, registry, Property
+from followthemoney import SE, StatementEntity as Entity
+
+from nomenklatura.store import View
+from nomenklatura.tui.util import comparison_props
+
+
+def render_column(entity: Entity) -> Text:
+    return Text.assemble(
+        (entity.schema.label, "blue"), " [%s]" % entity.id, no_wrap=True
+    )
+
+
+def render_values(
+    view: View[DS, SE], prop: Property, entity: SE, other: SE, latinize: bool
+) -> Text:
+    values = entity.get(prop, quiet=True)
+    other_values = other.get_type_values(prop.type)
+    text = Text()
+    for i, value in enumerate(sorted(values)):
+        caption = prop.type.caption(value)
+        if prop.type == registry.entity:
+            sub = view.get_entity(value)
+            if sub is not None:
+                caption = sub.caption
+        score = prop.type.compare_sets([value], other_values)
+        if latinize:
+            caption = latinize_text(caption) or caption
+        if prop.name == "wikidataId":
+            caption = f"https://wikidata.org/wiki/{value}"
+        style = "default"
+        if score > 0.7:
+            style = "orange1"
+        if score > 0.95:
+            style = "green1"
+        if caption is not None:
+            if i > 0:
+                text.append(" · ", "gray")
+            text.append(caption, style)
+    return text
+
+
+def render_comparison(
+    view: View[DS, SE],
+    left: SE,
+    right: SE,
+    score: float,
+    latinize: bool = False,
+    url_base: Optional[str] = None,
+) -> Union[Table, Text]:
+    if left is None or right is None:
+        return Text("No candidates loaded.", justify="center")
+
+    table = Table(expand=True)
+    score_text = "Score: %.3f" % score
+    table.add_column(score_text, justify="right", no_wrap=True, ratio=2)
+    table.add_column(render_column(left), ratio=5)
+    table.add_column(render_column(right), ratio=5)
+
+    for prop in comparison_props(left, right):
+        label = Text(prop.label, "white bold")
+        left_text = render_values(view, prop, left, right, latinize)
+        right_text = render_values(view, prop, right, left, latinize)
+        table.add_row(label, left_text, right_text)
+
+    ds_label = Text("Sources", "grey bold")
+    ds_left = Text(", ".join(left.datasets))
+    ds_right = Text(", ".join(right.datasets))
+    table.add_row(ds_label, ds_left, ds_right)
+
+    if url_base is not None:
+        ds_label = Text("URL", "grey bold")
+        ds_left = Text(url_base % left.id)
+        ds_right = Text(url_base % right.id)
+        table.add_row(ds_label, ds_left, ds_right)
+
+    return table

nomenklatura/tui/util.py

@@ -0,0 +1,35 @@
+from typing import Generator, Tuple
+from followthemoney import registry, Property, SE
+
+TYPE_ORDER = {
+    registry.name: -6,
+    registry.identifier: -5,
+    registry.date: -4,
+    registry.country: -3,
+    registry.string: -1,
+    registry.text: 3,
+}
+
+
+def comparison_props(left: SE, right: SE) -> Generator[Property, None, None]:
+    """Return an ordered list of properties to be shown in a comparison of
+    the two given entities."""
+    props = set(left.iterprops())
+    props.update(right.iterprops())
+    weights = {p.name: TYPE_ORDER.get(p.type, 0) for p in props}
+    for prop in props:
+        for schema in (left.schema, right.schema):
+            if prop.name in schema.featured:
+                weights[prop.name] -= 10
+
+    def sort_props(prop: Property) -> Tuple[int, str]:
+        return (weights[prop.name], prop.label)
+
+    for prop in sorted(props, key=sort_props):
+        if prop.hidden:
+            continue
+        if prop.type.matchable and not prop.matchable:
+            continue
+        # if prop.type == registry.entity:
+        #     continue
+        yield prop

nomenklatura/util.py

@@ -0,0 +1,26 @@
+import re
+import os
+from pathlib import Path
+from collections.abc import Mapping
+from typing import Iterable, TypeVar, List, Union, Optional
+from rigour.time import iso_datetime
+
+T = TypeVar("T")
+DATA_PATH = Path(os.path.join(os.path.dirname(__file__), "data")).resolve()
+ID_CLEAN = re.compile(r"[^A-Z0-9]+", re.UNICODE)
+HeadersType = Optional[Mapping[str, Union[str, bytes, None]]]
+
+
+def iso_to_version(value: str) -> Optional[str]:
+    ## Phase this out - it won't be used in new FtM metadata, is used by yente
+    dt = iso_datetime(value)
+    if dt is not None:
+        return dt.strftime("%Y%m%d%H%M%S")
+    return None
+
+
+def unroll(values: Iterable[Iterable[T]]) -> List[T]:
+    unrolled: List[T] = []
+    for sub in values:
+        unrolled.extend(sub)
+    return unrolled

nomenklatura/versions.py

@@ -0,0 +1,119 @@
+import os
+import json
+import string
+import random
+from rigour.time import utc_now
+from typing import Any, List, Iterator, Optional
+from datetime import datetime
+
+ALPHABET = string.ascii_lowercase
+
+
+class Version(object):
+    """A class to represent a dataset version, which consists of a timestamp
+    and a string tag."""
+
+    __slots__ = ["dt", "tag"]
+
+    def __init__(self, dt: datetime, tag: str) -> None:
+        self.dt: datetime = dt
+        self.tag: str = tag
+
+    @classmethod
+    def new(cls, tag: Optional[str] = None) -> "Version":
+        now = utc_now().replace(tzinfo=None)
+
+        if tag is None:
+            # This keeps the tag sortable but short.
+            tag_num = (now.microsecond // 1000) * 10
+            tag_num_ = tag_num + random.randint(0, 9)
+            tag = cls._tag_encode(int(tag_num_))
+
+        tag = tag.ljust(3, "x")[:3]
+        now = now.replace(microsecond=0)
+        return cls(now, tag)
+
+    @classmethod
+    def from_string(cls, id: str) -> "Version":
+        if "-" not in id:
+            raise ValueError(f"Invalid dataset version: {id}")
+        ts, tag = id.split("-", 1)
+        dt = datetime.strptime(ts, "%Y%m%d%H%M%S")
+        dt = dt.replace(tzinfo=None)
+        return cls(dt, tag)
+
+    @classmethod
+    def _tag_encode(cls, number: int, alphabet: str = ALPHABET) -> str:
+        """Converts an integer to a base36 string."""
+        assert number >= 0, "number must be positive"
+        if 0 <= number < len(alphabet):
+            return alphabet[number]
+
+        encoded = ""
+        while number != 0:
+            number, i = divmod(number, len(alphabet))
+            encoded = alphabet[i] + encoded
+        return encoded
+
+    @classmethod
+    def from_env(cls, name: str) -> "Version":
+        id = os.environ.get(name)
+        if id is None:
+            return cls.new()
+        return cls.from_string(id)
+
+    @property
+    def id(self) -> str:
+        return f"{self.dt.strftime('%Y%m%d%H%M%S')}-{self.tag}"
+
+    def __str__(self) -> str:
+        return self.id
+
+    def __repr__(self) -> str:
+        return f"Version({self.id})"
+
+    def __eq__(self, other: Any) -> bool:
+        return self.id == str(other)
+
+    def __hash__(self) -> int:
+        return hash(self.id)
+
+
+class VersionHistory(object):
+    """A class to represent a history of dataset versions."""
+
+    LENGTH = 100
+
+    def __init__(self, items: List[Version], max_length: int = LENGTH) -> None:
+        self.items = items
+        self.max_length = max_length
+
+    def append(self, version: Version) -> "VersionHistory":
+        """Creates a new history with the given RunID appended."""
+        items = list(self.items)
+        items.append(version)
+        return VersionHistory(items[-self.max_length :])
+
+    @property
+    def latest(self) -> Optional[Version]:
+        if not len(self.items):
+            return None
+        return self.items[-1]
+
+    def to_json(self) -> str:
+        """Return a JSON representation of the version history."""
+        items = [str(run) for run in self.items[-self.LENGTH :]]
+        return json.dumps({"items": items})
+
+    @classmethod
+    def from_json(cls, data: str) -> "VersionHistory":
+        """Create a run history from a JSON representation."""
+        items = json.loads(data).get("items", [])
+        items = [Version.from_string(item) for item in items]
+        return cls(items)
+
+    def __iter__(self) -> Iterator[Version]:
+        return iter(self.items)
+
+    def __len__(self) -> int:
+        return len(self.items)

nomenklatura/wikidata/__init__.py

@@ -0,0 +1,14 @@
+from nomenklatura.wikidata.client import WikidataClient
+from nomenklatura.wikidata.lang import LangText
+from nomenklatura.wikidata.model import Item, Claim
+from nomenklatura.wikidata.query import SparqlBinding, SparqlResponse, SparqlValue
+
+__all__ = [
+    "WikidataClient",
+    "LangText",
+    "Item",
+    "Claim",
+    "SparqlBinding",
+    "SparqlResponse",
+    "SparqlValue",
+]