libex-core 0.26.0__tar.gz → 0.27.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {libex_core-0.26.0 → libex_core-0.27.0}/PKG-INFO +1 -1
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/CHANGELOG.md +16 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/__init__.py +1 -1
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/models.py +7 -0
- libex_core-0.27.0/libex_core/storage/migrations/versions/c8d2a5e71f39_folded_search_indexes.py +253 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/models.py +109 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/people.py +57 -59
- libex_core-0.27.0/libex_core/storage/read/search.py +465 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/series.py +16 -23
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/shapes.py +16 -16
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/upgrade.py +6 -2
- {libex_core-0.26.0 → libex_core-0.27.0}/LICENSE +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/PYPI.md +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex-core-data/share/bash-completion/completions/libex-core +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex-core-data/share/fish/vendor_completions.d/libex-core.fish +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex-core-data/share/man/man1/libex-core.1 +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex-core-data/share/zsh/site-functions/_libex-core +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/__main__.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/asin.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/__init__.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/_concurrency.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/_retry.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/authors/__init__.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/authors/by_name.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/authors/catalog.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/authors/profile.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/authors/screens.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/books.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/chapters.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/client.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/extras.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/releases.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/search.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/series.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/__init__.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/_args.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/_db_args.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/_render.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/_run.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/_shaping.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/__init__.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/abs_search.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/author.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/book.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/completion.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/config.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/db.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/narrator.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/releases.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/search.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/series.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/environment.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/exit_codes.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/main.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/output.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/parser.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/session.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/store_state.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/exceptions.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/log_safety.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/__init__.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/_common.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/_shaping.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/_store.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/_walks.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/author_books.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/authors.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/books.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/ingest.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/releases.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/search.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/series.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/stored.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/py.typed +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/shaping.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/__init__.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/base.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/dialect.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/filtering.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/merge.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/env.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/script.py.mako +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/versions/438dbe70d041_create_core_tables.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/versions/6e1b4c9a2d57_add_credit_order_position_to_pivots.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/versions/9d2f6b4e8a17_index_book_series_by_series.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/versions/a4d91f7c3b26_key_books_and_series_by_asin_and_region.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/versions/b5e9c3a7f184_trigram_indexes_for_the_search_filters.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/versions/d3a8f1b65e90_add_manual_entry_at_to_books.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/versions/f2b8d6a40c17_index_books_by_manual_entry.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/__init__.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/_compat.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/_regions.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/books.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/stats.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/walks.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/sorting.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/store.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/types.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/walk_limits.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/__init__.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/books.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/entities.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/params.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/serialize.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/statements.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/support.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/walks.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/text.py +0 -0
- {libex_core-0.26.0 → libex_core-0.27.0}/pyproject.toml +0 -0
|
@@ -13,6 +13,22 @@ the first published version. Entries for earlier versions record changes made
|
|
|
13
13
|
before publication: they are historical record for whoever embeds this package,
|
|
14
14
|
not evidence that anyone consumed a given version at the time it was cut.
|
|
15
15
|
|
|
16
|
+
## [0.27.0]
|
|
17
|
+
|
|
18
|
+
This release adds a migration and a new requirement for Postgres stores: the `unaccent` extension, in the `public` schema, next to `pg_trgm`, which must now be in `public` too. Run `alembic upgrade head` on any store you already have. A SQLite store is not affected; the revision does nothing there, and the new readers do not run on it.
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
- **Ranked search readers for books, authors and series:** `search_books_ranked`, `search_authors_ranked` and `search_series_ranked` in `libex_core.storage.read.search`. They match ignoring case and accents, tolerate typos, and return the same dicts as the single lookups, best match first. They are Postgres only: on any other dialect they raise `NotImplementedError`. `limit` is 1 to 50 and anything else raises `ValueError`. Authors and series also take `exact` for a whole-name match.
|
|
22
|
+
- **`merge_author_rows`** in `libex_core.storage.read.people`, which turns every stored row of one author ASIN and region into the single record `get_author` returns. `get_author` now calls it and behaves as before; `search_authors_ranked` uses it, so a search result is the record `get_author` gives. It takes a non-empty list of `Author` rows ordered by id, and is not re-exported from a package root.
|
|
23
|
+
- **`SeriesSearchResponse`,** a `SeriesResponse` with `bookCount`, the number of stored books linked to the series in its region.
|
|
24
|
+
- **The migration creates the function `public.libex_fold` and nine indexes on Postgres.** The function lowercases text and removes accents. The indexes are four trigram indexes over it (book title with subtitle, author name, series title, narrator name), two btrees for exact author and series names, two on the order of ratings, and one on `book_narrator.narrator_name`. They are named `ix_books_fold_title_gin`, `ix_authors_fold_name_gin`, `ix_series_fold_title_gin`, `ix_narrators_fold_name_gin`, `ix_authors_fold_name_bt`, `ix_series_fold_title_bt`, `ix_books_fold_pop`, `ix_books_fold_region_pop` and `ix_book_narrator_fold_narrator_name`. The models declare them for Postgres only, and creating the tables from the models creates the function first; `unaccent` must already be installed.
|
|
25
|
+
- **The migration needs `unaccent` in `public`.** If it is missing, the migration creates it, but only when the role can: the server must offer it and the role needs `CREATE` on the database and on `public`. `pg_trgm` must already be in `public`. That check is made before anything is changed, and a role that cannot get there gets an error giving the command to run, with the transaction rolled back and no revision stamped. An existing `libex_fold` or index of those names is checked against the expected definition and never replaced; any difference raises. This chain upgrades in one transaction, so the indexes cannot be built concurrently: while each is built, reads of its table carry on and writes to it wait. Downgrading drops the indexes and the function and keeps both extensions.
|
|
26
|
+
|
|
27
|
+
### Changed
|
|
28
|
+
- **`hydrate_books` reads series positions for all its books in one batch,** where it made one query per book. The result and its order are the same.
|
|
29
|
+
- **Comparing the models with a database no longer reports the folded indexes.** Alembic cannot compare an expression index, so they are skipped on both sides and the migrations own them.
|
|
30
|
+
- **The indexes cost disk and write volume,** on top of those from 0.26.0.
|
|
31
|
+
|
|
16
32
|
## [0.26.0]
|
|
17
33
|
|
|
18
34
|
This release adds a migration and a new requirement for Postgres stores: the `pg_trgm` extension. Run `alembic upgrade head` on any store you already have. A SQLite store is not affected; the revision does nothing there.
|
|
@@ -607,6 +607,13 @@ class SeriesResponse(BaseModel):
|
|
|
607
607
|
)
|
|
608
608
|
|
|
609
609
|
|
|
610
|
+
class SeriesSearchResponse(SeriesResponse):
|
|
611
|
+
bookCount: int = Field(
|
|
612
|
+
default=0,
|
|
613
|
+
description="How many stored books are linked to this series, in its region.",
|
|
614
|
+
)
|
|
615
|
+
|
|
616
|
+
|
|
610
617
|
# ============================================================
|
|
611
618
|
# CATEGORIES
|
|
612
619
|
# ============================================================
|
libex_core-0.27.0/libex_core/storage/migrations/versions/c8d2a5e71f39_folded_search_indexes.py
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
"""folded search indexes for the ranked search
|
|
2
|
+
|
|
3
|
+
Revision ID: c8d2a5e71f39
|
|
4
|
+
Revises: b5e9c3a7f184
|
|
5
|
+
Create Date: 2026-11-06
|
|
6
|
+
|
|
7
|
+
The same objects, under the same names and definitions, as the hosted chain's
|
|
8
|
+
e6a1d4f93b27 (the chains must build one schema): the unaccent extension, the
|
|
9
|
+
function public.libex_fold that lowercases and removes accents, trigram indexes
|
|
10
|
+
over the folded book title and subtitle, author name, series title and narrator
|
|
11
|
+
name, btrees over the folded author name and series title for exact matches, the
|
|
12
|
+
two popularity orders on books, and a btree on book_narrator.narrator_name. The
|
|
13
|
+
raw-column indexes from b5e9c3a7f184 are left as they are. Postgres only: the
|
|
14
|
+
revision does nothing on SQLite.
|
|
15
|
+
|
|
16
|
+
This chain runs in one transaction under an advisory lock, so the builds cannot
|
|
17
|
+
be CONCURRENTLY: each CREATE INDEX takes a SHARE lock on its table for the
|
|
18
|
+
length of the build, and reads carry on while writes to that table wait.
|
|
19
|
+
|
|
20
|
+
pg_trgm and unaccent must both live in the public schema, where the function and
|
|
21
|
+
the search queries name them. What the role cannot do is checked first and
|
|
22
|
+
raises with the command a privileged role must run, so the transaction rolls
|
|
23
|
+
back with no revision stamped. Run the function's command as the role that owns
|
|
24
|
+
the tables: an existing public.libex_fold is checked against the definition below,
|
|
25
|
+
owner included, and is never replaced.
|
|
26
|
+
|
|
27
|
+
Downgrade drops the indexes and the function and keeps both extensions, which
|
|
28
|
+
other objects may depend on and which dropping needs ownership of. Nothing here
|
|
29
|
+
uses CASCADE.
|
|
30
|
+
"""
|
|
31
|
+
from typing import Sequence, Union
|
|
32
|
+
|
|
33
|
+
from alembic import op
|
|
34
|
+
from sqlalchemy import text
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
revision: str = 'c8d2a5e71f39'
|
|
38
|
+
down_revision: Union[str, Sequence[str], None] = 'b5e9c3a7f184'
|
|
39
|
+
branch_labels: Union[str, Sequence[str], None] = None
|
|
40
|
+
depends_on: Union[str, Sequence[str], None] = None
|
|
41
|
+
|
|
42
|
+
# The body is compared character for character against an existing function.
|
|
43
|
+
# Copies live in libex_core's models and in the hosted chain's revision, and a
|
|
44
|
+
# test holds them identical.
|
|
45
|
+
FOLD_FUNCTION_BODY = "SELECT pg_catalog.lower(public.unaccent('public.unaccent'::regdictionary, $1))"
|
|
46
|
+
FOLD_FUNCTION_DDL = (
|
|
47
|
+
"CREATE FUNCTION public.libex_fold(text) RETURNS text LANGUAGE sql "
|
|
48
|
+
f"IMMUTABLE STRICT PARALLEL SAFE AS $${FOLD_FUNCTION_BODY}$$"
|
|
49
|
+
)
|
|
50
|
+
EXTENSION_DDL = 'CREATE EXTENSION IF NOT EXISTS unaccent WITH SCHEMA public'
|
|
51
|
+
# True when public.unaccent and its dictionary are the extension's own.
|
|
52
|
+
UNACCENT_IS_THE_EXTENSION_SQL = (
|
|
53
|
+
"SELECT EXISTS (SELECT 1 FROM pg_ts_dict WHERE dictname = 'unaccent' AND dictnamespace = 'public'::regnamespace) AND EXISTS ("
|
|
54
|
+
"SELECT 1 FROM pg_depend d JOIN pg_proc p ON p.oid = d.objid "
|
|
55
|
+
"WHERE d.classid = 'pg_proc'::regclass AND d.refclassid = 'pg_extension'::regclass "
|
|
56
|
+
"AND d.deptype = 'e' AND d.refobjid = (SELECT oid FROM pg_extension WHERE extname = 'unaccent') "
|
|
57
|
+
"AND p.proname = 'unaccent' AND p.pronamespace = 'public'::regnamespace)"
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
# (index, table, what follows "ON <table> ", the definition Postgres reports for
|
|
61
|
+
# it with the public schema qualifier removed)
|
|
62
|
+
INDEXES = (
|
|
63
|
+
(
|
|
64
|
+
'ix_books_fold_title_gin', 'books',
|
|
65
|
+
"USING gin (public.libex_fold(title || ' ' || coalesce(subtitle, '')) public.gin_trgm_ops)",
|
|
66
|
+
"CREATE INDEX ix_books_fold_title_gin ON books USING gin "
|
|
67
|
+
"(libex_fold(((title || ' '::text) || COALESCE(subtitle, ''::text))) gin_trgm_ops)",
|
|
68
|
+
),
|
|
69
|
+
(
|
|
70
|
+
'ix_authors_fold_name_gin', 'authors',
|
|
71
|
+
'USING gin (public.libex_fold(name) public.gin_trgm_ops)',
|
|
72
|
+
'CREATE INDEX ix_authors_fold_name_gin ON authors USING gin (libex_fold(name) gin_trgm_ops)',
|
|
73
|
+
),
|
|
74
|
+
(
|
|
75
|
+
'ix_series_fold_title_gin', 'series',
|
|
76
|
+
'USING gin (public.libex_fold(title) public.gin_trgm_ops)',
|
|
77
|
+
'CREATE INDEX ix_series_fold_title_gin ON series USING gin (libex_fold(title) gin_trgm_ops)',
|
|
78
|
+
),
|
|
79
|
+
(
|
|
80
|
+
'ix_narrators_fold_name_gin', 'narrators',
|
|
81
|
+
'USING gin (public.libex_fold(name) public.gin_trgm_ops)',
|
|
82
|
+
'CREATE INDEX ix_narrators_fold_name_gin ON narrators USING gin (libex_fold(name) gin_trgm_ops)',
|
|
83
|
+
),
|
|
84
|
+
(
|
|
85
|
+
'ix_authors_fold_name_bt', 'authors',
|
|
86
|
+
'(public.libex_fold(name))',
|
|
87
|
+
'CREATE INDEX ix_authors_fold_name_bt ON authors USING btree (libex_fold(name))',
|
|
88
|
+
),
|
|
89
|
+
(
|
|
90
|
+
'ix_series_fold_title_bt', 'series',
|
|
91
|
+
'(public.libex_fold(title))',
|
|
92
|
+
'CREATE INDEX ix_series_fold_title_bt ON series USING btree (libex_fold(title))',
|
|
93
|
+
),
|
|
94
|
+
(
|
|
95
|
+
'ix_books_fold_pop', 'books',
|
|
96
|
+
'(num_ratings DESC NULLS LAST, asin, region) WHERE is_primary',
|
|
97
|
+
'CREATE INDEX ix_books_fold_pop ON books USING btree '
|
|
98
|
+
'(num_ratings DESC NULLS LAST, asin, region) WHERE is_primary',
|
|
99
|
+
),
|
|
100
|
+
(
|
|
101
|
+
'ix_books_fold_region_pop', 'books',
|
|
102
|
+
'(region, num_ratings DESC NULLS LAST, asin)',
|
|
103
|
+
'CREATE INDEX ix_books_fold_region_pop ON books USING btree '
|
|
104
|
+
'(region, num_ratings DESC NULLS LAST, asin)',
|
|
105
|
+
),
|
|
106
|
+
(
|
|
107
|
+
'ix_book_narrator_fold_narrator_name', 'book_narrator',
|
|
108
|
+
'(narrator_name)',
|
|
109
|
+
'CREATE INDEX ix_book_narrator_fold_narrator_name ON book_narrator USING btree (narrator_name)',
|
|
110
|
+
),
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
# The tables the indexes sit on.
|
|
114
|
+
ANALYZED_TABLES = ('books', 'authors', 'series', 'narrators', 'book_narrator')
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _scalar(sql: str):
|
|
118
|
+
return op.get_bind().execute(text(sql)).scalar()
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _extension_schema(name: str):
|
|
122
|
+
return _scalar(
|
|
123
|
+
"SELECT n.nspname FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace "
|
|
124
|
+
f"WHERE e.extname = '{name}'"
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _check_extensions() -> None:
|
|
129
|
+
"""pg_trgm and unaccent installed in public, unaccent created if the role
|
|
130
|
+
can and raising with the command if it cannot."""
|
|
131
|
+
trgm = _extension_schema('pg_trgm')
|
|
132
|
+
if trgm != 'public':
|
|
133
|
+
raise RuntimeError(
|
|
134
|
+
f'the search indexes use public.gin_trgm_ops, but pg_trgm is {"not installed" if trgm is None else f"installed in the {trgm} schema"}'
|
|
135
|
+
)
|
|
136
|
+
installed = _extension_schema('unaccent')
|
|
137
|
+
if installed not in (None, 'public'):
|
|
138
|
+
raise RuntimeError(
|
|
139
|
+
f'unaccent is installed in the {installed} schema; public.libex_fold calls public.unaccent'
|
|
140
|
+
)
|
|
141
|
+
if installed is None:
|
|
142
|
+
can_create = _scalar(
|
|
143
|
+
"SELECT EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = 'unaccent') "
|
|
144
|
+
"AND has_database_privilege(current_user, current_database(), 'CREATE') "
|
|
145
|
+
"AND has_schema_privilege(current_user, 'public', 'CREATE')"
|
|
146
|
+
)
|
|
147
|
+
if not can_create:
|
|
148
|
+
raise RuntimeError(
|
|
149
|
+
'the search indexes need the unaccent extension, which is not installed and '
|
|
150
|
+
'cannot be created by this role; a privileged role must run: ' + EXTENSION_DDL
|
|
151
|
+
)
|
|
152
|
+
op.execute(EXTENSION_DDL)
|
|
153
|
+
ok = _scalar(UNACCENT_IS_THE_EXTENSION_SQL)
|
|
154
|
+
if not ok:
|
|
155
|
+
raise RuntimeError(
|
|
156
|
+
'public.unaccent is not provided by the unaccent extension; public.libex_fold needs it'
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _fold_function_problems() -> list[str]:
|
|
161
|
+
"""What differs between the existing libex_fold and the definition above.
|
|
162
|
+
Empty when there is exactly one, and it matches."""
|
|
163
|
+
functions = op.get_bind().execute(text(
|
|
164
|
+
"SELECT p.prosrc, l.lanname, p.provolatile::text, p.proisstrict, p.proparallel::text, "
|
|
165
|
+
"p.prorettype::regtype::text, p.prosecdef, p.proconfig IS NOT NULL, p.proretset, "
|
|
166
|
+
"p.pronamespace::regnamespace::text, pg_get_function_identity_arguments(p.oid), "
|
|
167
|
+
"p.proowner = (SELECT relowner FROM pg_class WHERE oid = 'public.books'::regclass) "
|
|
168
|
+
"FROM pg_proc p JOIN pg_language l ON l.oid = p.prolang WHERE p.proname = 'libex_fold'"
|
|
169
|
+
)).all()
|
|
170
|
+
if len(functions) != 1:
|
|
171
|
+
return [f'{len(functions)} functions named libex_fold exist, expected exactly one']
|
|
172
|
+
(body, lang, volatile, strict, parallel, returns, definer, has_config, is_set,
|
|
173
|
+
namespace, args, same_owner) = functions[0]
|
|
174
|
+
expected = {
|
|
175
|
+
'body': (body, FOLD_FUNCTION_BODY),
|
|
176
|
+
'language': (lang, 'sql'),
|
|
177
|
+
'volatility': (volatile, 'i'),
|
|
178
|
+
'strict': (strict, True),
|
|
179
|
+
'parallel': (parallel, 's'),
|
|
180
|
+
'return type': (returns, 'text'),
|
|
181
|
+
'security definer': (definer, False),
|
|
182
|
+
'a SET clause': (has_config, False),
|
|
183
|
+
'returns a set': (is_set, False),
|
|
184
|
+
'schema': (namespace, 'public'),
|
|
185
|
+
'arguments': (args, 'text'),
|
|
186
|
+
"the owner being books' owner": (same_owner, True),
|
|
187
|
+
}
|
|
188
|
+
return [f'{what}: found {found!r}, expected {want!r}' for what, (found, want) in expected.items() if found != want]
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _ensure_fold_function() -> None:
|
|
192
|
+
if _scalar("SELECT count(*) FROM pg_proc WHERE proname = 'libex_fold'") == 0:
|
|
193
|
+
if not _scalar("SELECT has_schema_privilege(current_user, 'public', 'CREATE')"):
|
|
194
|
+
raise RuntimeError(
|
|
195
|
+
'the search indexes need the function public.libex_fold, which cannot be created '
|
|
196
|
+
'by this role; the role that owns the tables must run: ' + FOLD_FUNCTION_DDL
|
|
197
|
+
)
|
|
198
|
+
op.execute(FOLD_FUNCTION_DDL)
|
|
199
|
+
return
|
|
200
|
+
problems = _fold_function_problems()
|
|
201
|
+
if problems:
|
|
202
|
+
raise RuntimeError(
|
|
203
|
+
'an existing libex_fold does not match the definition this revision expects, and is '
|
|
204
|
+
'not replaced: ' + '; '.join(problems)
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _index_definition(name: str):
|
|
209
|
+
"""The definition of an existing index with the public qualifier removed,
|
|
210
|
+
or None when there is none."""
|
|
211
|
+
row = op.get_bind().execute(
|
|
212
|
+
text('SELECT pg_get_indexdef(indexrelid) FROM pg_index WHERE indexrelid = to_regclass(:name)'),
|
|
213
|
+
{'name': name},
|
|
214
|
+
).first()
|
|
215
|
+
return None if row is None else row[0].replace('public.', '')
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _index_is_usable(name: str) -> bool:
|
|
219
|
+
"""False for an index a failed build left behind (not valid or not ready)."""
|
|
220
|
+
return bool(_scalar(
|
|
221
|
+
f"SELECT indisvalid AND indisready FROM pg_index WHERE indexrelid = to_regclass('{name}')"
|
|
222
|
+
))
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def upgrade() -> None:
|
|
226
|
+
if op.get_bind().dialect.name != 'postgresql':
|
|
227
|
+
return
|
|
228
|
+
_check_extensions()
|
|
229
|
+
_ensure_fold_function()
|
|
230
|
+
for name, table, tail, expected in INDEXES:
|
|
231
|
+
found = _index_definition(name)
|
|
232
|
+
if found is None:
|
|
233
|
+
op.execute(f'CREATE INDEX {name} ON {table} {tail}')
|
|
234
|
+
elif found != expected:
|
|
235
|
+
raise RuntimeError(
|
|
236
|
+
f'{name} already exists with a different definition and is not replaced: '
|
|
237
|
+
f'found {found!r}, expected {expected!r}'
|
|
238
|
+
)
|
|
239
|
+
elif not _index_is_usable(name):
|
|
240
|
+
raise RuntimeError(
|
|
241
|
+
f'{name} already exists but is not valid and ready, as a failed build leaves it; '
|
|
242
|
+
'drop it and run the upgrade again'
|
|
243
|
+
)
|
|
244
|
+
for table in ANALYZED_TABLES:
|
|
245
|
+
op.execute(f'ANALYZE {table}')
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def downgrade() -> None:
|
|
249
|
+
if op.get_bind().dialect.name != 'postgresql':
|
|
250
|
+
return
|
|
251
|
+
for name, _table, _tail, _expected in INDEXES:
|
|
252
|
+
op.execute(f'DROP INDEX IF EXISTS {name}')
|
|
253
|
+
op.execute('DROP FUNCTION IF EXISTS public.libex_fold(text)')
|
|
@@ -16,6 +16,7 @@ from sqlalchemy import (
|
|
|
16
16
|
Boolean,
|
|
17
17
|
CheckConstraint,
|
|
18
18
|
Column,
|
|
19
|
+
DDL,
|
|
19
20
|
Double,
|
|
20
21
|
Enum,
|
|
21
22
|
ForeignKey,
|
|
@@ -26,6 +27,9 @@ from sqlalchemy import (
|
|
|
26
27
|
Table,
|
|
27
28
|
Text,
|
|
28
29
|
UniqueConstraint,
|
|
30
|
+
event,
|
|
31
|
+
func,
|
|
32
|
+
literal_column,
|
|
29
33
|
text,
|
|
30
34
|
true,
|
|
31
35
|
)
|
|
@@ -428,6 +432,105 @@ for _name, _column in (
|
|
|
428
432
|
).ddl_if(dialect="postgresql")
|
|
429
433
|
|
|
430
434
|
|
|
435
|
+
# ============================================================
|
|
436
|
+
# FOLDED SEARCH INDEXES
|
|
437
|
+
# ============================================================
|
|
438
|
+
# What the ranked /db search reads: the same text lowercased and with accents
|
|
439
|
+
# removed by public.libex_fold, indexed for trigram matching, plus the btrees
|
|
440
|
+
# behind exact matching and the popularity order. Postgres only; the migrations
|
|
441
|
+
# (e6a1d4f93b27 hosted, c8d2a5e71f39 here) build the same set under the same
|
|
442
|
+
# names. The separators in the title expression are SQL literals, not bound
|
|
443
|
+
# values: a bound separator makes the expression something a generic plan cannot
|
|
444
|
+
# match to the index. concat_ws is avoided because it is STABLE and cannot be
|
|
445
|
+
# indexed.
|
|
446
|
+
|
|
447
|
+
# The text of public.libex_fold. Each migration keeps its own copy, and a test
|
|
448
|
+
# holds them identical: an existing function's body is compared character for
|
|
449
|
+
# character, so it must not drift.
|
|
450
|
+
FOLD_FUNCTION_BODY = "SELECT pg_catalog.lower(public.unaccent('public.unaccent'::regdictionary, $1))"
|
|
451
|
+
FOLD_FUNCTION_DDL = (
|
|
452
|
+
"CREATE FUNCTION public.libex_fold(text) RETURNS text LANGUAGE sql "
|
|
453
|
+
f"IMMUTABLE STRICT PARALLEL SAFE AS $${FOLD_FUNCTION_BODY}$$"
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
# Every index this section declares. include_object_for keeps them out of a
|
|
457
|
+
# compare with a database: alembic cannot compare an expression index, and the
|
|
458
|
+
# migrations own them.
|
|
459
|
+
FOLDED_INDEX_NAMES = frozenset({
|
|
460
|
+
"ix_books_fold_title_gin",
|
|
461
|
+
"ix_authors_fold_name_gin",
|
|
462
|
+
"ix_series_fold_title_gin",
|
|
463
|
+
"ix_narrators_fold_name_gin",
|
|
464
|
+
"ix_authors_fold_name_bt",
|
|
465
|
+
"ix_series_fold_title_bt",
|
|
466
|
+
"ix_books_fold_pop",
|
|
467
|
+
"ix_books_fold_region_pop",
|
|
468
|
+
"ix_book_narrator_fold_narrator_name",
|
|
469
|
+
})
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _fold(expression):
|
|
473
|
+
return func.public.libex_fold(expression)
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
_BOOK_FOLDED_TEXT = _fold(
|
|
477
|
+
Book.title + literal_column("' '") + func.coalesce(Book.subtitle, literal_column("''"))
|
|
478
|
+
)
|
|
479
|
+
|
|
480
|
+
for _name, _expression in (
|
|
481
|
+
("ix_books_fold_title_gin", _BOOK_FOLDED_TEXT),
|
|
482
|
+
("ix_authors_fold_name_gin", _fold(Author.name)),
|
|
483
|
+
("ix_series_fold_title_gin", _fold(Series.title)),
|
|
484
|
+
("ix_narrators_fold_name_gin", _fold(Narrator.name)),
|
|
485
|
+
):
|
|
486
|
+
Index(
|
|
487
|
+
_name,
|
|
488
|
+
_expression,
|
|
489
|
+
postgresql_using="gin",
|
|
490
|
+
postgresql_ops={_expression.key: "public.gin_trgm_ops"},
|
|
491
|
+
).ddl_if(dialect="postgresql")
|
|
492
|
+
|
|
493
|
+
Index("ix_authors_fold_name_bt", _fold(Author.name)).ddl_if(dialect="postgresql")
|
|
494
|
+
Index("ix_series_fold_title_bt", _fold(Series.title)).ddl_if(dialect="postgresql")
|
|
495
|
+
|
|
496
|
+
# The popularity order, for the shortcut that takes the most popular matches
|
|
497
|
+
# first: one index for a search over every region's primary rows, one for a
|
|
498
|
+
# search inside a region.
|
|
499
|
+
Index(
|
|
500
|
+
"ix_books_fold_pop",
|
|
501
|
+
Book.num_ratings.desc().nulls_last(),
|
|
502
|
+
Book.asin,
|
|
503
|
+
Book.region,
|
|
504
|
+
postgresql_where=text("is_primary"),
|
|
505
|
+
).ddl_if(dialect="postgresql")
|
|
506
|
+
Index(
|
|
507
|
+
"ix_books_fold_region_pop",
|
|
508
|
+
Book.region,
|
|
509
|
+
Book.num_ratings.desc().nulls_last(),
|
|
510
|
+
Book.asin,
|
|
511
|
+
).ddl_if(dialect="postgresql")
|
|
512
|
+
|
|
513
|
+
# The indexes call the function, so it has to exist before they are built. It
|
|
514
|
+
# needs the unaccent extension, which this module does not create: whatever
|
|
515
|
+
# provides pg_trgm provides that too.
|
|
516
|
+
_ENSURE_FOLD_FUNCTION = DDL(
|
|
517
|
+
"DO $do$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_proc p "
|
|
518
|
+
"WHERE p.proname = 'libex_fold' AND p.pronamespace = 'public'::regnamespace "
|
|
519
|
+
"AND p.proargtypes::text = '25') THEN "
|
|
520
|
+
f"{FOLD_FUNCTION_DDL}; END IF; END $do$"
|
|
521
|
+
).execute_if(dialect="postgresql")
|
|
522
|
+
|
|
523
|
+
# Skipped rather than failed while something still uses the function, so a drop
|
|
524
|
+
# of only some of the tables does not abort on the indexes left behind.
|
|
525
|
+
_DROP_FOLD_FUNCTION = DDL(
|
|
526
|
+
"DO $do$ BEGIN DROP FUNCTION IF EXISTS public.libex_fold(text); "
|
|
527
|
+
"EXCEPTION WHEN dependent_objects_still_exist THEN NULL; END $do$"
|
|
528
|
+
).execute_if(dialect="postgresql")
|
|
529
|
+
|
|
530
|
+
event.listen(Base.metadata, "before_create", _ENSURE_FOLD_FUNCTION)
|
|
531
|
+
event.listen(Base.metadata, "after_drop", _DROP_FOLD_FUNCTION)
|
|
532
|
+
|
|
533
|
+
|
|
431
534
|
# ============================================================
|
|
432
535
|
# GENRES
|
|
433
536
|
# ============================================================
|
|
@@ -533,6 +636,12 @@ book_narrator = Table(
|
|
|
533
636
|
UniqueConstraint("book_asin", "book_region", "narrator_name", name="uq_book_narrator"),
|
|
534
637
|
)
|
|
535
638
|
|
|
639
|
+
# Serves the narrator search, which finds the matching narrators first and then
|
|
640
|
+
# their books by narrator_name.
|
|
641
|
+
Index(
|
|
642
|
+
"ix_book_narrator_fold_narrator_name", book_narrator.c.narrator_name
|
|
643
|
+
).ddl_if(dialect="postgresql")
|
|
644
|
+
|
|
536
645
|
book_series = Table(
|
|
537
646
|
"book_series",
|
|
538
647
|
Base.metadata,
|
|
@@ -19,6 +19,7 @@ from libex_core.storage.read._regions import only_first_stored
|
|
|
19
19
|
from libex_core.storage.read.shapes import (
|
|
20
20
|
BOOK_RELATIONS,
|
|
21
21
|
book_to_dict,
|
|
22
|
+
genre_to_dict,
|
|
22
23
|
hydrate_books,
|
|
23
24
|
narrator_to_dict,
|
|
24
25
|
series_positions_by_book,
|
|
@@ -28,6 +29,61 @@ from libex_core.storage.sorting import BOOK_SORT_FIELDS, NARRATOR_SORT_FIELDS, a
|
|
|
28
29
|
logger = logging.getLogger("libex")
|
|
29
30
|
|
|
30
31
|
|
|
32
|
+
def merge_author_rows(authors: list[Author]) -> dict[str, Any]:
|
|
33
|
+
"""One record from every stored row of an (asin, region), by the rules
|
|
34
|
+
get_author documents. `authors` is non-empty and ordered by id."""
|
|
35
|
+
base = authors[0]
|
|
36
|
+
|
|
37
|
+
def _measured_length(value: str | None) -> int:
|
|
38
|
+
"""Trimmed length, floored to -1 for absent — same idea as
|
|
39
|
+
merge.longer_wins' absent-sentinel, applied to whichever text is
|
|
40
|
+
being ranked (description or image)."""
|
|
41
|
+
stripped = value.strip() if value else ""
|
|
42
|
+
return len(stripped) if stripped else -1
|
|
43
|
+
|
|
44
|
+
description = base.description
|
|
45
|
+
best_length = _measured_length(description)
|
|
46
|
+
for a in authors[1:]:
|
|
47
|
+
length = _measured_length(a.description)
|
|
48
|
+
if length > best_length:
|
|
49
|
+
description = a.description
|
|
50
|
+
best_length = length
|
|
51
|
+
|
|
52
|
+
image = next(
|
|
53
|
+
(a.image for a in authors if _measured_length(a.image) >= 0),
|
|
54
|
+
base.image,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
# Defensive against a stored null slipping through despite the column
|
|
58
|
+
# being NOT NULL on every write path today: the `if a.updated_at`
|
|
59
|
+
# filter drops None candidates before max() ever sees them, so an
|
|
60
|
+
# all-None input leaves the generator empty rather than raising
|
|
61
|
+
# TypeError on a bad comparison — but max() on an empty iterable
|
|
62
|
+
# raises ValueError unless a default is given. `default=None` is
|
|
63
|
+
# what closes that: without it a null here would fail the whole
|
|
64
|
+
# read for a record that is otherwise readable.
|
|
65
|
+
updated_at = max((a.updated_at for a in authors if a.updated_at), default=None)
|
|
66
|
+
|
|
67
|
+
genres_by_asin: dict[str, Any] = {}
|
|
68
|
+
for a in authors:
|
|
69
|
+
for g in (a.genres or []):
|
|
70
|
+
genres_by_asin.setdefault(g.asin, g)
|
|
71
|
+
|
|
72
|
+
genres = [genre_to_dict(g) for g in genres_by_asin.values()]
|
|
73
|
+
|
|
74
|
+
return {
|
|
75
|
+
"id": base.id,
|
|
76
|
+
"asin": base.asin,
|
|
77
|
+
"name": base.name,
|
|
78
|
+
"description": description,
|
|
79
|
+
"image": image,
|
|
80
|
+
"region": base.region,
|
|
81
|
+
"regions": [base.region],
|
|
82
|
+
"genres": genres,
|
|
83
|
+
"updatedAt": updated_at.isoformat() if updated_at else None,
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
|
|
31
87
|
async def get_author(session: AsyncSession, asin: str, region: str) -> dict[str, Any] | None:
|
|
32
88
|
"""Fetches an author from the DB with genres.
|
|
33
89
|
|
|
@@ -97,65 +153,7 @@ async def get_author(session: AsyncSession, asin: str, region: str) -> dict[str,
|
|
|
97
153
|
},
|
|
98
154
|
)
|
|
99
155
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
def _measured_length(value: str | None) -> int:
|
|
103
|
-
"""Trimmed length, floored to -1 for absent — same idea as
|
|
104
|
-
merge.longer_wins' absent-sentinel, applied to whichever text is
|
|
105
|
-
being ranked (description or image)."""
|
|
106
|
-
stripped = value.strip() if value else ""
|
|
107
|
-
return len(stripped) if stripped else -1
|
|
108
|
-
|
|
109
|
-
description = base.description
|
|
110
|
-
best_length = _measured_length(description)
|
|
111
|
-
for a in authors[1:]:
|
|
112
|
-
length = _measured_length(a.description)
|
|
113
|
-
if length > best_length:
|
|
114
|
-
description = a.description
|
|
115
|
-
best_length = length
|
|
116
|
-
|
|
117
|
-
image = next(
|
|
118
|
-
(a.image for a in authors if _measured_length(a.image) >= 0),
|
|
119
|
-
base.image,
|
|
120
|
-
)
|
|
121
|
-
|
|
122
|
-
# Defensive against a stored null slipping through despite the column
|
|
123
|
-
# being NOT NULL on every write path today: the `if a.updated_at`
|
|
124
|
-
# filter drops None candidates before max() ever sees them, so an
|
|
125
|
-
# all-None input leaves the generator empty rather than raising
|
|
126
|
-
# TypeError on a bad comparison — but max() on an empty iterable
|
|
127
|
-
# raises ValueError unless a default is given. `default=None` is
|
|
128
|
-
# what closes that: without it a null here would fail the whole
|
|
129
|
-
# read for a record that is otherwise readable.
|
|
130
|
-
updated_at = max((a.updated_at for a in authors if a.updated_at), default=None)
|
|
131
|
-
|
|
132
|
-
genres_by_asin: dict[str, Any] = {}
|
|
133
|
-
for a in authors:
|
|
134
|
-
for g in (a.genres or []):
|
|
135
|
-
genres_by_asin.setdefault(g.asin, g)
|
|
136
|
-
|
|
137
|
-
genres = [
|
|
138
|
-
{
|
|
139
|
-
"asin": g.asin,
|
|
140
|
-
"name": g.name,
|
|
141
|
-
"type": g.type,
|
|
142
|
-
"betterType": g.type.lower().rstrip("s"),
|
|
143
|
-
"updatedAt": g.updated_at.isoformat() if g.updated_at else None,
|
|
144
|
-
}
|
|
145
|
-
for g in genres_by_asin.values()
|
|
146
|
-
]
|
|
147
|
-
|
|
148
|
-
return {
|
|
149
|
-
"id": base.id,
|
|
150
|
-
"asin": base.asin,
|
|
151
|
-
"name": base.name,
|
|
152
|
-
"description": description,
|
|
153
|
-
"image": image,
|
|
154
|
-
"region": base.region,
|
|
155
|
-
"regions": [base.region],
|
|
156
|
-
"genres": genres,
|
|
157
|
-
"updatedAt": updated_at.isoformat() if updated_at else None,
|
|
158
|
-
}
|
|
156
|
+
return merge_author_rows(authors)
|
|
159
157
|
|
|
160
158
|
|
|
161
159
|
async def get_author_book_asins(
|