libex-core 0.26.0__tar.gz → 0.27.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. {libex_core-0.26.0 → libex_core-0.27.0}/PKG-INFO +1 -1
  2. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/CHANGELOG.md +16 -0
  3. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/__init__.py +1 -1
  4. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/models.py +7 -0
  5. libex_core-0.27.0/libex_core/storage/migrations/versions/c8d2a5e71f39_folded_search_indexes.py +253 -0
  6. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/models.py +109 -0
  7. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/people.py +57 -59
  8. libex_core-0.27.0/libex_core/storage/read/search.py +465 -0
  9. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/series.py +16 -23
  10. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/shapes.py +16 -16
  11. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/upgrade.py +6 -2
  12. {libex_core-0.26.0 → libex_core-0.27.0}/LICENSE +0 -0
  13. {libex_core-0.26.0 → libex_core-0.27.0}/PYPI.md +0 -0
  14. {libex_core-0.26.0 → libex_core-0.27.0}/libex-core-data/share/bash-completion/completions/libex-core +0 -0
  15. {libex_core-0.26.0 → libex_core-0.27.0}/libex-core-data/share/fish/vendor_completions.d/libex-core.fish +0 -0
  16. {libex_core-0.26.0 → libex_core-0.27.0}/libex-core-data/share/man/man1/libex-core.1 +0 -0
  17. {libex_core-0.26.0 → libex_core-0.27.0}/libex-core-data/share/zsh/site-functions/_libex-core +0 -0
  18. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/__main__.py +0 -0
  19. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/asin.py +0 -0
  20. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/__init__.py +0 -0
  21. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/_concurrency.py +0 -0
  22. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/_retry.py +0 -0
  23. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/authors/__init__.py +0 -0
  24. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/authors/by_name.py +0 -0
  25. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/authors/catalog.py +0 -0
  26. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/authors/profile.py +0 -0
  27. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/authors/screens.py +0 -0
  28. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/books.py +0 -0
  29. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/chapters.py +0 -0
  30. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/client.py +0 -0
  31. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/extras.py +0 -0
  32. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/releases.py +0 -0
  33. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/search.py +0 -0
  34. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/audible/series.py +0 -0
  35. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/__init__.py +0 -0
  36. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/_args.py +0 -0
  37. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/_db_args.py +0 -0
  38. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/_render.py +0 -0
  39. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/_run.py +0 -0
  40. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/_shaping.py +0 -0
  41. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/__init__.py +0 -0
  42. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/abs_search.py +0 -0
  43. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/author.py +0 -0
  44. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/book.py +0 -0
  45. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/completion.py +0 -0
  46. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/config.py +0 -0
  47. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/db.py +0 -0
  48. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/narrator.py +0 -0
  49. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/releases.py +0 -0
  50. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/search.py +0 -0
  51. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/commands/series.py +0 -0
  52. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/environment.py +0 -0
  53. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/exit_codes.py +0 -0
  54. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/main.py +0 -0
  55. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/output.py +0 -0
  56. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/parser.py +0 -0
  57. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/session.py +0 -0
  58. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/cli/store_state.py +0 -0
  59. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/exceptions.py +0 -0
  60. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/log_safety.py +0 -0
  61. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/__init__.py +0 -0
  62. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/_common.py +0 -0
  63. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/_shaping.py +0 -0
  64. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/_store.py +0 -0
  65. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/_walks.py +0 -0
  66. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/author_books.py +0 -0
  67. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/authors.py +0 -0
  68. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/books.py +0 -0
  69. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/ingest.py +0 -0
  70. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/releases.py +0 -0
  71. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/search.py +0 -0
  72. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/series.py +0 -0
  73. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/lookup/stored.py +0 -0
  74. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/py.typed +0 -0
  75. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/shaping.py +0 -0
  76. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/__init__.py +0 -0
  77. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/base.py +0 -0
  78. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/dialect.py +0 -0
  79. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/filtering.py +0 -0
  80. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/merge.py +0 -0
  81. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/env.py +0 -0
  82. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/script.py.mako +0 -0
  83. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/versions/438dbe70d041_create_core_tables.py +0 -0
  84. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/versions/6e1b4c9a2d57_add_credit_order_position_to_pivots.py +0 -0
  85. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/versions/9d2f6b4e8a17_index_book_series_by_series.py +0 -0
  86. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/versions/a4d91f7c3b26_key_books_and_series_by_asin_and_region.py +0 -0
  87. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/versions/b5e9c3a7f184_trigram_indexes_for_the_search_filters.py +0 -0
  88. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/versions/d3a8f1b65e90_add_manual_entry_at_to_books.py +0 -0
  89. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/migrations/versions/f2b8d6a40c17_index_books_by_manual_entry.py +0 -0
  90. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/__init__.py +0 -0
  91. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/_compat.py +0 -0
  92. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/_regions.py +0 -0
  93. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/books.py +0 -0
  94. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/stats.py +0 -0
  95. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/read/walks.py +0 -0
  96. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/sorting.py +0 -0
  97. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/store.py +0 -0
  98. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/types.py +0 -0
  99. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/walk_limits.py +0 -0
  100. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/__init__.py +0 -0
  101. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/books.py +0 -0
  102. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/entities.py +0 -0
  103. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/params.py +0 -0
  104. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/serialize.py +0 -0
  105. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/statements.py +0 -0
  106. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/support.py +0 -0
  107. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/storage/write/walks.py +0 -0
  108. {libex_core-0.26.0 → libex_core-0.27.0}/libex_core/text.py +0 -0
  109. {libex_core-0.26.0 → libex_core-0.27.0}/pyproject.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: libex-core
3
- Version: 0.26.0
3
+ Version: 0.27.0
4
4
  Summary: Audible metadata fetched and normalized into data shaped after AudiMeta's.
5
5
  Requires-Python: >=3.12
6
6
  Description-Content-Type: text/markdown
@@ -13,6 +13,22 @@ the first published version. Entries for earlier versions record changes made
13
13
  before publication: they are historical record for whoever embeds this package,
14
14
  not evidence that anyone consumed a given version at the time it was cut.
15
15
 
16
+ ## [0.27.0]
17
+
18
+ This release adds a migration and a new requirement for Postgres stores: the `unaccent` extension, in the `public` schema, next to `pg_trgm`, which must now be in `public` too. Run `alembic upgrade head` on any store you already have. A SQLite store is not affected; the revision does nothing there, and the new readers do not run on it.
19
+
20
+ ### Added
21
+ - **Ranked search readers for books, authors and series:** `search_books_ranked`, `search_authors_ranked` and `search_series_ranked` in `libex_core.storage.read.search`. They match ignoring case and accents, tolerate typos, and return the same dicts as the single lookups, best match first. They are Postgres only: on any other dialect they raise `NotImplementedError`. `limit` is 1 to 50 and anything else raises `ValueError`. Authors and series also take `exact` for a whole-name match.
22
+ - **`merge_author_rows`** in `libex_core.storage.read.people`, which turns every stored row of one author ASIN and region into the single record `get_author` returns. `get_author` now calls it and behaves as before; `search_authors_ranked` uses it, so a search result is the record `get_author` gives. It takes a non-empty list of `Author` rows ordered by id, and is not re-exported from a package root.
23
+ - **`SeriesSearchResponse`,** a `SeriesResponse` with `bookCount`, the number of stored books linked to the series in its region.
24
+ - **The migration creates the function `public.libex_fold` and nine indexes on Postgres.** The function lowercases text and removes accents. The indexes are four trigram indexes over it (book title with subtitle, author name, series title, narrator name), two btrees for exact author and series names, two on the order of ratings, and one on `book_narrator.narrator_name`. They are named `ix_books_fold_title_gin`, `ix_authors_fold_name_gin`, `ix_series_fold_title_gin`, `ix_narrators_fold_name_gin`, `ix_authors_fold_name_bt`, `ix_series_fold_title_bt`, `ix_books_fold_pop`, `ix_books_fold_region_pop` and `ix_book_narrator_fold_narrator_name`. The models declare them for Postgres only, and creating the tables from the models creates the function first; `unaccent` must already be installed.
25
+ - **The migration needs `unaccent` in `public`.** If it is missing, the migration creates it, but only when the role can: the server must offer it and the role needs `CREATE` on the database and on `public`. `pg_trgm` must already be in `public`. That check is made before anything is changed, and a role that cannot get there gets an error giving the command to run, with the transaction rolled back and no revision stamped. An existing `libex_fold` or index of those names is checked against the expected definition and never replaced; any difference raises. This chain upgrades in one transaction, so the indexes cannot be built concurrently: while each is built, reads of its table carry on and writes to it wait. Downgrading drops the indexes and the function and keeps both extensions.
26
+
27
+ ### Changed
28
+ - **`hydrate_books` reads series positions for all its books in one batch,** where it made one query per book. The result and its order are the same.
29
+ - **Comparing the models with a database no longer reports the folded indexes.** Alembic cannot compare an expression index, so they are skipped on both sides and the migrations own them.
30
+ - **The indexes cost disk and write volume,** on top of those from 0.26.0.
31
+
16
32
  ## [0.26.0]
17
33
 
18
34
  This release adds a migration and a new requirement for Postgres stores: the `pg_trgm` extension. Run `alembic upgrade head` on any store you already have. A SQLite store is not affected; the revision does nothing there.
@@ -36,6 +36,6 @@ the hosted app that embeds it, and an embedder needs a version to pin against
36
36
  that isn't tied to the hosted app's own release line.
37
37
  """
38
38
 
39
- __version__ = "0.26.0"
39
+ __version__ = "0.27.0"
40
40
 
41
41
  __all__ = ["__version__"]
@@ -607,6 +607,13 @@ class SeriesResponse(BaseModel):
607
607
  )
608
608
 
609
609
 
610
+ class SeriesSearchResponse(SeriesResponse):
611
+ bookCount: int = Field(
612
+ default=0,
613
+ description="How many stored books are linked to this series, in its region.",
614
+ )
615
+
616
+
610
617
  # ============================================================
611
618
  # CATEGORIES
612
619
  # ============================================================
@@ -0,0 +1,253 @@
1
+ """folded search indexes for the ranked search
2
+
3
+ Revision ID: c8d2a5e71f39
4
+ Revises: b5e9c3a7f184
5
+ Create Date: 2026-11-06
6
+
7
+ The same objects, under the same names and definitions, as the hosted chain's
8
+ e6a1d4f93b27 (the chains must build one schema): the unaccent extension, the
9
+ function public.libex_fold that lowercases and removes accents, trigram indexes
10
+ over the folded book title and subtitle, author name, series title and narrator
11
+ name, btrees over the folded author name and series title for exact matches, the
12
+ two popularity orders on books, and a btree on book_narrator.narrator_name. The
13
+ raw-column indexes from b5e9c3a7f184 are left as they are. Postgres only: the
14
+ revision does nothing on SQLite.
15
+
16
+ This chain runs in one transaction under an advisory lock, so the builds cannot
17
+ be CONCURRENTLY: each CREATE INDEX takes a SHARE lock on its table for the
18
+ length of the build, and reads carry on while writes to that table wait.
19
+
20
+ pg_trgm and unaccent must both live in the public schema, where the function and
21
+ the search queries name them. What the role cannot do is checked first and
22
+ raises with the command a privileged role must run, so the transaction rolls
23
+ back with no revision stamped. Run the function's command as the role that owns
24
+ the tables: an existing public.libex_fold is checked against the definition below,
25
+ owner included, and is never replaced.
26
+
27
+ Downgrade drops the indexes and the function and keeps both extensions, which
28
+ other objects may depend on and which dropping needs ownership of. Nothing here
29
+ uses CASCADE.
30
+ """
31
+ from typing import Sequence, Union
32
+
33
+ from alembic import op
34
+ from sqlalchemy import text
35
+
36
+
37
+ revision: str = 'c8d2a5e71f39'
38
+ down_revision: Union[str, Sequence[str], None] = 'b5e9c3a7f184'
39
+ branch_labels: Union[str, Sequence[str], None] = None
40
+ depends_on: Union[str, Sequence[str], None] = None
41
+
42
+ # The body is compared character for character against an existing function.
43
+ # Copies live in libex_core's models and in the hosted chain's revision, and a
44
+ # test holds them identical.
45
+ FOLD_FUNCTION_BODY = "SELECT pg_catalog.lower(public.unaccent('public.unaccent'::regdictionary, $1))"
46
+ FOLD_FUNCTION_DDL = (
47
+ "CREATE FUNCTION public.libex_fold(text) RETURNS text LANGUAGE sql "
48
+ f"IMMUTABLE STRICT PARALLEL SAFE AS $${FOLD_FUNCTION_BODY}$$"
49
+ )
50
+ EXTENSION_DDL = 'CREATE EXTENSION IF NOT EXISTS unaccent WITH SCHEMA public'
51
+ # True when public.unaccent and its dictionary are the extension's own.
52
+ UNACCENT_IS_THE_EXTENSION_SQL = (
53
+ "SELECT EXISTS (SELECT 1 FROM pg_ts_dict WHERE dictname = 'unaccent' AND dictnamespace = 'public'::regnamespace) AND EXISTS ("
54
+ "SELECT 1 FROM pg_depend d JOIN pg_proc p ON p.oid = d.objid "
55
+ "WHERE d.classid = 'pg_proc'::regclass AND d.refclassid = 'pg_extension'::regclass "
56
+ "AND d.deptype = 'e' AND d.refobjid = (SELECT oid FROM pg_extension WHERE extname = 'unaccent') "
57
+ "AND p.proname = 'unaccent' AND p.pronamespace = 'public'::regnamespace)"
58
+ )
59
+
60
+ # (index, table, what follows "ON <table> ", the definition Postgres reports for
61
+ # it with the public schema qualifier removed)
62
+ INDEXES = (
63
+ (
64
+ 'ix_books_fold_title_gin', 'books',
65
+ "USING gin (public.libex_fold(title || ' ' || coalesce(subtitle, '')) public.gin_trgm_ops)",
66
+ "CREATE INDEX ix_books_fold_title_gin ON books USING gin "
67
+ "(libex_fold(((title || ' '::text) || COALESCE(subtitle, ''::text))) gin_trgm_ops)",
68
+ ),
69
+ (
70
+ 'ix_authors_fold_name_gin', 'authors',
71
+ 'USING gin (public.libex_fold(name) public.gin_trgm_ops)',
72
+ 'CREATE INDEX ix_authors_fold_name_gin ON authors USING gin (libex_fold(name) gin_trgm_ops)',
73
+ ),
74
+ (
75
+ 'ix_series_fold_title_gin', 'series',
76
+ 'USING gin (public.libex_fold(title) public.gin_trgm_ops)',
77
+ 'CREATE INDEX ix_series_fold_title_gin ON series USING gin (libex_fold(title) gin_trgm_ops)',
78
+ ),
79
+ (
80
+ 'ix_narrators_fold_name_gin', 'narrators',
81
+ 'USING gin (public.libex_fold(name) public.gin_trgm_ops)',
82
+ 'CREATE INDEX ix_narrators_fold_name_gin ON narrators USING gin (libex_fold(name) gin_trgm_ops)',
83
+ ),
84
+ (
85
+ 'ix_authors_fold_name_bt', 'authors',
86
+ '(public.libex_fold(name))',
87
+ 'CREATE INDEX ix_authors_fold_name_bt ON authors USING btree (libex_fold(name))',
88
+ ),
89
+ (
90
+ 'ix_series_fold_title_bt', 'series',
91
+ '(public.libex_fold(title))',
92
+ 'CREATE INDEX ix_series_fold_title_bt ON series USING btree (libex_fold(title))',
93
+ ),
94
+ (
95
+ 'ix_books_fold_pop', 'books',
96
+ '(num_ratings DESC NULLS LAST, asin, region) WHERE is_primary',
97
+ 'CREATE INDEX ix_books_fold_pop ON books USING btree '
98
+ '(num_ratings DESC NULLS LAST, asin, region) WHERE is_primary',
99
+ ),
100
+ (
101
+ 'ix_books_fold_region_pop', 'books',
102
+ '(region, num_ratings DESC NULLS LAST, asin)',
103
+ 'CREATE INDEX ix_books_fold_region_pop ON books USING btree '
104
+ '(region, num_ratings DESC NULLS LAST, asin)',
105
+ ),
106
+ (
107
+ 'ix_book_narrator_fold_narrator_name', 'book_narrator',
108
+ '(narrator_name)',
109
+ 'CREATE INDEX ix_book_narrator_fold_narrator_name ON book_narrator USING btree (narrator_name)',
110
+ ),
111
+ )
112
+
113
+ # The tables the indexes sit on.
114
+ ANALYZED_TABLES = ('books', 'authors', 'series', 'narrators', 'book_narrator')
115
+
116
+
117
+ def _scalar(sql: str):
118
+ return op.get_bind().execute(text(sql)).scalar()
119
+
120
+
121
+ def _extension_schema(name: str):
122
+ return _scalar(
123
+ "SELECT n.nspname FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace "
124
+ f"WHERE e.extname = '{name}'"
125
+ )
126
+
127
+
128
+ def _check_extensions() -> None:
129
+ """pg_trgm and unaccent installed in public, unaccent created if the role
130
+ can and raising with the command if it cannot."""
131
+ trgm = _extension_schema('pg_trgm')
132
+ if trgm != 'public':
133
+ raise RuntimeError(
134
+ f'the search indexes use public.gin_trgm_ops, but pg_trgm is {"not installed" if trgm is None else f"installed in the {trgm} schema"}'
135
+ )
136
+ installed = _extension_schema('unaccent')
137
+ if installed not in (None, 'public'):
138
+ raise RuntimeError(
139
+ f'unaccent is installed in the {installed} schema; public.libex_fold calls public.unaccent'
140
+ )
141
+ if installed is None:
142
+ can_create = _scalar(
143
+ "SELECT EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = 'unaccent') "
144
+ "AND has_database_privilege(current_user, current_database(), 'CREATE') "
145
+ "AND has_schema_privilege(current_user, 'public', 'CREATE')"
146
+ )
147
+ if not can_create:
148
+ raise RuntimeError(
149
+ 'the search indexes need the unaccent extension, which is not installed and '
150
+ 'cannot be created by this role; a privileged role must run: ' + EXTENSION_DDL
151
+ )
152
+ op.execute(EXTENSION_DDL)
153
+ ok = _scalar(UNACCENT_IS_THE_EXTENSION_SQL)
154
+ if not ok:
155
+ raise RuntimeError(
156
+ 'public.unaccent is not provided by the unaccent extension; public.libex_fold needs it'
157
+ )
158
+
159
+
160
+ def _fold_function_problems() -> list[str]:
161
+ """What differs between the existing libex_fold and the definition above.
162
+ Empty when there is exactly one, and it matches."""
163
+ functions = op.get_bind().execute(text(
164
+ "SELECT p.prosrc, l.lanname, p.provolatile::text, p.proisstrict, p.proparallel::text, "
165
+ "p.prorettype::regtype::text, p.prosecdef, p.proconfig IS NOT NULL, p.proretset, "
166
+ "p.pronamespace::regnamespace::text, pg_get_function_identity_arguments(p.oid), "
167
+ "p.proowner = (SELECT relowner FROM pg_class WHERE oid = 'public.books'::regclass) "
168
+ "FROM pg_proc p JOIN pg_language l ON l.oid = p.prolang WHERE p.proname = 'libex_fold'"
169
+ )).all()
170
+ if len(functions) != 1:
171
+ return [f'{len(functions)} functions named libex_fold exist, expected exactly one']
172
+ (body, lang, volatile, strict, parallel, returns, definer, has_config, is_set,
173
+ namespace, args, same_owner) = functions[0]
174
+ expected = {
175
+ 'body': (body, FOLD_FUNCTION_BODY),
176
+ 'language': (lang, 'sql'),
177
+ 'volatility': (volatile, 'i'),
178
+ 'strict': (strict, True),
179
+ 'parallel': (parallel, 's'),
180
+ 'return type': (returns, 'text'),
181
+ 'security definer': (definer, False),
182
+ 'a SET clause': (has_config, False),
183
+ 'returns a set': (is_set, False),
184
+ 'schema': (namespace, 'public'),
185
+ 'arguments': (args, 'text'),
186
+ "the owner being books' owner": (same_owner, True),
187
+ }
188
+ return [f'{what}: found {found!r}, expected {want!r}' for what, (found, want) in expected.items() if found != want]
189
+
190
+
191
+ def _ensure_fold_function() -> None:
192
+ if _scalar("SELECT count(*) FROM pg_proc WHERE proname = 'libex_fold'") == 0:
193
+ if not _scalar("SELECT has_schema_privilege(current_user, 'public', 'CREATE')"):
194
+ raise RuntimeError(
195
+ 'the search indexes need the function public.libex_fold, which cannot be created '
196
+ 'by this role; the role that owns the tables must run: ' + FOLD_FUNCTION_DDL
197
+ )
198
+ op.execute(FOLD_FUNCTION_DDL)
199
+ return
200
+ problems = _fold_function_problems()
201
+ if problems:
202
+ raise RuntimeError(
203
+ 'an existing libex_fold does not match the definition this revision expects, and is '
204
+ 'not replaced: ' + '; '.join(problems)
205
+ )
206
+
207
+
208
+ def _index_definition(name: str):
209
+ """The definition of an existing index with the public qualifier removed,
210
+ or None when there is none."""
211
+ row = op.get_bind().execute(
212
+ text('SELECT pg_get_indexdef(indexrelid) FROM pg_index WHERE indexrelid = to_regclass(:name)'),
213
+ {'name': name},
214
+ ).first()
215
+ return None if row is None else row[0].replace('public.', '')
216
+
217
+
218
+ def _index_is_usable(name: str) -> bool:
219
+ """False for an index a failed build left behind (not valid or not ready)."""
220
+ return bool(_scalar(
221
+ f"SELECT indisvalid AND indisready FROM pg_index WHERE indexrelid = to_regclass('{name}')"
222
+ ))
223
+
224
+
225
+ def upgrade() -> None:
226
+ if op.get_bind().dialect.name != 'postgresql':
227
+ return
228
+ _check_extensions()
229
+ _ensure_fold_function()
230
+ for name, table, tail, expected in INDEXES:
231
+ found = _index_definition(name)
232
+ if found is None:
233
+ op.execute(f'CREATE INDEX {name} ON {table} {tail}')
234
+ elif found != expected:
235
+ raise RuntimeError(
236
+ f'{name} already exists with a different definition and is not replaced: '
237
+ f'found {found!r}, expected {expected!r}'
238
+ )
239
+ elif not _index_is_usable(name):
240
+ raise RuntimeError(
241
+ f'{name} already exists but is not valid and ready, as a failed build leaves it; '
242
+ 'drop it and run the upgrade again'
243
+ )
244
+ for table in ANALYZED_TABLES:
245
+ op.execute(f'ANALYZE {table}')
246
+
247
+
248
+ def downgrade() -> None:
249
+ if op.get_bind().dialect.name != 'postgresql':
250
+ return
251
+ for name, _table, _tail, _expected in INDEXES:
252
+ op.execute(f'DROP INDEX IF EXISTS {name}')
253
+ op.execute('DROP FUNCTION IF EXISTS public.libex_fold(text)')
@@ -16,6 +16,7 @@ from sqlalchemy import (
16
16
  Boolean,
17
17
  CheckConstraint,
18
18
  Column,
19
+ DDL,
19
20
  Double,
20
21
  Enum,
21
22
  ForeignKey,
@@ -26,6 +27,9 @@ from sqlalchemy import (
26
27
  Table,
27
28
  Text,
28
29
  UniqueConstraint,
30
+ event,
31
+ func,
32
+ literal_column,
29
33
  text,
30
34
  true,
31
35
  )
@@ -428,6 +432,105 @@ for _name, _column in (
428
432
  ).ddl_if(dialect="postgresql")
429
433
 
430
434
 
435
+ # ============================================================
436
+ # FOLDED SEARCH INDEXES
437
+ # ============================================================
438
+ # What the ranked /db search reads: the same text lowercased and with accents
439
+ # removed by public.libex_fold, indexed for trigram matching, plus the btrees
440
+ # behind exact matching and the popularity order. Postgres only; the migrations
441
+ # (e6a1d4f93b27 hosted, c8d2a5e71f39 here) build the same set under the same
442
+ # names. The separators in the title expression are SQL literals, not bound
443
+ # values: a bound separator makes the expression something a generic plan cannot
444
+ # match to the index. concat_ws is avoided because it is STABLE and cannot be
445
+ # indexed.
446
+
447
+ # The text of public.libex_fold. Each migration keeps its own copy, and a test
448
+ # holds them identical: an existing function's body is compared character for
449
+ # character, so it must not drift.
450
+ FOLD_FUNCTION_BODY = "SELECT pg_catalog.lower(public.unaccent('public.unaccent'::regdictionary, $1))"
451
+ FOLD_FUNCTION_DDL = (
452
+ "CREATE FUNCTION public.libex_fold(text) RETURNS text LANGUAGE sql "
453
+ f"IMMUTABLE STRICT PARALLEL SAFE AS $${FOLD_FUNCTION_BODY}$$"
454
+ )
455
+
456
+ # Every index this section declares. include_object_for keeps them out of a
457
+ # compare with a database: alembic cannot compare an expression index, and the
458
+ # migrations own them.
459
+ FOLDED_INDEX_NAMES = frozenset({
460
+ "ix_books_fold_title_gin",
461
+ "ix_authors_fold_name_gin",
462
+ "ix_series_fold_title_gin",
463
+ "ix_narrators_fold_name_gin",
464
+ "ix_authors_fold_name_bt",
465
+ "ix_series_fold_title_bt",
466
+ "ix_books_fold_pop",
467
+ "ix_books_fold_region_pop",
468
+ "ix_book_narrator_fold_narrator_name",
469
+ })
470
+
471
+
472
+ def _fold(expression):
473
+ return func.public.libex_fold(expression)
474
+
475
+
476
+ _BOOK_FOLDED_TEXT = _fold(
477
+ Book.title + literal_column("' '") + func.coalesce(Book.subtitle, literal_column("''"))
478
+ )
479
+
480
+ for _name, _expression in (
481
+ ("ix_books_fold_title_gin", _BOOK_FOLDED_TEXT),
482
+ ("ix_authors_fold_name_gin", _fold(Author.name)),
483
+ ("ix_series_fold_title_gin", _fold(Series.title)),
484
+ ("ix_narrators_fold_name_gin", _fold(Narrator.name)),
485
+ ):
486
+ Index(
487
+ _name,
488
+ _expression,
489
+ postgresql_using="gin",
490
+ postgresql_ops={_expression.key: "public.gin_trgm_ops"},
491
+ ).ddl_if(dialect="postgresql")
492
+
493
+ Index("ix_authors_fold_name_bt", _fold(Author.name)).ddl_if(dialect="postgresql")
494
+ Index("ix_series_fold_title_bt", _fold(Series.title)).ddl_if(dialect="postgresql")
495
+
496
+ # The popularity order, for the shortcut that takes the most popular matches
497
+ # first: one index for a search over every region's primary rows, one for a
498
+ # search inside a region.
499
+ Index(
500
+ "ix_books_fold_pop",
501
+ Book.num_ratings.desc().nulls_last(),
502
+ Book.asin,
503
+ Book.region,
504
+ postgresql_where=text("is_primary"),
505
+ ).ddl_if(dialect="postgresql")
506
+ Index(
507
+ "ix_books_fold_region_pop",
508
+ Book.region,
509
+ Book.num_ratings.desc().nulls_last(),
510
+ Book.asin,
511
+ ).ddl_if(dialect="postgresql")
512
+
513
+ # The indexes call the function, so it has to exist before they are built. It
514
+ # needs the unaccent extension, which this module does not create: whatever
515
+ # provides pg_trgm provides that too.
516
+ _ENSURE_FOLD_FUNCTION = DDL(
517
+ "DO $do$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_proc p "
518
+ "WHERE p.proname = 'libex_fold' AND p.pronamespace = 'public'::regnamespace "
519
+ "AND p.proargtypes::text = '25') THEN "
520
+ f"{FOLD_FUNCTION_DDL}; END IF; END $do$"
521
+ ).execute_if(dialect="postgresql")
522
+
523
+ # Skipped rather than failed while something still uses the function, so a drop
524
+ # of only some of the tables does not abort on the indexes left behind.
525
+ _DROP_FOLD_FUNCTION = DDL(
526
+ "DO $do$ BEGIN DROP FUNCTION IF EXISTS public.libex_fold(text); "
527
+ "EXCEPTION WHEN dependent_objects_still_exist THEN NULL; END $do$"
528
+ ).execute_if(dialect="postgresql")
529
+
530
+ event.listen(Base.metadata, "before_create", _ENSURE_FOLD_FUNCTION)
531
+ event.listen(Base.metadata, "after_drop", _DROP_FOLD_FUNCTION)
532
+
533
+
431
534
  # ============================================================
432
535
  # GENRES
433
536
  # ============================================================
@@ -533,6 +636,12 @@ book_narrator = Table(
533
636
  UniqueConstraint("book_asin", "book_region", "narrator_name", name="uq_book_narrator"),
534
637
  )
535
638
 
639
+ # Serves the narrator search, which finds the matching narrators first and then
640
+ # their books by narrator_name.
641
+ Index(
642
+ "ix_book_narrator_fold_narrator_name", book_narrator.c.narrator_name
643
+ ).ddl_if(dialect="postgresql")
644
+
536
645
  book_series = Table(
537
646
  "book_series",
538
647
  Base.metadata,
@@ -19,6 +19,7 @@ from libex_core.storage.read._regions import only_first_stored
19
19
  from libex_core.storage.read.shapes import (
20
20
  BOOK_RELATIONS,
21
21
  book_to_dict,
22
+ genre_to_dict,
22
23
  hydrate_books,
23
24
  narrator_to_dict,
24
25
  series_positions_by_book,
@@ -28,6 +29,61 @@ from libex_core.storage.sorting import BOOK_SORT_FIELDS, NARRATOR_SORT_FIELDS, a
28
29
  logger = logging.getLogger("libex")
29
30
 
30
31
 
32
+ def merge_author_rows(authors: list[Author]) -> dict[str, Any]:
33
+ """One record from every stored row of an (asin, region), by the rules
34
+ get_author documents. `authors` is non-empty and ordered by id."""
35
+ base = authors[0]
36
+
37
+ def _measured_length(value: str | None) -> int:
38
+ """Trimmed length, floored to -1 for absent — same idea as
39
+ merge.longer_wins' absent-sentinel, applied to whichever text is
40
+ being ranked (description or image)."""
41
+ stripped = value.strip() if value else ""
42
+ return len(stripped) if stripped else -1
43
+
44
+ description = base.description
45
+ best_length = _measured_length(description)
46
+ for a in authors[1:]:
47
+ length = _measured_length(a.description)
48
+ if length > best_length:
49
+ description = a.description
50
+ best_length = length
51
+
52
+ image = next(
53
+ (a.image for a in authors if _measured_length(a.image) >= 0),
54
+ base.image,
55
+ )
56
+
57
+ # Defensive against a stored null slipping through despite the column
58
+ # being NOT NULL on every write path today: the `if a.updated_at`
59
+ # filter drops None candidates before max() ever sees them, so an
60
+ # all-None input leaves the generator empty rather than raising
61
+ # TypeError on a bad comparison — but max() on an empty iterable
62
+ # raises ValueError unless a default is given. `default=None` is
63
+ # what closes that: without it a null here would fail the whole
64
+ # read for a record that is otherwise readable.
65
+ updated_at = max((a.updated_at for a in authors if a.updated_at), default=None)
66
+
67
+ genres_by_asin: dict[str, Any] = {}
68
+ for a in authors:
69
+ for g in (a.genres or []):
70
+ genres_by_asin.setdefault(g.asin, g)
71
+
72
+ genres = [genre_to_dict(g) for g in genres_by_asin.values()]
73
+
74
+ return {
75
+ "id": base.id,
76
+ "asin": base.asin,
77
+ "name": base.name,
78
+ "description": description,
79
+ "image": image,
80
+ "region": base.region,
81
+ "regions": [base.region],
82
+ "genres": genres,
83
+ "updatedAt": updated_at.isoformat() if updated_at else None,
84
+ }
85
+
86
+
31
87
  async def get_author(session: AsyncSession, asin: str, region: str) -> dict[str, Any] | None:
32
88
  """Fetches an author from the DB with genres.
33
89
 
@@ -97,65 +153,7 @@ async def get_author(session: AsyncSession, asin: str, region: str) -> dict[str,
97
153
  },
98
154
  )
99
155
 
100
- base = authors[0]
101
-
102
- def _measured_length(value: str | None) -> int:
103
- """Trimmed length, floored to -1 for absent — same idea as
104
- merge.longer_wins' absent-sentinel, applied to whichever text is
105
- being ranked (description or image)."""
106
- stripped = value.strip() if value else ""
107
- return len(stripped) if stripped else -1
108
-
109
- description = base.description
110
- best_length = _measured_length(description)
111
- for a in authors[1:]:
112
- length = _measured_length(a.description)
113
- if length > best_length:
114
- description = a.description
115
- best_length = length
116
-
117
- image = next(
118
- (a.image for a in authors if _measured_length(a.image) >= 0),
119
- base.image,
120
- )
121
-
122
- # Defensive against a stored null slipping through despite the column
123
- # being NOT NULL on every write path today: the `if a.updated_at`
124
- # filter drops None candidates before max() ever sees them, so an
125
- # all-None input leaves the generator empty rather than raising
126
- # TypeError on a bad comparison — but max() on an empty iterable
127
- # raises ValueError unless a default is given. `default=None` is
128
- # what closes that: without it a null here would fail the whole
129
- # read for a record that is otherwise readable.
130
- updated_at = max((a.updated_at for a in authors if a.updated_at), default=None)
131
-
132
- genres_by_asin: dict[str, Any] = {}
133
- for a in authors:
134
- for g in (a.genres or []):
135
- genres_by_asin.setdefault(g.asin, g)
136
-
137
- genres = [
138
- {
139
- "asin": g.asin,
140
- "name": g.name,
141
- "type": g.type,
142
- "betterType": g.type.lower().rstrip("s"),
143
- "updatedAt": g.updated_at.isoformat() if g.updated_at else None,
144
- }
145
- for g in genres_by_asin.values()
146
- ]
147
-
148
- return {
149
- "id": base.id,
150
- "asin": base.asin,
151
- "name": base.name,
152
- "description": description,
153
- "image": image,
154
- "region": base.region,
155
- "regions": [base.region],
156
- "genres": genres,
157
- "updatedAt": updated_at.isoformat() if updated_at else None,
158
- }
156
+ return merge_author_rows(authors)
159
157
 
160
158
 
161
159
  async def get_author_book_asins(