codex 1.8.6__py3-none-any.whl → 1.8.9__py3-none-any.whl

codex/choices/admin.py

@@ -109,6 +109,7 @@ ADMIN_TASK_GROUPS: MappingProxyType[
                         "value": "search_index_clear",
                         "title": "Clear Search Index",
                         "desc": "of all entries",
+                        "confirm": "Are you sure? Resyncing the search index can take a while.",
                     },
                     {
                         "value": "db_fts_integrity_check",

codex/librarian/scribe/importer/const.py

@@ -52,24 +52,29 @@ PARENT_FOLDER_FIELD_NAME = "parent_folder"
 VOLUME_COUNT_FIELD_NAME = "volume_count"
 ISSUE_COUNT_FIELD_NAME = "issue_count"
 PATH_FIELD_NAME = "path"
+IDENTIFIERS_FIELD_NAME = "identifiers"
 NON_FTS_FIELDS = frozenset(
     {
         # Attributes
         "critical_rating",
         "day",
+        "file_type",
+        "issue_number",
+        "issue_suffix",
         "metadata_mtime",
         "monochrome",
         "month",
+        "notes",
         "page_count",
         "path",
+        "reading_direction",
         "year",
-        "issue_number",
-        "issue_suffix",
         # FKs
         PARENT_FOLDER_FIELD_NAME,
         VOLUME_FIELD_NAME,
         # M2Ms
         FOLDERS_FIELD_NAME,
+        IDENTIFIERS_FIELD_NAME,
     }
 )
 
@@ -127,7 +132,6 @@ CREDIT_ROLE_FIELD_NAME = "role"
 STORY_ARC_NUMBERS_FIELD_NAME = "story_arc_numbers"
 STORY_ARC_FIELD_NAME = "story_arc"
 NUMBER_FIELD_NAME = "number"
-IDENTIFIERS_FIELD_NAME = "identifiers"
 IDENTIFIER_SOURCE_FIELD_NAME = "source"
 IDENTIFIER_TYPE_FIELD_NAME = "id_type"
 IDENTIFIER_ID_KEY_FIELD_NAME = "key"

codex/librarian/scribe/importer/create/comics.py

@@ -34,9 +34,6 @@ class CreateComicsImporter(CreateForeignKeyLinksImporter):
             if field_name not in NON_FTS_FIELDS:
                 self.metadata[key][sub_key][field_name] = (value,)
 
-        issue = (str(md.get("issue_number", "")) + md.get("issue_suffix", ""),)
-        self.metadata[key][sub_key]["issue"] = issue
-
     def _update_comic_values(self, comic: Comic, update_comics: list, comic_pks: list):
         md = self.metadata[UPDATE_COMICS].pop(comic.pk, {})
         for field_name, value in md.items():

codex/librarian/scribe/importer/delete/comics.py

@@ -47,10 +47,12 @@ class DeletedComicsImporter(DeletedCoversImporter):
 
     def bulk_comics_deleted(self, **kwargs) -> tuple[int, dict]:
         """Bulk delete comics found missing from the filesystem."""
+        count = 0
+        deleted_comic_groups = {}
         status = ImporterRemoveComicsStatus(0, len(self.task.files_deleted))
         try:
             if not self.task.files_deleted:
-                return 0, {}
+                return count, deleted_comic_groups
             self.status_controller.start(status)
             delete_qs = Comic.objects.filter(
                 library=self.library, path__in=self.task.files_deleted
@@ -60,7 +62,8 @@ class DeletedComicsImporter(DeletedCoversImporter):
             deleted_comic_groups = self._populate_deleted_comic_groups(delete_qs)
 
             delete_comic_pks = frozenset(delete_qs.values_list("pk", flat=True))
-            count, _ = delete_qs.delete()
+            delete_qs.delete()
+            count = len(delete_comic_pks)
 
             self.remove_covers(delete_comic_pks, custom=False)
         finally:

codex/librarian/scribe/importer/delete/folders.py

@@ -25,7 +25,8 @@ class DeletedFoldersImporter(DeletedComicsImporter):
                 .distinct()
                 .values_list("pk", flat=True)
             )
-            count, _ = folders.delete()
+            folders.delete()
+            count = len(delete_comic_pks)
 
             self.remove_covers(delete_comic_pks, custom=False)
         finally:

codex/librarian/scribe/importer/link/many_to_many.py

@@ -84,5 +84,6 @@ class LinkManyToManyImporter(LinkSumImporter):
                 except Exception:
                     self.log.exception(f"Error recreating m2m field: {field_name}")
         finally:
+            self.metadata.pop(LINK_M2MS, None)
             self.status_controller.finish(status)
         return created_total + del_total

codex/librarian/scribe/importer/link/prepare.py

@@ -11,6 +11,7 @@ from codex.librarian.scribe.importer.const import (
     FTS_CREATED_M2MS,
     IDENTIFIERS_FIELD_NAME,
     LINK_M2MS,
+    NON_FTS_FIELDS,
 )
 from codex.librarian.scribe.importer.link.const import COMPLEX_MODEL_FIELD_NAMES
 from codex.librarian.scribe.importer.link.covers import (
@@ -47,9 +48,13 @@ class LinkComicsImporterPrepare(LinkCoversImporter):
     ):
         if field_name == IDENTIFIERS_FIELD_NAME:
             # sources extracton must come before identifiers is minified
-            sources = tuple(subvalues[0] for subvalues in values)
+            # but now identifiers is not indexed at all, yet sources are.
+            sources = tuple(sorted({subvalues[0] for subvalues in values}))
             self.add_links_to_fts(comic_pk, "sources", sources)
 
+        if field_name in NON_FTS_FIELDS:
+            return
+
         field_name, fts_values = self.minify_complex_link_to_fts_tuple(
             field_name, values
         )

codex/librarian/scribe/importer/query/update_comics.py

@@ -13,12 +13,31 @@ from codex.librarian.scribe.importer.statii.create import (
     ImporterUpdateComicsStatus,
 )
 from codex.librarian.scribe.importer.statii.query import ImporterQueryComicUpdatesStatus
+from codex.librarian.status import Status
 from codex.models import Comic
 
 
 class QueryUpdateComics(QueryForeignKeysQueryImporter):
     """Move comics that need only updating into correct structure."""
 
+    def _query_update_comic(self, comic: Comic, status: Status):
+        """Query for update one comic."""
+        proposed_comic_dict = self.metadata[CREATE_COMICS].pop(comic.path, None)
+        if not proposed_comic_dict:
+            self.log.warning(
+                f"{comic.path} can be updated, but the update metadata was not found."
+            )
+            return
+        update_comic_dict = {
+            key: value
+            for key, value in proposed_comic_dict.items()
+            if getattr(comic, key) != value
+        }
+        # Update even if update_comic_dict is empty to set stat
+        self.metadata[UPDATE_COMICS][comic.pk] = update_comic_dict
+        status.increment_complete()
+        self.status_controller.update(status)
+
     def query_update_comics(self):
         """Pop existing comics from create & move to update if needed."""
         paths = tuple(self.metadata[CREATE_COMICS].keys())
@@ -34,18 +53,9 @@ class QueryUpdateComics(QueryForeignKeysQueryImporter):
             status.increment_complete(len(paths) - comics.count())
 
             for comic in comics:
-                if self.abort_event.is_set():
-                    return
-                proposed_comic_dict = self.metadata[CREATE_COMICS].pop(comic.path)
-                update_comic_dict = {
-                    key: value
-                    for key, value in proposed_comic_dict.items()
-                    if getattr(comic, key) != value
-                }
-                if update_comic_dict:
-                    self.metadata[UPDATE_COMICS][comic.pk] = update_comic_dict
-                status.increment_complete()
-                self.status_controller.update(status)
+                self._query_update_comic(comic, status)
+            if self.abort_event.is_set():
+                return
 
             create_status = ImporterCreateComicsStatus(
                 None, len(self.metadata[CREATE_COMICS])

codex/librarian/scribe/importer/read/aggregate_path.py

@@ -23,17 +23,56 @@ from codex.librarian.scribe.importer.statii.failed import (
 )
 from codex.librarian.scribe.importer.statii.read import ImporterAggregateStatus
 
-_UNUSED_COMICBOX_FIELDS = (
-    "alternate_images",
-    "bookmark",
-    "credit_primaries",
-    "ext",
-    "manga",
-    "pages",
-    "prices",
-    "remainders",
-    "reprints",
-    "updated_at",
+_USED_COMICBOX_FIELDS = frozenset(
+    {
+        # "alternate_images",
+        "age_rating",
+        "arcs",
+        # "bookmark",
+        "characters",
+        "collection_title",
+        "country",
+        "credits",
+        # "credit_primaries",
+        "critical_rating",
+        "date",
+        # "ext",
+        "file_type",  # extra
+        "genres",
+        "identifiers",
+        # "identifier_primary_source",
+        "imprint",
+        "issue",
+        "language",
+        "locations",
+        "metadata_mtime",  # extra
+        "monochrome",
+        "notes",
+        "original_format",
+        "path",  # extra
+        # "pages",
+        "page_count",
+        "protagonist",
+        # "prices",
+        "publisher",
+        "reading_direction",
+        # "remainders",
+        # "reprints",
+        "review",
+        # "rights",
+        "scan_info",
+        "series",
+        "series_groups",
+        "stories",
+        "summary",
+        "tagger",
+        "tags",
+        "teams",
+        "title",
+        "universes",
+        # "updated_at",
+        "volume",
+    }
 )
 
 
@@ -42,8 +81,9 @@ class AggregateMetadataImporter(AggregatePathMetadataImporter):
 
     @staticmethod
     def _transform_metadata(md):
-        for key in _UNUSED_COMICBOX_FIELDS:
-            md.pop(key, None)
+        for key in tuple(md.keys()):
+            if key not in _USED_COMICBOX_FIELDS:
+                md.pop(key, None)
 
         if date := md.pop(DATE_KEY, None):
             date.pop(COVER_DATE_KEY, None)

codex/librarian/scribe/importer/read/extract.py

@@ -1,6 +1,6 @@
 """Extract metadata from comic archive."""
 
-from datetime import datetime
+from datetime import datetime, timezone
 from tarfile import TarError
 from zipfile import BadZipFile, LargeZipFile
 
@@ -29,7 +29,10 @@ class ExtractMetadataImporter(AggregateMetadataImporter):
             comic = Comic.objects.get(path=path)
         except Comic.DoesNotExist:
             return None
-        return comic.metadata_mtime
+        mtime = comic.metadata_mtime
+        if mtime and (mtime.tzinfo is None or mtime.tzinfo.utcoffset(mtime) is None):
+            mtime = mtime.replace(tzinfo=timezone.utc)
+        return mtime
 
     def _set_import_metadata_flag(self):
         """Set import_metadata flag."""
@@ -50,17 +53,17 @@ class ExtractMetadataImporter(AggregateMetadataImporter):
             with Comicbox(path, config=COMICBOX_CONFIG, logger=self.log) as cb:
                 if import_metadata:
                     new_md_mtime = cb.get_metadata_mtime()
-                    if self.task.check_metadata_mtime:
-                        old_md_mtime = self._metadata_mtime(path)
-                        if (
-                            old_md_mtime
-                            and new_md_mtime
-                            and new_md_mtime <= old_md_mtime
-                        ):
-                            return md
-                    md = cb.to_dict()
-                    md = md.get("comicbox", {})
-                    md["metadata_mtime"] = new_md_mtime
+                    if (
+                        not self.task.check_metadata_mtime
+                        or not new_md_mtime
+                        or not (old_md_mtime := self._metadata_mtime(path))
+                        or (new_md_mtime > old_md_mtime)
+                    ):
+                        md = cb.to_dict()
+                        md = md.get("comicbox", {})
+                        md["metadata_mtime"] = new_md_mtime
+                    else:
+                        md["page_count"] = cb.get_page_count()
                 else:
                     md["page_count"] = cb.get_page_count()
                 md["file_type"] = cb.get_file_type()

codex/librarian/scribe/importer/search/prepare.py

@@ -6,7 +6,6 @@ from codex.librarian.scribe.importer.const import (
     FTS_CREATED_M2MS,
     FTS_EXISTING_M2MS,
     FTS_UPDATE,
-    IDENTIFIERS_FIELD_NAME,
     NON_FTS_FIELDS,
     STORY_ARC_FIELD_NAME,
     STORY_ARC_NUMBERS_FIELD_NAME,
@@ -25,8 +24,6 @@ class SearchIndexPrepareImporter(SearchIndexCreateUpdateImporter):
         """Only store the fts relevant parts of complex links."""
         if field_name == CREDITS_FIELD_NAME:
             values = tuple(subvalues[0] for subvalues in values)
-        if field_name == IDENTIFIERS_FIELD_NAME:
-            values = tuple(subvalues[-1] for subvalues in values)
         elif field_name == STORY_ARC_NUMBERS_FIELD_NAME:
             field_name = STORY_ARC_FIELD_NAME + "s"
         return field_name, tuple(values)
@@ -41,10 +38,6 @@ class SearchIndexPrepareImporter(SearchIndexCreateUpdateImporter):
         """Add the existing values for creating a changed search entry."""
         if field_name in NON_FTS_FIELDS or not values:
             return
-        if field_name == IDENTIFIERS_FIELD_NAME:
-            # sources extracton must come before identifiers is minified
-            sources = tuple(subvalues[0] for subvalues in values)
-            self.add_to_fts_existing(pk, "sources", sources)
         field_name, values = self.minify_complex_link_to_fts_tuple(field_name, values)
         fts_values = self._to_fts_tuple(values)
         if not fts_values:
@@ -72,7 +65,7 @@ class SearchIndexPrepareImporter(SearchIndexCreateUpdateImporter):
                 self.metadata[key] = {}
             if sub_key not in self.metadata[key]:
                 self.metadata[key][sub_key] = {}
-            self.log.warning(
+            self.log.debug(
                 f"FTS import anomaly, attempting FTS update for comic {sub_key} {field_name}"
             )
             # Alternative might be kicking off an FTS sync

codex/librarian/scribe/importer/search/sync_m2m.py

@@ -52,7 +52,8 @@ class SearchIndexSyncManyToManyImporter(FinishImporter):
             .only(*update_fields)
         )
         for comicfts in comicftss:
-            setattr(comicfts, field_name, comicfts.fts_value)  # pyright: ignore[reportAttributeAccessIssue]
+            value = comicfts.fts_value.strip(",")  # pyright: ignore[reportAttributeAccessIssue]
+            setattr(comicfts, field_name, value)
             comicfts.updated_at = Now()
             update_objs.append(comicfts)
 

codex/librarian/scribe/importer/search/update.py

@@ -1,11 +1,7 @@
 """Search Index update."""
 
-from contextlib import suppress
 from time import time
 
-from comicbox.enums.comicbox import IdSources
-from comicbox.enums.maps.identifiers import ID_SOURCE_NAME_MAP
-from django.db.models.functions.datetime import Now
 from humanize import naturaldelta
 
 from codex.librarian.scribe.importer.const import (
@@ -13,10 +9,6 @@ from codex.librarian.scribe.importer.const import (
     FTS_EXISTING_M2MS,
     FTS_UPDATE,
 )
-from codex.librarian.scribe.importer.search.const import (
-    COMICFTS_UPDATE_FIELDS,
-    PYCOUNTRY_FIELDS,
-)
 from codex.librarian.scribe.importer.search.sync_m2m import (
     SearchIndexSyncManyToManyImporter,
 )
@@ -25,6 +17,8 @@ from codex.librarian.scribe.importer.statii.search import (
     ImporterFTSStatus,
     ImporterFTSUpdateStatus,
 )
+from codex.librarian.scribe.search.const import COMICFTS_UPDATE_FIELDS
+from codex.librarian.scribe.search.prepare import SearchEntryPrepare
 from codex.librarian.status import Status
 from codex.models.comic import ComicFTS
 
@@ -32,73 +26,20 @@ from codex.models.comic import ComicFTS
 class SearchIndexCreateUpdateImporter(SearchIndexSyncManyToManyImporter):
     """Search Index update methods."""
 
-    @staticmethod
-    def _get_pycountry_fts_field(entry, field_name) -> str:
-        iso_code = entry.get(field_name)
-        if not iso_code:
-            return ""
-        field = PYCOUNTRY_FIELDS[field_name]
-        return ",".join((iso_code, field.to_representation(iso_code)))
-
-    @staticmethod
-    def _get_sources_fts_field(entry) -> tuple[str, ...]:
-        sources = entry.get("sources", ())
-        names = []
-        for source_str in sources:
-            names.append(source_str)
-            with suppress(ValueError):
-                id_source = IdSources(source_str)
-                if long_name := ID_SOURCE_NAME_MAP.get(id_source):
-                    names.append(long_name)
-        return tuple(names)
-
-    @staticmethod
-    def _create_comicfts_entry_attributes(entry, *, create: bool):
-        now = Now()
-        entry["updated_at"] = now
-        if create:
-            entry["created_at"] = now
-
-    @classmethod
-    def _create_comicfts_entry_fks(cls, entry):
-        entry["country"] = cls._get_pycountry_fts_field(entry, "country")
-        entry["language"] = cls._get_pycountry_fts_field(entry, "language")
-
-    @staticmethod
-    def _to_fts_values(value):
-        """Create a tuple of fts value strings."""
-        return tuple(
-            subvalue
-            for values_tuple in value
-            for subvalue in values_tuple
-            if isinstance(subvalue, str)
-        )
-
-    def _create_comicfts_entry_m2ms(self, pk: int, entry):
-        if sources := self._get_sources_fts_field(entry):
-            entry["sources"] = sources
-        if existing_values := self.metadata[FTS_EXISTING_M2MS].get(pk):
-            for field_name in tuple(existing_values.keys()):
-                if values := existing_values.get(field_name):
-                    entry[field_name] = entry.get(field_name, ()) + values
-
     def _create_comicfts_entry(
         self,
-        pk,
-        entry,
+        pk: int,
+        entry: dict,
         obj_list: list[ComicFTS],
         status: Status,
     ):
-        self._create_comicfts_entry_m2ms(pk, entry)
-        for field_name in entry:
-            value = entry[field_name]
-            if isinstance(value, tuple):
-                entry[field_name] = ",".join(sorted(value))
-        self._create_comicfts_entry_attributes(entry, create=True)
-        self._create_comicfts_entry_fks(entry)
-        comicfts = ComicFTS(comic_id=pk, **entry)
-        obj_list.append(comicfts)
-        status.increment_complete()
+        if entry:
+            existing_m2m_values = self.metadata[FTS_EXISTING_M2MS].get(pk)
+            SearchEntryPrepare.prepare_import_fts_entry(
+                pk, entry, existing_m2m_values, None, obj_list, status, create=True
+            )
+        else:
+            status.decrement_total()
         self.status_controller.update(status)
 
     def _update_comicfts_entry(
@@ -108,18 +49,19 @@ class SearchIndexCreateUpdateImporter(SearchIndexSyncManyToManyImporter):
         status: Status,
     ):
         comic_id = comicfts.comic_id  # pyright:ignore[reportAttributeAccessIssue], # ty: ignore[unresolved-attribute]
-        entry = self.metadata[FTS_UPDATE].pop(comic_id)
-        self._create_comicfts_entry_m2ms(comic_id, entry)
-        for field_name in entry:
-            value = entry[field_name]
-            if isinstance(value, tuple):
-                entry[field_name] = ",".join(sorted(value))
-        self._create_comicfts_entry_attributes(entry, create=True)
-        self._create_comicfts_entry_fks(entry)
-        for field_name, value in entry.items():
-            setattr(comicfts, field_name, value)
-        obj_list.append(comicfts)
-        status.increment_complete()
+        if entry := self.metadata[FTS_UPDATE].pop(comic_id):
+            existing_m2m_values = self.metadata[FTS_EXISTING_M2MS].get(comic_id)
+            SearchEntryPrepare.prepare_import_fts_entry(
+                comic_id,
+                entry,
+                existing_m2m_values,
+                comicfts,
+                obj_list,
+                status,
+                create=False,
+            )
+        else:
+            status.decrement_total()
         self.status_controller.update(status)
 
     def _update_search_index_operate_get_status(
@@ -134,9 +76,9 @@ class SearchIndexCreateUpdateImporter(SearchIndexSyncManyToManyImporter):
         entries = self.metadata.pop(FTS_CREATE, {})
         pks = tuple(sorted(entries.keys()))
         obj_list = []
+        if self.abort_event.is_set():
+            return tuple(obj_list), ()
         for pk in pks:
-            if self.abort_event.is_set():
-                return tuple(obj_list), ()
             entry = entries.pop(pk)
             self._create_comicfts_entry(pk, entry, obj_list, status)
         return tuple(obj_list), pks
@@ -147,9 +89,9 @@ class SearchIndexCreateUpdateImporter(SearchIndexSyncManyToManyImporter):
         obj_list = []
         if pks := tuple(sorted(self.metadata.get(FTS_UPDATE, {}).keys())):
             comicftss = ComicFTS.objects.filter(comic_id__in=pks)
+            if self.abort_event.is_set():
+                return tuple(obj_list), ()
             for comicfts in comicftss:
-                if self.abort_event.is_set():
-                    return tuple(obj_list), ()
                 self._update_comicfts_entry(comicfts, obj_list, status)
             if self.metadata[FTS_UPDATE]:
                 # If updates not popped, turn them into creates.

codex/librarian/scribe/{importer/search → search}/const.py

@@ -1,16 +1,8 @@
-"""Search import consts."""
-
-from types import MappingProxyType
-
-from codex.serializers.fields.browser import CountryField, LanguageField
+"""FTS constants."""
 
 _COMICFTS_ATTRIBUTES = (
     "collection_title",
-    "file_type",
-    "issue",
     "name",
-    "notes",
-    "reading_direction",
     "review",
     "summary",
     "updated_at",
@@ -30,7 +22,6 @@ _COMICFTS_M2MS = (
     "characters",
     "credits",
     "genres",
-    "identifiers",
     "locations",
     "series_groups",
     "sources",
@@ -45,6 +36,3 @@ COMICFTS_UPDATE_FIELDS = (
     *_COMICFTS_FKS,
     *_COMICFTS_M2MS,
 )
-PYCOUNTRY_FIELDS = MappingProxyType(
-    {"country": CountryField(), "language": LanguageField()}
-)