codex 1.8.5__py3-none-any.whl → 1.8.8__py3-none-any.whl

codex/librarian/scribe/search/remove.py

@@ -1,11 +1,13 @@
 """Search Index cleanup."""
 
+from django.db.models import F, Max
+
 from codex.librarian.scribe.search.optimize import SearchIndexerOptimize
 from codex.librarian.scribe.search.status import (
     SearchIndexCleanStatus,
     SearchIndexClearStatus,
 )
-from codex.models.comic import ComicFTS
+from codex.models.comic import Comic, ComicFTS
 
 
 class SearchIndexerRemove(SearchIndexerOptimize):
@@ -22,7 +24,9 @@ class SearchIndexerRemove(SearchIndexerOptimize):
         """Remove records not in the database from the index."""
         self.status_controller.start(status)
         self.log.debug("Finding stale records to remove...")
-        delete_comicfts = ComicFTS.objects.filter(comic__isnull=True)
+        delete_comicfts = ComicFTS.objects.exclude(
+            comic_id__in=Comic.objects.only("pk")
+        )
         status.total = delete_comicfts.count()
         self.status_controller.update(status)
         if status.total:
@@ -42,3 +46,19 @@ class SearchIndexerRemove(SearchIndexerOptimize):
         finally:
             self.status_controller.finish(status)
         return count
+
+    def remove_duplicate_records(self) -> int:
+        """Remove duplicate FTS records."""
+        self.log.debug("Looking for duplicate search entries...")
+        duplicates = ComicFTS.objects.annotate(max_updated_at=Max("updated_at")).filter(
+            updated_at__lt=F("max_updated_at")
+        )
+
+        num_dupes = duplicates.count()
+        self.log.debug(f"Found {num_dupes} duplicate search entries")
+        count = num_dupes
+        if num_dupes:
+            count, _ = duplicates.delete()
+            if count:
+                self.log.info(f"Deleted {count} duplicate search entries")
+        return count

codex/librarian/scribe/search/sync.py

@@ -1,17 +1,21 @@
 """Search Index update."""
 
 from datetime import datetime
+from math import floor
 from time import time
+from typing import TYPE_CHECKING
 from zoneinfo import ZoneInfo
 
-from comicbox.enums.maps.identifiers import ID_SOURCE_NAME_MAP
+from django.db.models import Q
 from django.db.models.aggregates import Max
 from django.db.models.expressions import F, Value
 from django.db.models.functions import Concat
-from django.db.models.functions.datetime import Now
 from django.db.models.query import QuerySet
 from humanize import intcomma, naturaldelta
 
+from codex.librarian.memory import get_mem_limit
+from codex.librarian.scribe.search.const import COMICFTS_UPDATE_FIELDS
+from codex.librarian.scribe.search.prepare import SearchEntryPrepare
 from codex.librarian.scribe.search.remove import SearchIndexerRemove
 from codex.librarian.scribe.search.status import (
     SEARCH_INDEX_STATII,
@@ -20,56 +24,16 @@ from codex.librarian.scribe.search.status import (
     SearchIndexSyncCreateStatus,
     SearchIndexSyncUpdateStatus,
 )
-from codex.librarian.status import Status
 from codex.models import Comic
 from codex.models.comic import ComicFTS
 from codex.models.functions import GroupConcat
-from codex.serializers.fields.browser import CountryField, LanguageField, PyCountryField
-from codex.settings import SEARCH_INDEX_BATCH_SIZE
+from codex.settings import SEARCH_SYNC_BATCH_MEMORY_RATIO
+
+if TYPE_CHECKING:
+    from codex.librarian.status import Status
 
-_COMICFTS_ATTRIBUTES = (
-    "collection_title",
-    "file_type",
-    "issue",
-    "name",
-    "notes",
-    "reading_direction",
-    "review",
-    "summary",
-    "updated_at",
-)
-_COMICFTS_FKS = (
-    "publisher",
-    "imprint",
-    "series",
-    "age_rating",
-    "country",
-    "language",
-    "original_format",
-    "scan_info",
-    "tagger",
-)
-_COMICFTS_M2MS = (
-    "characters",
-    "credits",
-    "genres",
-    "identifiers",
-    "locations",
-    "series_groups",
-    "sources",
-    "stories",
-    "story_arcs",
-    "tags",
-    "teams",
-    "universes",
-)
-_COMICFTS_UPDATE_FIELDS = (
-    *_COMICFTS_ATTRIBUTES,
-    *_COMICFTS_FKS,
-    *_COMICFTS_M2MS,
-)
 _MIN_UTC_DATE = datetime.min.replace(tzinfo=ZoneInfo("UTC"))
-_CHUNK_HUMAN_SIZE = intcomma(SEARCH_INDEX_BATCH_SIZE)
+_ALL_FTS_COMIC_IDS_QUERY = Q(pk__in=ComicFTS.objects.values_list("comic_id", flat=True))
 
 
 class SearchIndexerSync(SearchIndexerRemove):
@@ -167,22 +131,14 @@ class SearchIndexerSync(SearchIndexerRemove):
             ),
         )
 
-    @staticmethod
-    def _get_pycountry_fts_field(field: PyCountryField, iso_code) -> str:
-        if not iso_code:
-            return ""
-        return ",".join((iso_code, field.to_representation(iso_code)))
-
-    @staticmethod
-    def _get_sources_fts_field(sources) -> str | None:
-        names = []
-        if not sources:
-            return None
-        for source in sources.split(","):
-            names.append(source)
-            if long_name := ID_SOURCE_NAME_MAP.get(source):
-                names.append(long_name)
-        return ",".join(names)
+    def _update_search_index_operate_get_status(
+        self, total_comics: int, chunk_human_size: str, *, create: bool
+    ):
+        status_class = (
+            SearchIndexSyncCreateStatus if create else SearchIndexSyncUpdateStatus
+        )
+        subtitle = f"Chunks of {chunk_human_size}" if total_comics else ""
+        return status_class(total=total_comics, subtitle=subtitle)
 
     def _update_search_index_create_or_update(
         self,
@@ -202,133 +158,75 @@ class SearchIndexerSync(SearchIndexerRemove):
         if create:
             ComicFTS.objects.bulk_create(obj_list)
         else:
-            ComicFTS.objects.bulk_update(obj_list, _COMICFTS_UPDATE_FIELDS)
-        obj_list.clear()
-
-    def _create_comicfts_entry(
-        self,
-        comic,
-        obj_list: list[ComicFTS],
-        status: Status,
-        *,
-        create: bool,
-    ):
-        country_field: PyCountryField = CountryField()  # pyright: ignore[reportAssignmentType]
-        language_field: PyCountryField = LanguageField()  # pyright: ignore[reportAssignmentType]
-
-        country = self._get_pycountry_fts_field(country_field, comic.fts_country)
-        language = self._get_pycountry_fts_field(language_field, comic.fts_language)
-        sources = self._get_sources_fts_field(comic.fts_sources)
-
-        now = Now()
-        comicfts = ComicFTS(
-            comic_id=comic.pk,
-            # Attributes
-            collection_title=comic.collection_title,
-            file_type=comic.file_type,
-            issue=comic.issue(),
-            name=comic.name,
-            notes=comic.notes,
-            reading_direction=comic.reading_direction,
-            review=comic.review,
-            summary=comic.summary,
-            updated_at=now,
-            # Group FKs
-            publisher=comic.fts_publisher,
-            imprint=comic.fts_imprint,
-            series=comic.fts_series,
-            # FKS
-            age_rating=comic.fts_age_rating,
-            country=country,
-            language=language,
-            original_format=comic.fts_original_format,
-            scan_info=comic.fts_scan_info,
-            tagger=comic.fts_tagger,
-            # ManyToMany
-            characters=comic.fts_characters,
-            credits=comic.fts_credits,
-            genres=comic.fts_genres,
-            identifiers=comic.fts_identifiers,
-            sources=sources,  # comic.fts_sources,
-            locations=comic.fts_locations,
-            series_groups=comic.fts_series_groups,
-            stories=comic.fts_stories,
-            story_arcs=comic.fts_story_arcs,
-            tags=comic.fts_tags,
-            teams=comic.fts_teams,
-            universes=comic.fts_universes,
-        )
-        if create:
-            comicfts.created_at = now
+            ComicFTS.objects.bulk_update(obj_list, COMICFTS_UPDATE_FIELDS)
 
-        obj_list.append(comicfts)
-        status.increment_complete()
-        self.status_controller.update(status)
+        status.increment_complete(num_comic_fts)
+        self.status_controller.update(status, notify=True)
 
-    def _update_search_index_operate_get_status(
-        self, total_comics: int, *, create: bool
-    ):
-        status_class = (
-            SearchIndexSyncCreateStatus if create else SearchIndexSyncUpdateStatus
-        )
-        subtitle = f"Chunks of {_CHUNK_HUMAN_SIZE}" if total_comics else ""
-        return status_class(total=total_comics, subtitle=subtitle)
+    @staticmethod
+    def _get_operation_comics_query(qs, *, create: bool):
+        if create and not ComicFTS.objects.exists():
+            qs = Comic.objects.all()
+        return qs
 
     def _update_search_index_operate(
         self, comics_filtered_qs: QuerySet, *, create: bool
     ):
-        total_comics = comics_filtered_qs.count()
+        # Smaller systems may run out of virtual memory unless this is auto governed.
+        mem_limit_gb = get_mem_limit("g")
+        search_index_batch_size = floor(
+            (mem_limit_gb / SEARCH_SYNC_BATCH_MEMORY_RATIO) * 1000
+        )
+        chunk_human_size = intcomma(search_index_batch_size)
 
         verb = "create" if create else "update"
+        self.log.debug(f"Counting total search index entries to {verb}...")
+        total_comics = self._get_operation_comics_query(
+            comics_filtered_qs, create=create
+        )
+        total_comics = total_comics.count()
         status = self._update_search_index_operate_get_status(
-            total_comics, create=create
+            total_comics, chunk_human_size, create=create
         )
 
         try:
-            if total_comics:
-                reason = (
-                    f"Batching this search engine {verb} operation in to chunks of {_CHUNK_HUMAN_SIZE}."
-                    f" Search engine {verb}s run much faster as one large batch but then there's no progress updates."
-                    " You may adjust the batch size with the environment variable CODEX_SEARCH_INDEX_BATCH_SIZE."
-                )
-            else:
-                reason = f"No search entries to {verb}."
-            self.log.debug(reason)
             if not total_comics:
+                self.log.debug(f"No search entries to {verb}.")
                 return total_comics
 
-            self.status_controller.start(status)
-            comics = self._select_related_fts_query(comics_filtered_qs)
-            comics = self._prefetch_related_fts_query(comics)
-            comics = self._annotate_fts_query(comics)
-            comics = comics.order_by("pk")
-
-            obj_list = []
-            prep_num = min(SEARCH_INDEX_BATCH_SIZE, total_comics)
-            self.log.debug(f"Preparing {prep_num} comics for search indexing...")
-            for comic in comics.iterator(chunk_size=SEARCH_INDEX_BATCH_SIZE):
+            self.status_controller.start(status, notify=True)
+            start = 0
+            while start < total_comics:
+                obj_list = []
                 if self.abort_event.is_set():
                     break
-                self._create_comicfts_entry(comic, obj_list, status, create=create)
-                if len(obj_list) >= SEARCH_INDEX_BATCH_SIZE:
-                    self._update_search_index_create_or_update(
-                        obj_list,
-                        status,
-                        create=create,
+                # Not using standard iterator chunking to control memory and really
+                # do this in batches.
+                self.log.debug(
+                    f"Preparing up to {chunk_human_size} comics for search indexing..."
+                )
+                # This query is supposed to get only comics that don't need creating but it doesn't
+                # So the start total exit assists it with that.
+                comics = self._get_operation_comics_query(
+                    comics_filtered_qs, create=create
+                )
+                comics = self._prefetch_related_fts_query(comics)
+                comics = comics.order_by("pk")
+                comics = comics[:search_index_batch_size]
+                comics = self._annotate_fts_query(comics)
+                for comic in comics.values():
+                    SearchEntryPrepare.prepare_sync_fts_entry(
+                        comic, obj_list, create=create
                     )
-                    complete = status.complete or 0
-                    if prep_num := min(
-                        SEARCH_INDEX_BATCH_SIZE, total_comics - complete
-                    ):
-                        self.log.debug(
-                            f"Preparing {prep_num} comics for search indexing..."
-                        )
 
-            self._update_search_index_create_or_update(
-                obj_list,
-                status,
-                create=create,
-            )
+                if not obj_list:
+                    break
+                self._update_search_index_create_or_update(
+                    obj_list,
+                    status,
+                    create=create,
+                )
+                start += search_index_batch_size
 
         finally:
             self.status_controller.finish(status)
@@ -336,25 +234,36 @@ class SearchIndexerSync(SearchIndexerRemove):
 
     def _update_search_index_update(self):
         """Update out of date search entries."""
+        out_of_date_comics = Comic.objects.filter(_ALL_FTS_COMIC_IDS_QUERY)
+        self.log.debug("Looking for search index watermark...")
         fts_watermark = ComicFTS.objects.aggregate(max=Max("updated_at"))["max"]
-        since = fts_watermark or "the fracturing of the multiverse"
+        if fts_watermark:
+            since = fts_watermark
+            out_of_date_comics = out_of_date_comics.filter(updated_at__gt=fts_watermark)
+        else:
+            since = "the fracturing of the multiverse"
+            fts_watermark = fts_watermark or _MIN_UTC_DATE
         self.log.info(f"Looking for search entries to update since {since}...")
-        fts_watermark = fts_watermark or _MIN_UTC_DATE
-        out_of_date_comics = (
-            Comic.objects.select_related("comicfts")
-            .exclude(comicfts__isnull=True)
-            .filter(updated_at__gt=fts_watermark)
-        )
+        count = out_of_date_comics.count()
+        self.log.debug(f"Found {count} comics with out of date search entries.")
+
         return self._update_search_index_operate(out_of_date_comics, create=False)
 
     def _update_search_index_create(self):
         """Create missing search entries."""
         self.log.info("Looking for missing search entries to create...")
-        missing_comics = Comic.objects.filter(comicfts__isnull=True)
+        missing_comics = Comic.objects.all()
+        count = None if ComicFTS.objects.exists() else missing_comics.count()
+        missing_comics = missing_comics.exclude(_ALL_FTS_COMIC_IDS_QUERY)
+        if count is None:
+            count = missing_comics.count()
+
+        self.log.debug(f"Found {count} comics missing from the search index.")
         return self._update_search_index_operate(missing_comics, create=True)
 
     def _update_search_index(self, *, rebuild: bool):
         """Update or Rebuild the search index."""
+        self.log.debug("In update search index before init statii.")
         start_time = time()
         self._init_statuses(rebuild)
 

codex/librarian/status_controller.py

@@ -111,6 +111,29 @@ class StatusController:
             return
         self._update(status, notify=notify)
 
+    @staticmethod
+    def _finish_status_prepare(
+        positive_statii: MappingProxyType[str, Status | type[Status]],
+        updates: dict[str, Any],
+    ):
+        # Filter all active or preactive statii
+        ls_filter = Q(active__isnull=False) | Q(preactive__isnull=False)
+        if positive_statii:
+            ls_filter_dict: dict[str, Any] = {"status_type__in": positive_statii.keys()}
+            # Filter on status codes
+            ls_filter &= Q(**ls_filter_dict)
+        lses = LibrarianStatus.objects.filter(ls_filter)
+        update_ls = []
+        log_statii = []
+        for ls in lses.iterator():
+            for key, value in updates.items():
+                setattr(ls, key, value)
+            update_ls.append(ls)
+            status = positive_statii[ls.status_type]
+            if isinstance(status, Status):
+                log_statii.append(status)
+        return update_ls, log_statii
+
     def _log_finish(self, status: Status, *, clear_subtitle: bool):
         """Log finish of status with stats."""
         level = "INFO"
@@ -140,31 +163,6 @@ class StatusController:
 
         self.log.log(level, f"{prefix}{suffix}.")
 
-    @staticmethod
-    def _finish_status_prepare(
-        positive_statii: MappingProxyType[str, Status | type[Status]],
-        updates: dict[str, Any],
-    ):
-        # Filter on submitted codes or all statii if nothing.
-        ls_filter_dict: dict[str, Any] = (
-            {"status_type__in": positive_statii.keys()} if positive_statii else {}
-        )
-        # Filter all active or preactive statii
-        ls_filter = (Q(active__isnull=False) | Q(preactive__isnull=False)) & Q(
-            **ls_filter_dict
-        )
-        lses = LibrarianStatus.objects.filter(ls_filter)
-        update_ls = []
-        log_statii = []
-        for ls in lses.iterator():
-            for key, value in updates.items():
-                setattr(ls, key, value)
-            update_ls.append(ls)
-            status = positive_statii[ls.status_type]
-            if isinstance(status, Status):
-                log_statii.append(status)
-        return update_ls, log_statii
-
     def finish_many(
         self,
         statii: Iterable[Status | type[Status] | None],
@@ -191,8 +189,8 @@ class StatusController:
                 self.log.info("Cleared all librarian statuses")
             for status in log_statii:
                 self._log_finish(status, clear_subtitle=clear_subtitle)
-        except Exception as exc:
-            self.log.warning(f"Finish status {positive_statii}: {exc}")
+        except Exception:
+            self.log.exception(f"Finish status {positive_statii}")
 
     def finish(
         self, status: Status | None, *, notify: bool = True, clear_subtitle=True

codex/librarian/watchdog/event_batcherd.py

@@ -14,6 +14,7 @@ from types import MappingProxyType
 from typing_extensions import override
 from watchdog.events import EVENT_TYPE_MOVED, FileSystemEvent
 
+from codex.librarian.memory import get_mem_limit
 from codex.librarian.scribe.importer.tasks import ImportTask
 from codex.librarian.threads import AggregateMessageQueuedThread
 from codex.librarian.watchdog.const import EVENT_CLASS_DIFF_ALL_MAP
@@ -21,7 +22,6 @@ from codex.librarian.watchdog.events import (
     EVENT_TYPE_FINISH_POLL,
     EVENT_TYPE_START_POLL,
 )
-from codex.librarian.watchdog.memory import get_mem_limit
 
 _IMPORT_TASK_PARAMS = MappingProxyType(
     {

codex/migrations/0035_fts_optmize.py

@@ -0,0 +1,136 @@
+"""Generated by Django 5.2.4 on 2025-07-21 15:05."""
+
+from itertools import chain
+
+import django.db.models.deletion
+from comicbox.enums.maps.identifiers import ID_SOURCE_NAME_MAP, get_id_source_by_alias
+from django.db import migrations, models
+
+
+def _map_identifiers_to_canonical_names(apps):
+    id_source_model = apps.get_model("codex", "identifiersource")
+    identifier_source_map = {}
+
+    for source in id_source_model.objects.all():
+        if id_source_enum := get_id_source_by_alias(source.name, default=None):
+            canon_name = ID_SOURCE_NAME_MAP.get(id_source_enum, source.name)
+            if canon_name != source.name:
+                if canon_name not in identifier_source_map:
+                    identifier_source_map[canon_name] = set()
+                identifier_source_map[canon_name] |= set(
+                    source.identifier_set.values_list("pk", flat=True)
+                )
+    return identifier_source_map
+
+
+def _prepare_canonical_id_sources(apps, identifier_source_map):
+    id_source_model = apps.get_model("codex", "identifiersource")
+    source_names = identifier_source_map.keys()
+    existing_id_sources = id_source_model.objects.filter(name__in=source_names).only(
+        "name"
+    )
+    existing_id_source_names = id_source_model.objects.filter(
+        name__in=source_names
+    ).values_list("name", flat=True)
+    create_id_source_names = sorted(
+        frozenset(set(identifier_source_map.keys()) - set(existing_id_source_names))
+    )
+    return tuple(
+        id_source_model(name=name) for name in create_id_source_names
+    ), existing_id_sources
+
+
+def _create_link_map(identifier_source_map, sources):
+    link_map = {}
+    for id_source in chain(*sources):
+        for id_pk in identifier_source_map[id_source.name]:
+            link_map[id_pk] = id_source
+    return link_map
+
+
+def _prepare_updatatable_identifiers(apps, identifier_source_map, sources):
+    link_map = _create_link_map(identifier_source_map, sources)
+    identifier_model = apps.get_model("codex", "identifier")
+    updateable_identifiers = identifier_model.objects.filter(pk__in=link_map.keys())
+    obj_list = []
+    for identifier in updateable_identifiers:
+        identifier.source = link_map[identifier.pk]
+        obj_list.append(identifier)
+    return tuple(obj_list)
+
+
+def _create_canonical_sources(apps):
+    print("Examining identifier sources for conversion to canonical sources...")
+    identifier_source_map = _map_identifiers_to_canonical_names(apps)
+    print("Preparing missing canonical identifiers sources for creation...")
+    obj_list, existing_id_sources = _prepare_canonical_id_sources(
+        apps, identifier_source_map
+    )
+    id_source_model = apps.get_model("codex", "identifiersource")
+    created_id_sources = id_source_model.objects.bulk_create(obj_list)
+    count = len(obj_list)
+    print(f"Created {count} canonical identifier sources")
+    return identifier_source_map, (existing_id_sources, created_id_sources)
+
+
+def _update_identifiers_with_canonical_sources(apps, identifier_source_map, sources):
+    print("Preparing identifiers for update with canonical sources...")
+    obj_list = _prepare_updatatable_identifiers(apps, identifier_source_map, sources)
+    identifier_model = apps.get_model("codex", "identifier")
+    identifier_model.objects.bulk_update(obj_list, fields=("source",))
+    count = len(obj_list)
+    print(f"Updated {count} identifiers with canonical sources.")
+
+
+def _convert_identifier_sources(apps, _schema_editor):
+    identifier_source_map, sources = _create_canonical_sources(apps)
+    _update_identifiers_with_canonical_sources(apps, identifier_source_map, sources)
+
+
+class Migration(migrations.Migration):
+    """Migrate db."""
+
+    dependencies = [
+        ("codex", "0034_comicbox2"),
+    ]
+
+    operations = [
+        migrations.RunPython(_convert_identifier_sources),
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunSQL(
+                    sql="DROP TABLE IF EXISTS codex_comicfts;",
+                ),
+                migrations.RunSQL(
+                    sql="CREATE VIRTUAL TABLE codex_comicfts USING fts5(comic_id UNINDEXED, created_at UNINDEXED, updated_at UNINDEXED, publisher, imprint, series, name, collection_title, age_rating, country, language, original_format, review, scan_info, summary, tagger, characters, credits, genres, locations, roles, series_groups, stories, story_arcs, tags, teams, universes, sources)",
+                    reverse_sql="DROP TABLE IF EXISTS codex_comicfts",
+                ),
+            ],
+            state_operations=[
+                migrations.DeleteModel(
+                    name="ComicFTS",
+                ),
+                migrations.CreateModel(
+                    name="ComicFTS",
+                    fields=[
+                        ("created_at", models.DateTimeField(auto_now_add=True)),
+                        ("updated_at", models.DateTimeField(auto_now=True)),
+                        (
+                            "comic",
+                            models.OneToOneField(
+                                on_delete=django.db.models.deletion.CASCADE,
+                                primary_key=True,
+                                serialize=False,
+                                to="codex.comic",
+                            ),
+                        ),
+                        ("body", models.TextField()),
+                    ],
+                    options={
+                        "get_latest_by": "updated_at",
+                        "managed": False,
+                    },
+                ),
+            ],
+        ),
+    ]

codex/models/comic.py

@@ -326,23 +326,12 @@ class Comic(WatchedPathBrowserGroup):
         """Most common text representation for logging."""
         return self.get_title(self, volume=True, name=True, filename_fallback=True)
 
-    def issue(self) -> str:
-        """Combine issue parts for search."""
-        res = ""
-        if self.issue_number is not None:
-            res += str(self.issue_number.normalize())
-        if self.issue_suffix:
-            res += self.issue_suffix
-        return res
-
 
 class ComicFTS(BaseModel):
     comic = OneToOneField(primary_key=True, to=Comic, on_delete=CASCADE)
     # Attributes
     collection_title = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
-    issue = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
     name = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
-    notes = TextField(db_collation="nocase")
     review = TextField(db_collation="nocase")
     summary = TextField(db_collation="nocase")
     # FK groups
@@ -352,7 +341,6 @@ class ComicFTS(BaseModel):
     # FK
     age_rating = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
     country = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
-    file_type = CharField(db_collation="nocase", max_length=max_choices_len(FileType))
     language = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
     original_format = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
     scan_info = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
@@ -361,12 +349,8 @@ class ComicFTS(BaseModel):
     characters = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
     credits = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
     genres = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
-    identifiers = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
     sources = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
     locations = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
-    reading_direction = CharField(
-        db_collation="nocase", max_length=max_choices_len(ReadingDirection)
-    )
     series_groups = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
     stories = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)
     story_arcs = CharField(db_collation="nocase", max_length=MAX_NAME_LEN)