codex 1.6.0__py3-none-any.whl → 1.6.0a0__py3-none-any.whl

codex/librarian/importer/query_fks.py

@@ -1,6 +1,5 @@
 """Query the missing foreign keys for comics and contributors."""
 
-from itertools import chain
 from logging import DEBUG, INFO
 from pathlib import Path
 from types import MappingProxyType
@@ -14,16 +13,6 @@ from codex.librarian.importer.const import (
     DICT_MODEL_CLASS_FIELDS_MAP,
     DICT_MODEL_FIELD_MODEL_MAP,
     DICT_MODEL_REL_MAP,
-    FK_CREATE,
-    FKC_CONTRIBUTORS,
-    FKC_CREATE_FKS,
-    FKC_CREATE_GROUPS,
-    FKC_FOLDER_PATHS,
-    FKC_IDENTIFIERS,
-    FKC_STORY_ARC_NUMBERS,
-    FKC_TOTAL_FKS,
-    FKC_UPDATE_GROUPS,
-    FKS,
     GROUP_COMPARE_FIELDS,
     GROUP_TREES,
     IDENTIFIERS_FIELD_NAME,
@@ -33,8 +22,7 @@ from codex.librarian.importer.const import (
     SERIES,
     STORY_ARCS_METADATA_KEY,
 )
-from codex.librarian.importer.query_covers import QueryCustomCoversImporter
-from codex.librarian.importer.status import ImportStatusTypes
+from codex.librarian.importer.status import ImportStatusTypes, status_notify
 from codex.logger.logging import get_logger
 from codex.models import (
     Comic,
@@ -49,7 +37,10 @@ from codex.models import (
 from codex.models.named import Identifier
 from codex.settings.settings import FILTER_BATCH_SIZE
 from codex.status import Status
+from codex.threads import QueuedThread
 
+_CREATE_GROUPS = "create_groups"
+_UPDATE_GROUPS = "update_groups"
 _CLASS_QUERY_FIELDS_MAP = MappingProxyType(
     {
         Contributor: ("role__name", "person__name"),
@@ -65,23 +56,18 @@ _DEFAULT_QUERY_FIELDS = ("name",)
 _EXTRA_UPDATE_FIELDS_MAP = MappingProxyType(
     {Series: ("volume_count",), Volume: ("issue_count",)}
 )
-_DICT_MODEL_KEY_MAP = MappingProxyType(
-    {
-        CONTRIBUTORS_FIELD_NAME: FKC_CONTRIBUTORS,
-        STORY_ARCS_METADATA_KEY: FKC_STORY_ARC_NUMBERS,
-        IDENTIFIERS_FIELD_NAME: FKC_IDENTIFIERS,
-    }
-)
+
 LOG = get_logger(__name__)
 
 
-class QueryForeignKeysImporter(QueryCustomCoversImporter):
+class QueryForeignKeysMixin(QueuedThread):
     """Methods for querying what fks are missing."""
 
-    def _get_query_fks_totals(self):
+    @staticmethod
+    def _get_query_fks_totals(fks):
         """Get the query foreign keys totals."""
         fks_total = 0
-        for key, objs in self.metadata[FKS].items():
+        for key, objs in fks.items():
             if key == GROUP_TREES:
                 for groups in objs.values():
                     fks_total += len(groups)
@@ -106,11 +92,11 @@ class QueryForeignKeysImporter(QueryCustomCoversImporter):
         update_mds,
         q_obj: Q,
         status,
-        and_q_obj: Q | None = None,
     ):
         """Get create metadata by comparing proposed meatada to existing rows."""
-        vnp = fk_cls._meta.verbose_name_plural.title()
-        status.subtitle = vnp
+        if status:
+            vnp = fk_cls._meta.verbose_name_plural.title()
+            status.subtitle = vnp
         offset = 0
         num_qs = len(q_obj.children)
         while offset < num_qs:
@@ -118,8 +104,6 @@ class QueryForeignKeysImporter(QueryCustomCoversImporter):
             # django.db.utils.OperationalError: Expression tree is too large (maximum depth 1000)
             children_chunk = q_obj.children[offset : offset + FILTER_BATCH_SIZE]
             filter_chunk = Q(*children_chunk, _connector=Q.OR)
-            if and_q_obj:
-                filter_chunk = and_q_obj & filter_chunk
 
             existing_mds = self._query_existing_mds(fk_cls, filter_chunk)
             create_mds.difference_update(existing_mds)
@@ -131,12 +115,13 @@ class QueryForeignKeysImporter(QueryCustomCoversImporter):
                 )
                 update_mds.difference_update(match_with_extra_fields_mds)
 
-            status.add_complete(len(filter_chunk))
+            status.complete += len(filter_chunk)
             self.status_controller.update(status)
 
             offset += FILTER_BATCH_SIZE
 
-        status.subtitle = ""
+        if status:
+            status.subtitle = ""
 
         return num_qs
 
@@ -185,11 +170,11 @@ class QueryForeignKeysImporter(QueryCustomCoversImporter):
         update_group_set.update(update_groups)
         return count
 
+    @classmethod
     def _update_action_group(
-        self, group_cls, action_groups_key, group_tree, count_value: int | None
+        cls, group_cls, action_groups, group_tree, count_value: int | None
     ):
         """Update the create or update group dict with the count dict."""
-        action_groups = self.metadata[FK_CREATE][action_groups_key]
         if group_cls not in action_groups:
             action_groups[group_cls] = {}
         if group_tree not in action_groups[group_cls]:
@@ -202,12 +187,13 @@ class QueryForeignKeysImporter(QueryCustomCoversImporter):
             create_group_set,
             update_group_set,
             group_cls,
+            create_and_update_groups,
             status,
         ) = data
         if group_tree in create_group_set:
             self._update_action_group(
                 group_cls,
-                FKC_CREATE_GROUPS,
+                create_and_update_groups[_CREATE_GROUPS],
                 group_tree,
                 count_value,
             )
@@ -215,11 +201,14 @@ class QueryForeignKeysImporter(QueryCustomCoversImporter):
             # This always updates Volume or Series
             self._update_action_group(
                 group_cls,
-                FKC_UPDATE_GROUPS,
+                create_and_update_groups[_UPDATE_GROUPS],
                 group_tree,
                 count_value,
             )
-        status.add_complete(1)
+        if status:
+            status.complete = status.complete or 0
+            status.complete += 1
+            self.status_controller.update(status)
 
     @staticmethod
     def _prune_group_updates_get_filter(group_cls, update_group_trees):
@@ -290,19 +279,22 @@ class QueryForeignKeysImporter(QueryCustomCoversImporter):
         prune_count = len(update_group_trees) - len(pruned_update_group_trees)
         self.log.debug(f"Pruned {prune_count} {vnp} from updates.")
 
+    @status_notify()
     def _query_missing_group(
         self,
         groups,
         group_cls,
-        status,
+        create_and_update_groups,
+        status=None,
     ):
         """Get missing groups from proposed groups to create."""
         count = 0
         if not groups:
             return count
 
-        vnp = group_cls._meta.verbose_name_plural.title()
-        status.subtitle = vnp
+        if status:
+            vnp = group_cls._meta.verbose_name_plural.title()
+            status.subtitle = vnp
 
         create_group_set = set()
         update_group_set = set()
@@ -314,23 +306,24 @@ class QueryForeignKeysImporter(QueryCustomCoversImporter):
             create_group_set,
             update_group_set,
             group_cls,
+            create_and_update_groups,
             status,
         )
         for group_tree, count_value in groups.items():
             self._query_group_tree(data, group_tree, count_value)
-        self.status_controller.update(status)
 
         # after counts have been tallied, prune the ones that don't need an update.
-        update_groups = self.metadata[FK_CREATE][FKC_UPDATE_GROUPS]
+        update_groups = create_and_update_groups.get(_UPDATE_GROUPS)
         self._prune_group_updates(group_cls, update_groups)
 
-        status.subtitle = ""
+        if status:
+            status.subtitle = ""
 
         create_count = len(
-            self.metadata[FK_CREATE][FKC_CREATE_GROUPS].get(group_cls, {})
+            create_and_update_groups.get(_CREATE_GROUPS, {}).get(group_cls, {})
         )
         update_count = len(
-            self.metadata[FK_CREATE][FKC_UPDATE_GROUPS].get(group_cls, {})
+            create_and_update_groups.get(_UPDATE_GROUPS, {}).get(group_cls, {})
         )
         count = create_count + update_count
         if count:
@@ -419,16 +412,16 @@ class QueryForeignKeysImporter(QueryCustomCoversImporter):
     ):
         """Restore urls to only the identifier create objs."""
         if field_name != "identifiers":
-            return possible_create_objs
+            return
         restored_create_objs = []
         for create_obj in possible_create_objs:
             url = url_restore_map.get(create_obj)
             restored_create_objs.append((*create_obj, url))
-        return restored_create_objs
+        possible_create_objs = restored_create_objs
 
-    def _query_missing_dict_model(self, field_name, create_objs_key, status):
+    def _query_missing_dict_model(self, field_name, fks, create_objs, status):
         """Find missing dict type m2m models."""
-        possible_objs = self.metadata[FKS].pop(field_name, None)
+        possible_objs = fks.pop(field_name, None)
         if not possible_objs:
             return 0
 
@@ -448,51 +441,38 @@ class QueryForeignKeysImporter(QueryCustomCoversImporter):
             )
 
         # Build combined query object from the value_filter
-        model = DICT_MODEL_FIELD_MODEL_MAP[field_name]
-        if model == Identifier:
-            for filter_and_prefix, value_filter in query_filter_map.items():
-                self._query_create_metadata(
-                    model,
-                    possible_create_objs,
-                    None,
-                    value_filter,
-                    status,
-                    and_q_obj=filter_and_prefix,
-                )
-        else:
-            combined_q_obj = Q()
-            for filter_and_prefix, value_filter in query_filter_map.items():
-                combined_q_obj |= filter_and_prefix & value_filter
-            self._query_create_metadata(
-                model,
-                possible_create_objs,
-                None,
-                combined_q_obj,
-                status,
-            )
+        combined_q_obj = Q()
+        for filter_and_prefix, value_filter in query_filter_map.items():
+            combined_q_obj |= filter_and_prefix & value_filter
 
         # Finally run the query and get only the correct create_objs
-        possible_create_objs = self._query_missing_dict_model_identifiers_restore_urls(
+        model = DICT_MODEL_FIELD_MODEL_MAP[field_name]
+        self._query_create_metadata(
+            model, possible_create_objs, None, combined_q_obj, status
+        )
+        self._query_missing_dict_model_identifiers_restore_urls(
             field_name, possible_create_objs, url_restore_map
         )
 
         # Final Cleanup
-        self.metadata[FK_CREATE][create_objs_key].update(possible_create_objs)
-        count = len(self.metadata[FK_CREATE][create_objs_key])
+        create_objs.update(possible_create_objs)
+        count = len(create_objs)
         if count:
             model = DICT_MODEL_FIELD_MODEL_MAP[field_name]
             vnp = model._meta.verbose_name_plural
             vnp = vnp.title() if vnp else "Nothings"
             self.log.info(f"Prepared {count} new {vnp}.")
-        status.add_complete(count)
-        self.status_controller.update(status, notify=False)
+        if status:
+            status.complete = status.complete or 0
+            status.complete += count
         return count
 
     ##########
     # SIMPLE #
     ##########
 
-    def _query_missing_simple_models(self, names, fk_data, status):
+    @status_notify()
+    def _query_missing_simple_models(self, names, fk_data, status=None):
         """Find missing named models and folders."""
         # count = 0
         if not names:
@@ -506,8 +486,9 @@ class QueryForeignKeysImporter(QueryCustomCoversImporter):
         create_names = set(names)
         num_proposed_names = len(proposed_names)
 
-        vnp = fk_cls._meta.verbose_name_plural.title()
-        status.subtitle = vnp
+        if status:
+            vnp = fk_cls._meta.verbose_name_plural.title()
+            status.subtitle = vnp
         while start < num_proposed_names:
             # Do this in batches so as not to exceed the 1k line sqlite limit
             end = start + FILTER_BATCH_SIZE
@@ -516,8 +497,11 @@ class QueryForeignKeysImporter(QueryCustomCoversImporter):
             fk_filter = Q(**filter_args)
             create_names -= self._query_existing_mds(fk_cls, fk_filter)
             num_in_batch = len(batch_proposed_names)
-            status.add_complete(num_in_batch)
-            self.status_controller.update(status)
+            # count += num_in_batch
+            if status:
+                status.complete = status.complete or 0
+                status.complete += num_in_batch
+                self.status_controller.update(status)
             start += FILTER_BATCH_SIZE
 
         if status:
@@ -534,7 +518,7 @@ class QueryForeignKeysImporter(QueryCustomCoversImporter):
         self.log.log(level, f"Prepared {len(create_names)} new {vnp}.")
         return len(names)
 
-    def _query_one_simple_model(self, fk_field, status):
+    def _query_one_simple_model(self, fk_field, fks, create_fks, status):
         """Batch query one simple model name."""
         for cls, names in DICT_MODEL_CLASS_FIELDS_MAP.items():
             if fk_field in names:
@@ -542,24 +526,25 @@ class QueryForeignKeysImporter(QueryCustomCoversImporter):
                 break
         else:
             base_cls = Comic
-        fk_data = self.metadata[FK_CREATE][FKC_CREATE_FKS], base_cls, fk_field, "name"
-        names = self.metadata[FKS].pop(fk_field)
-        status.add_complete(
-            self._query_missing_simple_models(
-                names,
-                fk_data,
-                status,
-            )
+        fk_data = create_fks, base_cls, fk_field, "name"
+        names = fks.pop(fk_field)
+        status.complete += self._query_missing_simple_models(
+            names,
+            fk_data,
+            status=status,
         )
-        self.status_controller.update(status, notify=False)
 
     ###########
     # FOLDERS #
     ###########
-    def query_missing_folder_paths(self, comic_paths, status):
+
+    @status_notify()
+    def query_missing_folder_paths(
+        self, comic_paths, library_path, create_folder_paths, status=None
+    ):
         """Find missing folder paths."""
         # Get the proposed folder_paths
-        library_path = Path(self.library.path)
+        library_path = Path(library_path)
         proposed_folder_paths = set()
         for comic_path in comic_paths:
             for path in Path(comic_path).parents:
@@ -572,79 +557,74 @@ class QueryForeignKeysImporter(QueryCustomCoversImporter):
         self._query_missing_simple_models(
             proposed_folder_paths,
             fk_data,
-            status,
+            status=status,
         )
-        create_folder_paths = create_folder_paths_dict.get(Folder, set())
-        if FK_CREATE not in self.metadata:
-            self.metadata[FK_CREATE] = {}
-        if FKC_FOLDER_PATHS not in self.metadata[FK_CREATE]:
-            self.metadata[FK_CREATE][FKC_FOLDER_PATHS] = set()
-        self.metadata[FK_CREATE][FKC_FOLDER_PATHS] |= create_folder_paths
+        folder_paths_set = create_folder_paths_dict.get(Folder, set())
+        create_folder_paths |= folder_paths_set
         count = len(create_folder_paths)
         if count:
             self.log.info(f"Prepared {count} new Folders.")
         return count
 
-    def _get_create_fks_totals(self):
-        fkc = self.metadata[FK_CREATE]
-        total_fks = 0
-        for data_group in chain(
-            fkc[FKC_CREATE_GROUPS].values(),
-            fkc[FKC_UPDATE_GROUPS].values(),
-            fkc[FKC_CREATE_FKS].values(),
-        ):
-            total_fks += len(data_group)
-        total_fks += (
-            len(fkc[FKC_FOLDER_PATHS])
-            + len(fkc[FKC_CONTRIBUTORS])
-            + len(fkc[FKC_STORY_ARC_NUMBERS])
-            + len(fkc[FKC_IDENTIFIERS])
-        )
-        return total_fks
-
-    def query_all_missing_fks(self):
+    def query_all_missing_fks(self, library_path, fks):
         """Get objects to create by querying existing objects for the proposed fks."""
-        if FKS not in self.metadata:
-            return
-        self.metadata[FK_CREATE] = {
-            FKC_CONTRIBUTORS: set(),
-            FKC_STORY_ARC_NUMBERS: set(),
-            FKC_IDENTIFIERS: set(),
-            FKC_CREATE_GROUPS: {},
-            FKC_UPDATE_GROUPS: {},
-            FKC_FOLDER_PATHS: set(),
-            FKC_CREATE_FKS: {},
-        }
-        self.log.debug(
-            f"Querying existing foreign keys for comics in {self.library.path}"
+        create_contributors = set()
+        create_story_arc_numbers = set()
+        create_identifiers = set()
+        create_groups = {}
+        update_groups = {}
+        create_folder_paths = set()
+        create_fks = {}
+        self.log.debug(f"Querying existing foreign keys for comics in {library_path}")
+        fks_total = self._get_query_fks_totals(fks)
+        dict_model_map = MappingProxyType(
+            {
+                CONTRIBUTORS_FIELD_NAME: create_contributors,
+                STORY_ARCS_METADATA_KEY: create_story_arc_numbers,
+                IDENTIFIERS_FIELD_NAME: create_identifiers,
+            }
         )
-        status = Status(ImportStatusTypes.QUERY_MISSING_FKS)
+        status = Status(ImportStatusTypes.QUERY_MISSING_FKS, 0, fks_total)
         try:
             self.status_controller.start(status)
-            for field_name, create_objs_key in _DICT_MODEL_KEY_MAP.items():
+            for field_name, create_objs in dict_model_map.items():
                 self._query_missing_dict_model(
                     field_name,
-                    create_objs_key,
+                    fks,
+                    create_objs,
                     status,
                 )
-            for group_class, groups in self.metadata[FKS].pop(GROUP_TREES, {}).items():
+
+            create_and_update_groups = {
+                _CREATE_GROUPS: create_groups,
+                _UPDATE_GROUPS: update_groups,
+            }
+            for group_class, groups in fks.pop(GROUP_TREES, {}).items():
                 self._query_missing_group(
                     groups,
                     group_class,
-                    status,
+                    create_and_update_groups,
+                    status=status,
                 )
 
             self.query_missing_folder_paths(
-                self.metadata[FKS].pop(COMIC_PATHS, ()), status
+                fks.pop(COMIC_PATHS, ()),
+                library_path,
+                create_folder_paths,
+                status=status,
             )
 
-            for fk_field in sorted(self.metadata[FKS].keys()):
-                self._query_one_simple_model(fk_field, status)
-            self.metadata.pop(FKS)
+            for fk_field in sorted(fks.keys()):
+                self._query_one_simple_model(fk_field, fks, create_fks, status)
         finally:
             self.status_controller.finish(status)
 
-        total_fks = self._get_create_fks_totals()
-        status = Status(ImportStatusTypes.CREATE_FKS, 0, total_fks)
-        self.status_controller.update(status, notify=False)
-        self.metadata[FK_CREATE][FKC_TOTAL_FKS] = total_fks
+        return (
+            create_groups,
+            update_groups,
+            create_folder_paths,
+            create_fks,
+            create_contributors,
+            create_story_arc_numbers,
+            create_identifiers,
+        )

codex/librarian/importer/status.py

@@ -1,7 +1,13 @@
 """Librarian Status for imports."""
 
+from collections.abc import Callable
+from enum import Enum
+from time import time
+
 from django.db.models import Choices
 
+from codex.status import Status
+
 
 class ImportStatusTypes(Choices):
     """Keys for Import tasks."""
@@ -19,11 +25,36 @@ class ImportStatusTypes(Choices):
     DIRS_DELETED = "IDD"
     FILES_DELETED = "IFD"
     FAILED_IMPORTS = "IFI"
-    QUERY_MISSING_COVERS = "ICQ"
-    COVERS_MOVED = "ICM"
-    COVERS_MODIFIED = "ICU"
-    COVERS_CREATED = "ICC"
-    COVERS_DELETED = "ICD"
-    COVERS_LINK = "ICL"
-    GROUP_UPDATE = "IGU"
-    ADOPT_FOLDERS = "IAF"
+
+
+def status_notify(status_type: str | Enum = "", updates=True):
+    """Wrap a function with status changes."""
+    # https://stackoverflow.com/questions/5929107/decorators-with-parameters
+
+    def decorator(func) -> Callable[..., int]:
+        def wrapper(self, data, *args, status=None, **kwargs) -> int:
+            """Run a function bracketed by status changes."""
+            num_elements = len(data)
+            if not num_elements:
+                return 0
+
+            if status:
+                finish = False
+            else:
+                complete = 0 if updates else None
+                status = Status(status_type, complete, num_elements, time())
+                self.status_controller.start(status)
+                finish = True
+
+            kwargs["status"] = status
+            try:
+                count = func(self, data, *args, **kwargs)
+            finally:
+                if finish:
+                    self.status_controller.finish(status)
+
+            return count
+
+        return wrapper
+
+    return decorator

codex/librarian/importer/tasks.py

@@ -1,8 +1,6 @@
 """DB Import Tasks."""
 
-from collections.abc import Mapping
-from dataclasses import dataclass, field
-from datetime import datetime
+from dataclasses import dataclass
 
 
 @dataclass
@@ -15,22 +13,14 @@ class ImportDBDiffTask(ImportTask):
     """For sending to the updater."""
 
     library_id: int
-
-    dirs_moved: Mapping[str, str] = field(default_factory=dict)
-    dirs_modified: frozenset[str] = frozenset()
-    # dirs_created: frozenset[str] | None = frozenset()
-    dirs_deleted: frozenset[str] = frozenset()
-
-    files_moved: Mapping[str, str] = field(default_factory=dict)
-    files_modified: frozenset[str] = frozenset()
-    files_created: frozenset[str] = frozenset()
-    files_deleted: frozenset[str] = frozenset()
-
-    covers_moved: Mapping[str, str] = field(default_factory=dict)
-    covers_modified: frozenset[str] = frozenset()
-    covers_created: frozenset[str] = frozenset()
-    covers_deleted: frozenset[str] = frozenset()
-
+    dirs_moved: dict[str, str]
+    files_moved: dict[str, str]
+    dirs_modified: frozenset[str]
+    files_modified: frozenset[str]
+    # dirs_created
+    files_created: frozenset[str]
+    dirs_deleted: frozenset[str]
+    files_deleted: frozenset[str]
     force_import_metadata: bool = False
 
 
@@ -44,10 +34,3 @@ class LazyImportComicsTask(ImportTask):
 @dataclass
 class AdoptOrphanFoldersTask(ImportTask):
     """Move orphaned folders into a correct tree position."""
-
-
-@dataclass
-class UpdateGroupsTask(ImportTask):
-    """Force the update of group timestamp."""
-
-    start_time: datetime | None = None