codex 1.4.0a0__py3-none-any.whl → 1.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of codex might be problematic. Click here for more details.
- codex/config_default.yaml +12 -4
- codex/db_functions.py +4 -2
- codex/integrity.py +17 -6
- codex/librarian/covers/coverd.py +3 -11
- codex/librarian/covers/create.py +15 -21
- codex/librarian/covers/tasks.py +2 -16
- codex/librarian/importer/aggregate_metadata.py +75 -41
- codex/librarian/importer/clean_metadata.py +30 -7
- codex/librarian/importer/create_fks.py +154 -55
- codex/librarian/importer/deleted.py +11 -2
- codex/librarian/importer/failed_imports.py +44 -5
- codex/librarian/importer/importerd.py +37 -12
- codex/librarian/importer/link_comics.py +54 -31
- codex/librarian/importer/moved.py +55 -11
- codex/librarian/importer/query_fks.py +210 -48
- codex/librarian/importer/tasks.py +7 -7
- codex/librarian/janitor/cleanup.py +17 -5
- codex/librarian/librariand.py +10 -0
- codex/librarian/watchdog/events.py +11 -14
- codex/librarian/watchdog/observers.py +5 -1
- codex/logger/loggerd.py +7 -3
- codex/logger/logging.py +1 -1
- codex/migrations/0024_comic_gtin_comic_story_arc_number.py +24 -0
- codex/migrations/0025_add_story_arc_number.py +83 -0
- codex/models.py +21 -11
- codex/search/backend.py +1 -1
- codex/search/indexes.py +1 -1
- codex/serializers/browser.py +1 -0
- codex/serializers/metadata.py +5 -1
- codex/serializers/models.py +16 -1
- codex/serializers/opds/v1.py +1 -0
- codex/serializers/opds/v2.py +5 -2
- codex/serializers/reader.py +55 -16
- codex/settings/settings.py +1 -1
- codex/static_root/assets/admin-12749881.ef0f50bac290.js +41 -0
- codex/static_root/assets/admin-12749881.ef0f50bac290.js.br +0 -0
- codex/static_root/assets/admin-12749881.ef0f50bac290.js.gz +0 -0
- codex/static_root/assets/admin-12749881.js +41 -0
- codex/static_root/assets/admin-12749881.js.br +0 -0
- codex/static_root/assets/admin-12749881.js.gz +0 -0
- codex/static_root/assets/admin-beda768d.a614eee46307.css +1 -0
- codex/static_root/assets/admin-beda768d.a614eee46307.css.br +0 -0
- codex/static_root/assets/admin-beda768d.a614eee46307.css.gz +0 -0
- codex/static_root/assets/admin-beda768d.css +1 -0
- codex/static_root/assets/admin-beda768d.css.br +0 -0
- codex/static_root/assets/admin-beda768d.css.gz +0 -0
- codex/static_root/assets/admin-drawer-panel-41c225cc.3f84583b435b.css +1 -0
- codex/static_root/assets/admin-drawer-panel-41c225cc.3f84583b435b.css.br +0 -0
- codex/static_root/assets/admin-drawer-panel-41c225cc.3f84583b435b.css.gz +0 -0
- codex/static_root/assets/admin-drawer-panel-41c225cc.css +1 -0
- codex/static_root/assets/admin-drawer-panel-41c225cc.css.br +0 -0
- codex/static_root/assets/admin-drawer-panel-41c225cc.css.gz +0 -0
- codex/static_root/assets/admin-drawer-panel-522f1e6c.089d70878270.js +1 -0
- codex/static_root/assets/admin-drawer-panel-522f1e6c.089d70878270.js.br +0 -0
- codex/static_root/assets/admin-drawer-panel-522f1e6c.089d70878270.js.gz +0 -0
- codex/static_root/assets/admin-drawer-panel-522f1e6c.js +1 -0
- codex/static_root/assets/admin-drawer-panel-522f1e6c.js.br +0 -0
- codex/static_root/assets/admin-drawer-panel-522f1e6c.js.gz +0 -0
- codex/static_root/assets/browser-7f7d7134.0fe3749b0f2f.css +1 -0
- codex/static_root/assets/browser-7f7d7134.0fe3749b0f2f.css.br +0 -0
- codex/static_root/assets/browser-7f7d7134.0fe3749b0f2f.css.gz +0 -0
- codex/static_root/assets/browser-7f7d7134.css +1 -0
- codex/static_root/assets/browser-7f7d7134.css.br +0 -0
- codex/static_root/assets/browser-7f7d7134.css.gz +0 -0
- codex/static_root/assets/browser-af622672.d51aca96d64d.js +1 -0
- codex/static_root/assets/browser-af622672.d51aca96d64d.js.br +0 -0
- codex/static_root/assets/browser-af622672.d51aca96d64d.js.gz +0 -0
- codex/static_root/assets/browser-af622672.js +1 -0
- codex/static_root/assets/browser-af622672.js.br +0 -0
- codex/static_root/assets/browser-af622672.js.gz +0 -0
- codex/static_root/assets/http-error-5e17b794.77ceeb2d4641.js +1 -0
- codex/static_root/assets/http-error-5e17b794.77ceeb2d4641.js.br +0 -0
- codex/static_root/assets/http-error-5e17b794.77ceeb2d4641.js.gz +0 -0
- codex/static_root/assets/http-error-5e17b794.js +1 -0
- codex/static_root/assets/http-error-5e17b794.js.br +0 -0
- codex/static_root/assets/http-error-5e17b794.js.gz +0 -0
- codex/static_root/assets/{main-a6ac9581.2fd9e52cbcc3.css → main-0898f4bb.181e0145c642.css} +1 -1
- codex/static_root/assets/main-0898f4bb.181e0145c642.css.br +0 -0
- codex/static_root/assets/{main-a6ac9581.2fd9e52cbcc3.css.gz → main-0898f4bb.181e0145c642.css.gz} +0 -0
- codex/static_root/assets/{main-a6ac9581.css → main-0898f4bb.css} +1 -1
- codex/static_root/assets/main-0898f4bb.css.br +0 -0
- codex/static_root/assets/{main-a6ac9581.css.gz → main-0898f4bb.css.gz} +0 -0
- codex/static_root/assets/main-9e76a4c3.6844a407d14c.js +1 -0
- codex/static_root/assets/main-9e76a4c3.6844a407d14c.js.br +0 -0
- codex/static_root/assets/main-9e76a4c3.6844a407d14c.js.gz +0 -0
- codex/static_root/assets/main-9e76a4c3.js +1 -0
- codex/static_root/assets/main-9e76a4c3.js.br +0 -0
- codex/static_root/assets/main-9e76a4c3.js.gz +0 -0
- codex/static_root/assets/metadata-dialog-62c29ce0.8418785c0453.js +1 -0
- codex/static_root/assets/metadata-dialog-62c29ce0.8418785c0453.js.br +0 -0
- codex/static_root/assets/metadata-dialog-62c29ce0.8418785c0453.js.gz +0 -0
- codex/static_root/assets/metadata-dialog-62c29ce0.js +1 -0
- codex/static_root/assets/metadata-dialog-62c29ce0.js.br +0 -0
- codex/static_root/assets/metadata-dialog-62c29ce0.js.gz +0 -0
- codex/static_root/assets/{metadata-dialog-785c4cfc.694a251cda37.css → metadata-dialog-cb306ffd.cc304996d7bb.css} +1 -1
- codex/static_root/assets/metadata-dialog-cb306ffd.cc304996d7bb.css.br +0 -0
- codex/static_root/assets/metadata-dialog-cb306ffd.cc304996d7bb.css.gz +0 -0
- codex/static_root/assets/{metadata-dialog-785c4cfc.css → metadata-dialog-cb306ffd.css} +1 -1
- codex/static_root/assets/metadata-dialog-cb306ffd.css.br +0 -0
- codex/static_root/assets/metadata-dialog-cb306ffd.css.gz +0 -0
- codex/static_root/assets/{page-pdf-abfd509d.3870dab8eaf4.js → page-pdf-157ba97e.613d7c2beb77.js} +61 -51
- codex/static_root/assets/page-pdf-157ba97e.613d7c2beb77.js.br +0 -0
- codex/static_root/assets/page-pdf-157ba97e.613d7c2beb77.js.gz +0 -0
- codex/static_root/assets/{page-pdf-abfd509d.js → page-pdf-157ba97e.js} +61 -51
- codex/static_root/assets/page-pdf-157ba97e.js.br +0 -0
- codex/static_root/assets/page-pdf-157ba97e.js.gz +0 -0
- codex/static_root/assets/reader-36266549.0b2cf1291f27.js +1 -0
- codex/static_root/assets/reader-36266549.0b2cf1291f27.js.br +0 -0
- codex/static_root/assets/reader-36266549.0b2cf1291f27.js.gz +0 -0
- codex/static_root/assets/reader-36266549.js +1 -0
- codex/static_root/assets/reader-36266549.js.br +0 -0
- codex/static_root/assets/reader-36266549.js.gz +0 -0
- codex/static_root/assets/reader-7f004141.506eecc6954b.css +1 -0
- codex/static_root/assets/reader-7f004141.506eecc6954b.css.br +0 -0
- codex/static_root/assets/reader-7f004141.506eecc6954b.css.gz +0 -0
- codex/static_root/assets/reader-7f004141.css +1 -0
- codex/static_root/assets/reader-7f004141.css.br +0 -0
- codex/static_root/assets/reader-7f004141.css.gz +0 -0
- codex/static_root/js/choices-admin.24cecf0a0568.json +1 -0
- codex/static_root/js/choices-admin.24cecf0a0568.json.br +0 -0
- codex/static_root/js/choices-admin.24cecf0a0568.json.gz +0 -0
- codex/static_root/js/choices-admin.json +1 -1
- codex/static_root/js/choices-admin.json.br +0 -0
- codex/static_root/js/choices-admin.json.gz +0 -0
- codex/static_root/js/choices.8c58714cf5b2.json +1 -0
- codex/static_root/js/choices.8c58714cf5b2.json.br +5 -0
- codex/static_root/js/choices.8c58714cf5b2.json.gz +0 -0
- codex/static_root/js/choices.json +1 -1
- codex/static_root/js/choices.json.br +0 -0
- codex/static_root/js/choices.json.gz +0 -0
- codex/static_root/{manifest.64a989215af8.json → manifest.d2f93a519ada.json} +34 -34
- codex/static_root/manifest.d2f93a519ada.json.br +0 -0
- codex/static_root/manifest.d2f93a519ada.json.gz +0 -0
- codex/static_root/manifest.json +34 -34
- codex/static_root/manifest.json.br +0 -0
- codex/static_root/manifest.json.gz +0 -0
- codex/static_root/staticfiles.json +1 -1
- codex/templates/headers-script-globals.html +1 -1
- codex/templates/{opds → opds_v1}/index.xml +3 -1
- codex/templates/{opds/opensearch.xml → opds_v1/opensearch_v1.xml} +1 -1
- codex/templates/search/indexes/codex/comic_text.txt +2 -2
- codex/urls/converters.py +1 -1
- codex/urls/opds/authentication.py +1 -1
- codex/urls/opds/root.py +8 -12
- codex/urls/opds/v1.py +12 -5
- codex/urls/opds/v2.py +2 -2
- codex/views/admin/tasks.py +6 -1
- codex/views/bookmark.py +2 -2
- codex/views/browser/base.py +23 -7
- codex/views/browser/browser.py +66 -56
- codex/views/browser/browser_annotations.py +159 -50
- codex/views/browser/browser_order_by.py +51 -105
- codex/views/browser/choices.py +75 -38
- codex/views/browser/filters/bookmark.py +6 -9
- codex/views/browser/filters/field.py +9 -6
- codex/views/browser/filters/group.py +12 -27
- codex/views/browser/filters/search.py +5 -10
- codex/views/browser/metadata.py +44 -19
- codex/views/download.py +1 -1
- codex/views/frontend.py +2 -3
- codex/views/mixins.py +15 -2
- codex/views/opds/const.py +8 -1
- codex/views/opds/util.py +37 -1
- codex/views/opds/v1/__init__.py +1 -1
- codex/views/opds/v1/data.py +21 -0
- codex/views/opds/v1/entry/__init__.py +1 -0
- codex/views/opds/v1/entry/data.py +23 -0
- codex/views/opds/v1/entry/entry.py +151 -0
- codex/views/opds/v1/entry/links.py +135 -0
- codex/views/opds/v1/facets.py +190 -0
- codex/views/opds/v1/feed.py +199 -0
- codex/views/opds/v1/links.py +198 -0
- codex/views/opds/{opensearch.py → v1/opensearch_v1.py} +3 -3
- codex/views/opds/v2/__init__.py +1 -1
- codex/views/opds/v2/const.py +10 -2
- codex/views/opds/v2/feed.py +82 -21
- codex/views/opds/v2/links.py +1 -1
- codex/views/opds/v2/publications.py +1 -1
- codex/views/opds/v2/top_links.py +1 -1
- codex/views/reader/page.py +6 -7
- codex/views/reader/reader.py +191 -61
- codex/views/session.py +2 -1
- {codex-1.4.0a0.dist-info → codex-1.4.1.dist-info}/METADATA +10 -41
- {codex-1.4.0a0.dist-info → codex-1.4.1.dist-info}/RECORD +187 -185
- codex/librarian/importer/db_ops.py +0 -248
- codex/pdf.py +0 -115
- codex/static_root/assets/admin-73d93dc7.2c3eb62e50a0.js +0 -48
- codex/static_root/assets/admin-73d93dc7.2c3eb62e50a0.js.br +0 -0
- codex/static_root/assets/admin-73d93dc7.2c3eb62e50a0.js.gz +0 -0
- codex/static_root/assets/admin-73d93dc7.js +0 -48
- codex/static_root/assets/admin-73d93dc7.js.br +0 -0
- codex/static_root/assets/admin-73d93dc7.js.gz +0 -0
- codex/static_root/assets/admin-79555229.5f2c4cb3a73c.css +0 -1
- codex/static_root/assets/admin-79555229.5f2c4cb3a73c.css.br +0 -0
- codex/static_root/assets/admin-79555229.5f2c4cb3a73c.css.gz +0 -0
- codex/static_root/assets/admin-79555229.css +0 -1
- codex/static_root/assets/admin-79555229.css.br +0 -0
- codex/static_root/assets/admin-79555229.css.gz +0 -0
- codex/static_root/assets/admin-drawer-panel-64bcc083.a85324c9ccd8.js +0 -1
- codex/static_root/assets/admin-drawer-panel-64bcc083.a85324c9ccd8.js.br +0 -0
- codex/static_root/assets/admin-drawer-panel-64bcc083.a85324c9ccd8.js.gz +0 -0
- codex/static_root/assets/admin-drawer-panel-64bcc083.js +0 -1
- codex/static_root/assets/admin-drawer-panel-64bcc083.js.br +0 -0
- codex/static_root/assets/admin-drawer-panel-64bcc083.js.gz +0 -0
- codex/static_root/assets/admin-drawer-panel-cce8c0aa.2c0814fa2a9b.css +0 -1
- codex/static_root/assets/admin-drawer-panel-cce8c0aa.2c0814fa2a9b.css.br +0 -2
- codex/static_root/assets/admin-drawer-panel-cce8c0aa.2c0814fa2a9b.css.gz +0 -0
- codex/static_root/assets/admin-drawer-panel-cce8c0aa.css +0 -1
- codex/static_root/assets/admin-drawer-panel-cce8c0aa.css.br +0 -2
- codex/static_root/assets/admin-drawer-panel-cce8c0aa.css.gz +0 -0
- codex/static_root/assets/browser-7325db61.css +0 -1
- codex/static_root/assets/browser-7325db61.css.br +0 -0
- codex/static_root/assets/browser-7325db61.css.gz +0 -0
- codex/static_root/assets/browser-7325db61.ed2cfbf8e8ee.css +0 -1
- codex/static_root/assets/browser-7325db61.ed2cfbf8e8ee.css.br +0 -0
- codex/static_root/assets/browser-7325db61.ed2cfbf8e8ee.css.gz +0 -0
- codex/static_root/assets/browser-d2caeed7.2262000a6d55.js +0 -1
- codex/static_root/assets/browser-d2caeed7.2262000a6d55.js.br +0 -0
- codex/static_root/assets/browser-d2caeed7.2262000a6d55.js.gz +0 -0
- codex/static_root/assets/browser-d2caeed7.js +0 -1
- codex/static_root/assets/browser-d2caeed7.js.br +0 -0
- codex/static_root/assets/browser-d2caeed7.js.gz +0 -0
- codex/static_root/assets/http-error-0221c37d.480d5066da92.js +0 -1
- codex/static_root/assets/http-error-0221c37d.480d5066da92.js.br +0 -0
- codex/static_root/assets/http-error-0221c37d.480d5066da92.js.gz +0 -0
- codex/static_root/assets/http-error-0221c37d.js +0 -1
- codex/static_root/assets/http-error-0221c37d.js.br +0 -0
- codex/static_root/assets/http-error-0221c37d.js.gz +0 -0
- codex/static_root/assets/main-a6ac9581.2fd9e52cbcc3.css.br +0 -0
- codex/static_root/assets/main-a6ac9581.css.br +0 -0
- codex/static_root/assets/main-e33dcfb0.a65044fc1a08.js +0 -1
- codex/static_root/assets/main-e33dcfb0.a65044fc1a08.js.br +0 -0
- codex/static_root/assets/main-e33dcfb0.a65044fc1a08.js.gz +0 -0
- codex/static_root/assets/main-e33dcfb0.js +0 -1
- codex/static_root/assets/main-e33dcfb0.js.br +0 -0
- codex/static_root/assets/main-e33dcfb0.js.gz +0 -0
- codex/static_root/assets/metadata-dialog-785c4cfc.694a251cda37.css.br +0 -0
- codex/static_root/assets/metadata-dialog-785c4cfc.694a251cda37.css.gz +0 -0
- codex/static_root/assets/metadata-dialog-785c4cfc.css.br +0 -0
- codex/static_root/assets/metadata-dialog-785c4cfc.css.gz +0 -0
- codex/static_root/assets/metadata-dialog-8b0e8aaa.d12b42b1c9da.js +0 -1
- codex/static_root/assets/metadata-dialog-8b0e8aaa.d12b42b1c9da.js.br +0 -0
- codex/static_root/assets/metadata-dialog-8b0e8aaa.d12b42b1c9da.js.gz +0 -0
- codex/static_root/assets/metadata-dialog-8b0e8aaa.js +0 -1
- codex/static_root/assets/metadata-dialog-8b0e8aaa.js.br +0 -0
- codex/static_root/assets/metadata-dialog-8b0e8aaa.js.gz +0 -0
- codex/static_root/assets/page-pdf-abfd509d.3870dab8eaf4.js.br +0 -0
- codex/static_root/assets/page-pdf-abfd509d.3870dab8eaf4.js.gz +0 -0
- codex/static_root/assets/page-pdf-abfd509d.js.br +0 -0
- codex/static_root/assets/page-pdf-abfd509d.js.gz +0 -0
- codex/static_root/assets/reader-a8b8f766.875abdd0d22e.css +0 -1
- codex/static_root/assets/reader-a8b8f766.875abdd0d22e.css.br +0 -0
- codex/static_root/assets/reader-a8b8f766.875abdd0d22e.css.gz +0 -0
- codex/static_root/assets/reader-a8b8f766.css +0 -1
- codex/static_root/assets/reader-a8b8f766.css.br +0 -0
- codex/static_root/assets/reader-a8b8f766.css.gz +0 -0
- codex/static_root/assets/reader-fe9345d2.759c31f82998.js +0 -1
- codex/static_root/assets/reader-fe9345d2.759c31f82998.js.br +0 -0
- codex/static_root/assets/reader-fe9345d2.759c31f82998.js.gz +0 -0
- codex/static_root/assets/reader-fe9345d2.js +0 -1
- codex/static_root/assets/reader-fe9345d2.js.br +0 -0
- codex/static_root/assets/reader-fe9345d2.js.gz +0 -0
- codex/static_root/js/choices-admin.3d958ea7f83b.json +0 -1
- codex/static_root/js/choices-admin.3d958ea7f83b.json.br +0 -0
- codex/static_root/js/choices-admin.3d958ea7f83b.json.gz +0 -0
- codex/static_root/js/choices.6bfc2a3d293f.json +0 -1
- codex/static_root/js/choices.6bfc2a3d293f.json.br +0 -0
- codex/static_root/js/choices.6bfc2a3d293f.json.gz +0 -0
- codex/static_root/manifest.64a989215af8.json.br +0 -0
- codex/static_root/manifest.64a989215af8.json.gz +0 -0
- codex/urls/opds/opensearch.py +0 -18
- codex/views/opds/v1/browser.py +0 -346
- codex/views/opds/v1/entry.py +0 -278
- codex/views/opds/v1/start.py +0 -28
- codex/views/opds/v1/util.py +0 -162
- codex/views/opds/v2/start.py +0 -28
- {codex-1.4.0a0.dist-info → codex-1.4.1.dist-info}/LICENSE +0 -0
- {codex-1.4.0a0.dist-info → codex-1.4.1.dist-info}/WHEEL +0 -0
- {codex-1.4.0a0.dist-info → codex-1.4.1.dist-info}/entry_points.txt +0 -0
|
@@ -4,20 +4,23 @@ from pathlib import Path
|
|
|
4
4
|
from django.db.models.functions import Now
|
|
5
5
|
|
|
6
6
|
from codex.librarian.importer.create_comics import CreateComicsMixin
|
|
7
|
-
from codex.librarian.importer.create_fks import
|
|
7
|
+
from codex.librarian.importer.create_fks import (
|
|
8
|
+
BULK_UPDATE_FOLDER_MODIFIED_FIELDS,
|
|
9
|
+
CreateForeignKeysMixin,
|
|
10
|
+
)
|
|
8
11
|
from codex.librarian.importer.query_fks import QueryForeignKeysMixin
|
|
9
12
|
from codex.librarian.importer.status import ImportStatusTypes, status_notify
|
|
10
13
|
from codex.models import Comic, Folder, Library
|
|
11
14
|
|
|
12
|
-
|
|
13
|
-
|
|
15
|
+
_MOVED_BULK_COMIC_UPDATE_FIELDS = ("path", "parent_folder")
|
|
16
|
+
_MOVED_BULK_FOLDER_UPDATE_FIELDS = ("path", "parent_folder", "name")
|
|
14
17
|
|
|
15
18
|
|
|
16
19
|
class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixin):
|
|
17
20
|
"""Methods for moving comics and folders."""
|
|
18
21
|
|
|
19
22
|
@status_notify(status_type=ImportStatusTypes.FILES_MOVED, updates=False)
|
|
20
|
-
def
|
|
23
|
+
def _bulk_comics_moved(self, moved_paths, library, status=None):
|
|
21
24
|
"""Move comcis."""
|
|
22
25
|
count = 0
|
|
23
26
|
if not moved_paths:
|
|
@@ -57,7 +60,7 @@ class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixi
|
|
|
57
60
|
except Exception:
|
|
58
61
|
self.log.exception(f"moving {comic.path}")
|
|
59
62
|
|
|
60
|
-
Comic.objects.bulk_update(comics,
|
|
63
|
+
Comic.objects.bulk_update(comics, _MOVED_BULK_COMIC_UPDATE_FIELDS)
|
|
61
64
|
|
|
62
65
|
# Update m2m field
|
|
63
66
|
count = len(comics)
|
|
@@ -67,7 +70,7 @@ class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixi
|
|
|
67
70
|
|
|
68
71
|
return count
|
|
69
72
|
|
|
70
|
-
def _get_parent_folders(self, library, dest_folder_paths):
|
|
73
|
+
def _get_parent_folders(self, library, dest_folder_paths, status):
|
|
71
74
|
"""Get destination parent folders."""
|
|
72
75
|
# Determine parent folder paths.
|
|
73
76
|
dest_parent_folder_paths = set()
|
|
@@ -82,7 +85,7 @@ class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixi
|
|
|
82
85
|
create_folder_paths = frozenset(
|
|
83
86
|
dest_parent_folder_paths - frozenset(existing_folder_paths)
|
|
84
87
|
)
|
|
85
|
-
self.bulk_folders_create(create_folder_paths, library)
|
|
88
|
+
self.bulk_folders_create(create_folder_paths, library, status=status)
|
|
86
89
|
|
|
87
90
|
# get parent folders path to model obj dict
|
|
88
91
|
dest_parent_folders_objs = Folder.objects.filter(
|
|
@@ -94,13 +97,16 @@ class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixi
|
|
|
94
97
|
return dest_parent_folders
|
|
95
98
|
|
|
96
99
|
@status_notify(status_type=ImportStatusTypes.DIRS_MOVED, updates=False)
|
|
97
|
-
def
|
|
100
|
+
def _bulk_folders_moved(self, folders_moved, library, **kwargs):
|
|
98
101
|
"""Move folders in the database instead of recreating them."""
|
|
99
102
|
if not folders_moved:
|
|
100
103
|
return 0
|
|
101
104
|
|
|
102
105
|
dest_folder_paths = frozenset(folders_moved.values())
|
|
103
|
-
|
|
106
|
+
status = kwargs.get("status")
|
|
107
|
+
dest_parent_folders = self._get_parent_folders(
|
|
108
|
+
library, dest_folder_paths, status
|
|
109
|
+
)
|
|
104
110
|
|
|
105
111
|
src_folder_paths = frozenset(folders_moved.keys())
|
|
106
112
|
folders = Folder.objects.filter(library=library, path__in=src_folder_paths)
|
|
@@ -119,11 +125,34 @@ class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixi
|
|
|
119
125
|
|
|
120
126
|
update_folders = sorted(update_folders, key=lambda x: len(Path(x.path).parts))
|
|
121
127
|
|
|
122
|
-
Folder.objects.bulk_update(update_folders,
|
|
128
|
+
Folder.objects.bulk_update(update_folders, _MOVED_BULK_FOLDER_UPDATE_FIELDS)
|
|
123
129
|
count = len(update_folders)
|
|
124
130
|
self.log.info(f"Moved {count} folders.")
|
|
125
131
|
return count
|
|
126
132
|
|
|
133
|
+
@status_notify(status_type=ImportStatusTypes.DIRS_MODIFIED, updates=False)
|
|
134
|
+
def bulk_folders_modified(self, paths, library, **kwargs):
|
|
135
|
+
"""Update folders stat and nothing else."""
|
|
136
|
+
count = 0
|
|
137
|
+
if not paths:
|
|
138
|
+
return count
|
|
139
|
+
folders = Folder.objects.filter(library=library, path__in=paths).only(
|
|
140
|
+
"stat", "updated_at"
|
|
141
|
+
)
|
|
142
|
+
update_folders = []
|
|
143
|
+
now = Now()
|
|
144
|
+
for folder in folders.iterator():
|
|
145
|
+
if Path(folder.path).exists():
|
|
146
|
+
folder.set_stat()
|
|
147
|
+
folder.updated_at = now
|
|
148
|
+
update_folders.append(folder)
|
|
149
|
+
Folder.objects.bulk_update(
|
|
150
|
+
update_folders, fields=BULK_UPDATE_FOLDER_MODIFIED_FIELDS
|
|
151
|
+
)
|
|
152
|
+
count += len(update_folders)
|
|
153
|
+
self.log.info(f"Modified {count} folders")
|
|
154
|
+
return count
|
|
155
|
+
|
|
127
156
|
def adopt_orphan_folders(self):
|
|
128
157
|
"""Find orphan folders and move them into their correct place."""
|
|
129
158
|
libraries = Library.objects.only("pk", "path")
|
|
@@ -138,4 +167,19 @@ class MovedMixin(CreateComicsMixin, CreateForeignKeysMixin, QueryForeignKeysMixi
|
|
|
138
167
|
for path in orphan_folder_paths:
|
|
139
168
|
folders_moved[path] = path
|
|
140
169
|
|
|
141
|
-
self.
|
|
170
|
+
self._bulk_folders_moved(folders_moved, library)
|
|
171
|
+
|
|
172
|
+
def move_and_modify_dirs(self, library, task):
|
|
173
|
+
"""Move files and dirs and modify dirs."""
|
|
174
|
+
# TODO add status?
|
|
175
|
+
changed = 0
|
|
176
|
+
changed += self._bulk_folders_moved(task.dirs_moved, library)
|
|
177
|
+
task.dirs_moved = None
|
|
178
|
+
|
|
179
|
+
changed += self._bulk_comics_moved(task.files_moved, library)
|
|
180
|
+
task.files_moved = None
|
|
181
|
+
|
|
182
|
+
changed += self.bulk_folders_modified(task.dirs_modified, library)
|
|
183
|
+
task.dirs_modified = None
|
|
184
|
+
|
|
185
|
+
return changed
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
"""Query the missing foreign keys for comics and creators."""
|
|
2
|
+
from collections.abc import Iterable
|
|
2
3
|
from pathlib import Path
|
|
3
4
|
|
|
4
5
|
from django.db.models import Q
|
|
5
6
|
|
|
6
|
-
from codex.librarian.importer.status import status_notify
|
|
7
|
+
from codex.librarian.importer.status import ImportStatusTypes, status_notify
|
|
7
8
|
from codex.models import (
|
|
8
9
|
Comic,
|
|
9
10
|
Creator,
|
|
@@ -11,12 +12,15 @@ from codex.models import (
|
|
|
11
12
|
Imprint,
|
|
12
13
|
Publisher,
|
|
13
14
|
Series,
|
|
15
|
+
StoryArcNumber,
|
|
14
16
|
Volume,
|
|
15
17
|
)
|
|
18
|
+
from codex.status import Status
|
|
16
19
|
from codex.threads import QueuedThread
|
|
17
20
|
|
|
18
21
|
_CLASS_QUERY_FIELDS_MAP = {
|
|
19
22
|
Creator: ("role__name", "person__name"),
|
|
23
|
+
StoryArcNumber: ("story_arc__name", "number"),
|
|
20
24
|
Folder: ("path",),
|
|
21
25
|
Imprint: ("publisher__name", "name"),
|
|
22
26
|
Series: ("publisher__name", "imprint__name", "name"),
|
|
@@ -27,6 +31,7 @@ _DEFAULT_QUERY_FIELDS = ("name",)
|
|
|
27
31
|
# fixes this in the bulk_create & bulk_update functions. So for complicated
|
|
28
32
|
# queries I gotta batch them myself. Filter arg count is a proxy, but it works.
|
|
29
33
|
_SQLITE_FILTER_ARG_MAX = 990
|
|
34
|
+
_CREATOR_FK_NAMES = ("role", "person")
|
|
30
35
|
|
|
31
36
|
|
|
32
37
|
class QueryForeignKeysMixin(QueuedThread):
|
|
@@ -44,7 +49,13 @@ class QueryForeignKeysMixin(QueuedThread):
|
|
|
44
49
|
.values_list(*fields, flat=flat)
|
|
45
50
|
)
|
|
46
51
|
|
|
47
|
-
def _query_create_metadata(
|
|
52
|
+
def _query_create_metadata(
|
|
53
|
+
self,
|
|
54
|
+
fk_cls,
|
|
55
|
+
create_mds,
|
|
56
|
+
all_filter_args: Iterable[tuple[tuple[str, str], ...]],
|
|
57
|
+
status,
|
|
58
|
+
):
|
|
48
59
|
"""Get create metadata by comparing proposed meatada to existing rows."""
|
|
49
60
|
# Do this in batches so as not to exceed the 1k line sqlite limit
|
|
50
61
|
fk_filter = Q()
|
|
@@ -78,7 +89,9 @@ class QueryForeignKeysMixin(QueuedThread):
|
|
|
78
89
|
return count
|
|
79
90
|
|
|
80
91
|
@staticmethod
|
|
81
|
-
def _add_parent_group_filter(
|
|
92
|
+
def _add_parent_group_filter(
|
|
93
|
+
filter_args: dict[str, str], group_name: str, field_name: str
|
|
94
|
+
):
|
|
82
95
|
"""Get the parent group filter by name."""
|
|
83
96
|
key = f"{field_name}__" if field_name else ""
|
|
84
97
|
|
|
@@ -88,9 +101,9 @@ class QueryForeignKeysMixin(QueuedThread):
|
|
|
88
101
|
|
|
89
102
|
def _get_create_group_set(self, groups, group_cls, create_group_set, status):
|
|
90
103
|
"""Create the create group set."""
|
|
91
|
-
all_filter_args = set()
|
|
104
|
+
all_filter_args: set[tuple[tuple[str, str], ...]] = set()
|
|
92
105
|
for group_tree in groups:
|
|
93
|
-
filter_args = {}
|
|
106
|
+
filter_args: dict[str, str] = {}
|
|
94
107
|
self._add_parent_group_filter(filter_args, group_tree[-1], "")
|
|
95
108
|
if group_cls in (Imprint, Series, Volume):
|
|
96
109
|
self._add_parent_group_filter(filter_args, group_tree[0], "publisher")
|
|
@@ -99,7 +112,10 @@ class QueryForeignKeysMixin(QueuedThread):
|
|
|
99
112
|
if group_cls == Volume:
|
|
100
113
|
self._add_parent_group_filter(filter_args, group_tree[2], "series")
|
|
101
114
|
|
|
102
|
-
|
|
115
|
+
serialized_filter_args: tuple[tuple[str, str], ...] = tuple(
|
|
116
|
+
sorted(filter_args.items())
|
|
117
|
+
)
|
|
118
|
+
all_filter_args.add(serialized_filter_args)
|
|
103
119
|
|
|
104
120
|
candidate_groups = set(groups.keys())
|
|
105
121
|
count = self._query_create_metadata(
|
|
@@ -126,8 +142,36 @@ class QueryForeignKeysMixin(QueuedThread):
|
|
|
126
142
|
update_groups[group_cls][group_tree] = {}
|
|
127
143
|
update_groups[group_cls][group_tree].update(count_dict)
|
|
128
144
|
|
|
145
|
+
def _query_group_tree(self, data, group_tree, count_dict):
|
|
146
|
+
"""Query missing groups for one group tree depth."""
|
|
147
|
+
(
|
|
148
|
+
create_group_set,
|
|
149
|
+
group_cls,
|
|
150
|
+
create_and_update_groups,
|
|
151
|
+
status,
|
|
152
|
+
) = data
|
|
153
|
+
apply_count_dict = count_dict if count_dict else {}
|
|
154
|
+
if group_tree in create_group_set:
|
|
155
|
+
self._update_create_group(
|
|
156
|
+
group_cls,
|
|
157
|
+
create_and_update_groups["create_groups"],
|
|
158
|
+
group_tree,
|
|
159
|
+
apply_count_dict,
|
|
160
|
+
)
|
|
161
|
+
elif group_cls in (Series, Volume):
|
|
162
|
+
self._update_update_group(
|
|
163
|
+
group_cls,
|
|
164
|
+
create_and_update_groups["update_groups"],
|
|
165
|
+
group_tree,
|
|
166
|
+
apply_count_dict,
|
|
167
|
+
)
|
|
168
|
+
if status:
|
|
169
|
+
status.complete = status.complete or 0
|
|
170
|
+
status.complete += 1
|
|
171
|
+
self.status_controller.update(status)
|
|
172
|
+
|
|
129
173
|
@status_notify()
|
|
130
|
-
def
|
|
174
|
+
def _query_missing_group(
|
|
131
175
|
self,
|
|
132
176
|
groups,
|
|
133
177
|
group_cls,
|
|
@@ -144,68 +188,97 @@ class QueryForeignKeysMixin(QueuedThread):
|
|
|
144
188
|
|
|
145
189
|
if status:
|
|
146
190
|
status.subtitle = group_cls.__name__
|
|
191
|
+
|
|
192
|
+
data = (create_group_set, group_cls, create_and_update_groups, status)
|
|
147
193
|
for group_tree, count_dict in groups.items():
|
|
148
|
-
|
|
149
|
-
if group_tree in create_group_set:
|
|
150
|
-
self._update_create_group(
|
|
151
|
-
group_cls,
|
|
152
|
-
create_and_update_groups["create_groups"],
|
|
153
|
-
group_tree,
|
|
154
|
-
apply_count_dict,
|
|
155
|
-
)
|
|
156
|
-
elif group_cls in (Series, Volume):
|
|
157
|
-
self._update_update_group(
|
|
158
|
-
group_cls,
|
|
159
|
-
create_and_update_groups["update_groups"],
|
|
160
|
-
group_tree,
|
|
161
|
-
apply_count_dict,
|
|
162
|
-
)
|
|
163
|
-
count += 1
|
|
164
|
-
if status:
|
|
165
|
-
status.complete += 1
|
|
166
|
-
self.status_controller.update(status)
|
|
194
|
+
self._query_group_tree(data, group_tree, count_dict)
|
|
167
195
|
|
|
168
196
|
if status:
|
|
169
197
|
status.subtitle = ""
|
|
198
|
+
count = len(groups)
|
|
170
199
|
if count:
|
|
171
200
|
self.log.info(f"Prepared {count} new {group_cls.__name__}s.")
|
|
172
201
|
return count
|
|
173
202
|
|
|
174
|
-
@
|
|
175
|
-
def
|
|
176
|
-
"""
|
|
203
|
+
@staticmethod
|
|
204
|
+
def _get_query_filter_creators(creator_tuple):
|
|
205
|
+
"""Get serialized comparison object data for Creators."""
|
|
206
|
+
creator_dict = dict(creator_tuple)
|
|
207
|
+
role = creator_dict.get("role")
|
|
208
|
+
person = creator_dict["person"]
|
|
209
|
+
filter_args = {
|
|
210
|
+
"person__name": person,
|
|
211
|
+
"role__name": role,
|
|
212
|
+
}
|
|
213
|
+
comparison_tuple = (role, person)
|
|
214
|
+
return filter_args, comparison_tuple
|
|
215
|
+
|
|
216
|
+
@staticmethod
|
|
217
|
+
def _get_query_filter_story_arc_numbers(story_arc_number_tuple):
|
|
218
|
+
"""Get serialized comparison object data for StoryArcNumbers."""
|
|
219
|
+
story_arc, number = story_arc_number_tuple
|
|
220
|
+
filter_args = {
|
|
221
|
+
"story_arc__name": story_arc,
|
|
222
|
+
"number": number,
|
|
223
|
+
}
|
|
224
|
+
comparison_tuple = (story_arc, number)
|
|
225
|
+
return filter_args, comparison_tuple
|
|
226
|
+
|
|
227
|
+
def _query_missing_dict_model(self, possible_objs, model_data, create_objs, status):
|
|
228
|
+
"""Find missing dict type m2m models with a supplied filter method."""
|
|
177
229
|
count = 0
|
|
178
|
-
if not
|
|
230
|
+
if not possible_objs:
|
|
179
231
|
return count
|
|
180
232
|
# create the filter
|
|
181
|
-
|
|
233
|
+
(get_query_filter_method, model) = model_data
|
|
234
|
+
comparison_objs = set()
|
|
182
235
|
all_filter_args = set()
|
|
183
|
-
for
|
|
184
|
-
|
|
185
|
-
role = creator_dict.get("role")
|
|
186
|
-
person = creator_dict["person"]
|
|
187
|
-
filter_args = {
|
|
188
|
-
"person__name": person,
|
|
189
|
-
"role__name": role,
|
|
190
|
-
}
|
|
236
|
+
for obj_tuple in possible_objs:
|
|
237
|
+
filter_args, comparison_tuple = get_query_filter_method(obj_tuple)
|
|
191
238
|
all_filter_args.add(tuple(sorted(filter_args.items())))
|
|
239
|
+
comparison_objs.add(comparison_tuple)
|
|
192
240
|
|
|
193
|
-
|
|
194
|
-
comparison_creators.add(comparison_tuple)
|
|
195
|
-
|
|
196
|
-
# get the create metadata
|
|
241
|
+
# get the obj metadata
|
|
197
242
|
count = self._query_create_metadata(
|
|
198
|
-
|
|
243
|
+
model, comparison_objs, all_filter_args, status
|
|
199
244
|
)
|
|
200
|
-
|
|
245
|
+
create_objs.update(comparison_objs)
|
|
201
246
|
if count:
|
|
202
|
-
self.log.info(f"Prepared {count} new
|
|
247
|
+
self.log.info(f"Prepared {count} new {model.__class__.__name__}s.")
|
|
203
248
|
if status:
|
|
249
|
+
status.complete = status.complete or 0
|
|
204
250
|
status.complete += count
|
|
205
251
|
return count
|
|
206
252
|
|
|
207
253
|
@status_notify()
|
|
208
|
-
def
|
|
254
|
+
def _query_missing_creators(self, creators, create_creators, status=None):
|
|
255
|
+
"""Find missing creator objects."""
|
|
256
|
+
model_data = (self._get_query_filter_creators, Creator)
|
|
257
|
+
return self._query_missing_dict_model(
|
|
258
|
+
creators,
|
|
259
|
+
model_data,
|
|
260
|
+
create_creators,
|
|
261
|
+
status,
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
@status_notify()
|
|
265
|
+
def _query_missing_story_arc_numbers(
|
|
266
|
+
self, story_arc_numbers, create_story_arc_numbers, status=None
|
|
267
|
+
):
|
|
268
|
+
"""Find missing story arc number objects."""
|
|
269
|
+
model_data = (
|
|
270
|
+
self._get_query_filter_story_arc_numbers,
|
|
271
|
+
StoryArcNumber,
|
|
272
|
+
)
|
|
273
|
+
return self._query_missing_dict_model(
|
|
274
|
+
story_arc_numbers,
|
|
275
|
+
model_data,
|
|
276
|
+
create_story_arc_numbers,
|
|
277
|
+
status,
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
@status_notify()
|
|
281
|
+
def _query_missing_simple_models(self, names, fk_data, status=None):
|
|
209
282
|
"""Find missing named models and folders."""
|
|
210
283
|
count = 0
|
|
211
284
|
if not names:
|
|
@@ -231,6 +304,7 @@ class QueryForeignKeysMixin(QueuedThread):
|
|
|
231
304
|
num_in_batch = len(batch_proposed_names)
|
|
232
305
|
count += num_in_batch
|
|
233
306
|
if status:
|
|
307
|
+
status.complete = status.complete or 0
|
|
234
308
|
status.complete += num_in_batch
|
|
235
309
|
self.status_controller.update(status)
|
|
236
310
|
start += _SQLITE_FILTER_ARG_MAX
|
|
@@ -260,7 +334,7 @@ class QueryForeignKeysMixin(QueuedThread):
|
|
|
260
334
|
# get the create metadata
|
|
261
335
|
create_folder_paths_dict = {}
|
|
262
336
|
fk_data = (create_folder_paths_dict, Comic, "parent_folder", "path")
|
|
263
|
-
self.
|
|
337
|
+
self._query_missing_simple_models(
|
|
264
338
|
proposed_folder_paths,
|
|
265
339
|
fk_data,
|
|
266
340
|
status=status,
|
|
@@ -269,3 +343,91 @@ class QueryForeignKeysMixin(QueuedThread):
|
|
|
269
343
|
count = len(comic_paths)
|
|
270
344
|
self.log.info(f"Prepared {count} new Folders.")
|
|
271
345
|
return count
|
|
346
|
+
|
|
347
|
+
@staticmethod
|
|
348
|
+
def _get_query_fks_totals(fks):
|
|
349
|
+
"""Get the query foreign keys totals."""
|
|
350
|
+
fks_total = 0
|
|
351
|
+
for key, objs in fks.items():
|
|
352
|
+
if key == "group_trees":
|
|
353
|
+
for groups in objs.values():
|
|
354
|
+
fks_total += len(groups)
|
|
355
|
+
else:
|
|
356
|
+
fks_total += len(objs)
|
|
357
|
+
return fks_total
|
|
358
|
+
|
|
359
|
+
def _query_one_simple_model(self, fk_field, names, create_fks, status):
|
|
360
|
+
"""Batch query one simple model name."""
|
|
361
|
+
if fk_field in _CREATOR_FK_NAMES:
|
|
362
|
+
base_cls = Creator
|
|
363
|
+
elif fk_field == "story_arc":
|
|
364
|
+
base_cls = StoryArcNumber
|
|
365
|
+
else:
|
|
366
|
+
base_cls = Comic
|
|
367
|
+
fk_data = create_fks, base_cls, fk_field, "name"
|
|
368
|
+
status.complete += self._query_missing_simple_models(
|
|
369
|
+
names,
|
|
370
|
+
fk_data,
|
|
371
|
+
status=status,
|
|
372
|
+
)
|
|
373
|
+
|
|
374
|
+
def query_all_missing_fks(self, library_path, fks):
|
|
375
|
+
"""Get objects to create by querying existing objects for the proposed fks."""
|
|
376
|
+
create_creators = set()
|
|
377
|
+
create_story_arc_numbers = set()
|
|
378
|
+
create_groups = {}
|
|
379
|
+
update_groups = {}
|
|
380
|
+
create_folder_paths = set()
|
|
381
|
+
create_fks = {}
|
|
382
|
+
self.log.debug(f"Querying existing foreign keys for comics in {library_path}")
|
|
383
|
+
fks_total = self._get_query_fks_totals(fks)
|
|
384
|
+
status = Status(ImportStatusTypes.QUERY_MISSING_FKS, 0, fks_total)
|
|
385
|
+
try:
|
|
386
|
+
self.status_controller.start(status)
|
|
387
|
+
|
|
388
|
+
self._query_missing_creators(
|
|
389
|
+
fks.pop("creators", {}),
|
|
390
|
+
create_creators,
|
|
391
|
+
status=status,
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
self._query_missing_story_arc_numbers(
|
|
395
|
+
fks.pop("story_arc_numbers", {}),
|
|
396
|
+
create_story_arc_numbers,
|
|
397
|
+
status=status,
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
create_and_update_groups = {
|
|
401
|
+
"create_groups": create_groups,
|
|
402
|
+
"update_groups": update_groups,
|
|
403
|
+
}
|
|
404
|
+
for group_class, groups in fks.pop("group_trees", {}).items():
|
|
405
|
+
self._query_missing_group(
|
|
406
|
+
groups,
|
|
407
|
+
group_class,
|
|
408
|
+
create_and_update_groups,
|
|
409
|
+
status=status,
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
self.query_missing_folder_paths(
|
|
413
|
+
fks.pop("comic_paths", ()),
|
|
414
|
+
library_path,
|
|
415
|
+
create_folder_paths,
|
|
416
|
+
status=status,
|
|
417
|
+
)
|
|
418
|
+
|
|
419
|
+
for fk_field in sorted(fks.keys()):
|
|
420
|
+
self._query_one_simple_model(
|
|
421
|
+
fk_field, fks.pop(fk_field), create_fks, status
|
|
422
|
+
)
|
|
423
|
+
finally:
|
|
424
|
+
self.status_controller.finish(status)
|
|
425
|
+
|
|
426
|
+
return (
|
|
427
|
+
create_groups,
|
|
428
|
+
update_groups,
|
|
429
|
+
create_folder_paths,
|
|
430
|
+
create_fks,
|
|
431
|
+
create_creators,
|
|
432
|
+
create_story_arc_numbers,
|
|
433
|
+
)
|
|
@@ -12,14 +12,14 @@ class UpdaterDBDiffTask(UpdaterTask):
|
|
|
12
12
|
"""For sending to the updater."""
|
|
13
13
|
|
|
14
14
|
library_id: int
|
|
15
|
-
dirs_moved: dict
|
|
16
|
-
files_moved: dict
|
|
17
|
-
dirs_modified: frozenset
|
|
18
|
-
files_modified: frozenset
|
|
15
|
+
dirs_moved: dict[str, str]
|
|
16
|
+
files_moved: dict[str, str]
|
|
17
|
+
dirs_modified: frozenset[str]
|
|
18
|
+
files_modified: frozenset[str]
|
|
19
19
|
# dirs_created
|
|
20
|
-
files_created: frozenset
|
|
21
|
-
dirs_deleted: frozenset
|
|
22
|
-
files_deleted: frozenset
|
|
20
|
+
files_created: frozenset[str]
|
|
21
|
+
dirs_deleted: frozenset[str]
|
|
22
|
+
files_deleted: frozenset[str]
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
@dataclass
|
|
@@ -19,6 +19,7 @@ from codex.models import (
|
|
|
19
19
|
Series,
|
|
20
20
|
SeriesGroup,
|
|
21
21
|
StoryArc,
|
|
22
|
+
StoryArcNumber,
|
|
22
23
|
Tag,
|
|
23
24
|
Team,
|
|
24
25
|
Volume,
|
|
@@ -38,11 +39,22 @@ _COMIC_FK_CLASSES = (
|
|
|
38
39
|
Character,
|
|
39
40
|
Location,
|
|
40
41
|
SeriesGroup,
|
|
41
|
-
|
|
42
|
+
StoryArcNumber,
|
|
42
43
|
Genre,
|
|
43
44
|
)
|
|
44
45
|
_CREATOR_FK_CLASSES = (CreatorRole, CreatorPerson)
|
|
45
|
-
|
|
46
|
+
_STORY_ARC_NUMBER_FK_CLASSES = (StoryArc,)
|
|
47
|
+
TOTAL_NUM_FK_CLASSES = (
|
|
48
|
+
len(_COMIC_FK_CLASSES)
|
|
49
|
+
+ len(_CREATOR_FK_CLASSES)
|
|
50
|
+
+ len(_STORY_ARC_NUMBER_FK_CLASSES)
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
CLEANUP_MAP = [
|
|
54
|
+
(_COMIC_FK_CLASSES, "comic"),
|
|
55
|
+
(_CREATOR_FK_CLASSES, "creator"),
|
|
56
|
+
(_STORY_ARC_NUMBER_FK_CLASSES, "storyarcnumber"),
|
|
57
|
+
]
|
|
46
58
|
|
|
47
59
|
|
|
48
60
|
class CleanupMixin(WorkerBaseMixin):
|
|
@@ -57,7 +69,7 @@ class CleanupMixin(WorkerBaseMixin):
|
|
|
57
69
|
query.delete()
|
|
58
70
|
if count:
|
|
59
71
|
self.log.info(f"Deleted {count} orphan {cls.__name__}s")
|
|
60
|
-
status.complete +=
|
|
72
|
+
status.complete += count
|
|
61
73
|
self.status_controller.update(status)
|
|
62
74
|
|
|
63
75
|
def cleanup_fks(self):
|
|
@@ -66,8 +78,8 @@ class CleanupMixin(WorkerBaseMixin):
|
|
|
66
78
|
try:
|
|
67
79
|
self.status_controller.start(status)
|
|
68
80
|
self.log.debug("Cleaning up unused foreign keys...")
|
|
69
|
-
|
|
70
|
-
|
|
81
|
+
for classes, field_name in CLEANUP_MAP:
|
|
82
|
+
self._bulk_cleanup_fks(classes, field_name, status)
|
|
71
83
|
level = logging.INFO if status.complete else logging.DEBUG
|
|
72
84
|
self.log.log(level, f"Cleaned up {status.complete} unused foreign keys.")
|
|
73
85
|
finally:
|
codex/librarian/librariand.py
CHANGED
|
@@ -4,6 +4,8 @@ from multiprocessing import Manager, Process
|
|
|
4
4
|
from threading import active_count
|
|
5
5
|
|
|
6
6
|
from caseconverter import snakecase
|
|
7
|
+
from comicbox.comic_archive import ComicArchive
|
|
8
|
+
from comicbox.exceptions import UnsupportedArchiveTypeError
|
|
7
9
|
|
|
8
10
|
from codex.librarian.covers.coverd import CoverCreatorThread
|
|
9
11
|
from codex.librarian.covers.tasks import CoverTask
|
|
@@ -113,11 +115,19 @@ class LibrarianDaemon(Process, LoggerBaseMixin):
|
|
|
113
115
|
self.log.debug(f"Active threads before thread creation: {active_count()}")
|
|
114
116
|
threads = {}
|
|
115
117
|
kwargs = {"librarian_queue": self.queue, "log_queue": self.log_queue}
|
|
118
|
+
try:
|
|
119
|
+
ComicArchive.check_unrar_executable()
|
|
120
|
+
unrar = True
|
|
121
|
+
except UnsupportedArchiveTypeError as exc:
|
|
122
|
+
self.log.warn(f"{exc}. Not detecting .cbr archives.")
|
|
123
|
+
unrar = False
|
|
116
124
|
for name, thread_class in self._THREAD_CLASS_MAP.items():
|
|
117
125
|
if thread_class == NotifierThread:
|
|
118
126
|
thread = thread_class(self.broadcast_queue, **kwargs)
|
|
119
127
|
elif thread_class == SearchIndexerThread:
|
|
120
128
|
thread = thread_class(self.search_indexer_abort_event, **kwargs)
|
|
129
|
+
elif thread_class in (LibraryEventObserver, LibraryPollingObserver):
|
|
130
|
+
thread = thread_class(unrar=unrar, **kwargs)
|
|
121
131
|
else:
|
|
122
132
|
thread = thread_class(**kwargs)
|
|
123
133
|
threads[name] = thread
|
|
@@ -15,9 +15,6 @@ from watchdog.events import (
|
|
|
15
15
|
from codex.librarian.watchdog.tasks import WatchdogEventTask
|
|
16
16
|
from codex.logger_base import LoggerBaseMixin
|
|
17
17
|
|
|
18
|
-
_COMIC_REGEX = r"\.(cb[zrt]|pdf)$"
|
|
19
|
-
_COMIC_MATCHER = re.compile(_COMIC_REGEX, re.IGNORECASE)
|
|
20
|
-
|
|
21
18
|
|
|
22
19
|
class CodexLibraryEventHandler(FileSystemEventHandler, LoggerBaseMixin):
|
|
23
20
|
"""Handle watchdog events for comics in a library."""
|
|
@@ -30,25 +27,26 @@ class CodexLibraryEventHandler(FileSystemEventHandler, LoggerBaseMixin):
|
|
|
30
27
|
self.librarian_queue = kwargs.pop("librarian_queue")
|
|
31
28
|
log_queue = kwargs.pop("log_queue")
|
|
32
29
|
self.init_logger(log_queue)
|
|
30
|
+
unrar = kwargs.pop("unrar", False)
|
|
31
|
+
comic_regex = r"\.(cb[zrt]|pdf)$" if unrar else r"\.(cb[zt]|pdf)$"
|
|
32
|
+
self._comic_matcher = re.compile(comic_regex, re.IGNORECASE)
|
|
33
33
|
super().__init__(*args, **kwargs)
|
|
34
34
|
|
|
35
|
-
|
|
36
|
-
def _match_comic_suffix(path):
|
|
35
|
+
def _match_comic_suffix(self, path):
|
|
37
36
|
"""Match a supported comic suffix."""
|
|
38
37
|
if not path:
|
|
39
38
|
return False
|
|
40
39
|
# We don't care about general suffixes. Only length four.
|
|
41
40
|
suffix = path[-4:]
|
|
42
41
|
suffix = fsdecode(suffix)
|
|
43
|
-
return
|
|
42
|
+
return self._comic_matcher.match(suffix) is not None
|
|
44
43
|
|
|
45
|
-
|
|
46
|
-
def _transform_file_event(cls, event):
|
|
44
|
+
def _transform_file_event(self, event):
|
|
47
45
|
"""Transform file events into other events."""
|
|
48
|
-
source_match =
|
|
46
|
+
source_match = self._match_comic_suffix(event.src_path)
|
|
49
47
|
if event.event_type == EVENT_TYPE_MOVED:
|
|
50
48
|
# Some types of file moves need to be cast as other events.
|
|
51
|
-
dest_match =
|
|
49
|
+
dest_match = self._match_comic_suffix(event.dest_path)
|
|
52
50
|
if not source_match and dest_match:
|
|
53
51
|
# Moved from an ignored file extension into a comic type,
|
|
54
52
|
# so create a new comic.
|
|
@@ -61,17 +59,16 @@ class CodexLibraryEventHandler(FileSystemEventHandler, LoggerBaseMixin):
|
|
|
61
59
|
event = None
|
|
62
60
|
return event
|
|
63
61
|
|
|
64
|
-
|
|
65
|
-
def _transform_event(cls, event):
|
|
62
|
+
def _transform_event(self, event):
|
|
66
63
|
"""Transform events into other events."""
|
|
67
|
-
if event.event_type in
|
|
64
|
+
if event.event_type in self.IGNORED_EVENTS:
|
|
68
65
|
event = None
|
|
69
66
|
elif event.is_directory:
|
|
70
67
|
if event.event_type == EVENT_TYPE_CREATED:
|
|
71
68
|
# Directories are only created by comics
|
|
72
69
|
event = None
|
|
73
70
|
else:
|
|
74
|
-
event =
|
|
71
|
+
event = self._transform_file_event(event)
|
|
75
72
|
return event
|
|
76
73
|
|
|
77
74
|
def dispatch(self, event):
|
|
@@ -23,6 +23,7 @@ class UatuMixin(BaseObserver, WorkerBaseMixin):
|
|
|
23
23
|
log_queue = kwargs.pop("log_queue")
|
|
24
24
|
librarian_queue = kwargs.pop("librarian_queue")
|
|
25
25
|
self.init_worker(log_queue, librarian_queue)
|
|
26
|
+
self._unrar = kwargs.pop("unrar", False)
|
|
26
27
|
super().__init__(*args, **kwargs)
|
|
27
28
|
|
|
28
29
|
def _get_watch(self, path):
|
|
@@ -50,7 +51,10 @@ class UatuMixin(BaseObserver, WorkerBaseMixin):
|
|
|
50
51
|
|
|
51
52
|
# Set up the watch
|
|
52
53
|
handler = CodexLibraryEventHandler(
|
|
53
|
-
library,
|
|
54
|
+
library,
|
|
55
|
+
librarian_queue=self.librarian_queue,
|
|
56
|
+
log_queue=self.log_queue,
|
|
57
|
+
unrar=self._unrar,
|
|
54
58
|
)
|
|
55
59
|
self.schedule(handler, library.path, recursive=True)
|
|
56
60
|
self.log.info(f"Started {watching_log}")
|