udata 9.1.2.dev30355__py2.py3-none-any.whl → 9.1.2.dev30454__py2.py3-none-any.whl

udata/migrations/2021-08-17-harvest-integrity.py

@@ -1,20 +1,20 @@
-'''
+"""
 Remove Harvest db integrity problems
 ⚠️ long migration
-'''
+"""
+
 import logging
 
 import mongoengine
 
 from udata.core.jobs.models import PeriodicTask
-from udata.harvest.models import HarvestSource, HarvestJob
+from udata.harvest.models import HarvestJob, HarvestSource
 
 log = logging.getLogger(__name__)
 
 
 def migrate(db):
-
-    log.info('Processing HarvestJob source references.')
+    log.info("Processing HarvestJob source references.")
 
     harvest_jobs = HarvestJob.objects().no_cache().all()
     count = 0
@@ -25,9 +25,9 @@ def migrate(db):
             count += 1
             harvest_job.delete()
 
-    log.info(f'Completed, removed {count} HarvestJob objects')
+    log.info(f"Completed, removed {count} HarvestJob objects")
 
-    log.info('Processing HarvestJob items references.')
+    log.info("Processing HarvestJob items references.")
 
     harvest_jobs = HarvestJob.objects.filter(items__0__exists=True).no_cache().all()
     count = 0
@@ -40,14 +40,15 @@ def migrate(db):
                 item.dataset = None
                 harvest_job.save()
 
-    log.info(f'Completed, modified {count} HarvestJob objects')
+    log.info(f"Completed, modified {count} HarvestJob objects")
 
-    log.info('Processing PeriodicTask objects.')
+    log.info("Processing PeriodicTask objects.")
 
-    harvest_schedules = PeriodicTask.objects.filter(name__startswith='Harvest',
-                                                    description='Periodic Harvesting')
+    harvest_schedules = PeriodicTask.objects.filter(
+        name__startswith="Harvest", description="Periodic Harvesting"
+    )
     hs = HarvestSource.objects.filter(periodic_task__in=harvest_schedules)
     unlegit_schedules = harvest_schedules.filter(id__nin=[h.periodic_task.id for h in hs])
     count = unlegit_schedules.delete()
 
-    log.info(f'Completed, removed {count} PeriodicTask objects')
+    log.info(f"Completed, removed {count} PeriodicTask objects")

udata/migrations/2021-08-17-oauth2client-integrity.py

@@ -1,6 +1,7 @@
-'''
+"""
 Remove Oauth2Client db integrity problems
-'''
+"""
+
 import logging
 
 import mongoengine
@@ -11,7 +12,7 @@ log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Processing Transfer objects.')
+    log.info("Processing Transfer objects.")
 
     clients = OAuth2Client.objects.no_cache().all()
 
@@ -24,4 +25,4 @@ def migrate(db):
             client.organization = None
             client.save()
 
-    log.info(f'Completed, modified {count} OAuth2Client objects')
+    log.info(f"Completed, modified {count} OAuth2Client objects")

udata/migrations/2021-08-17-transfer-integrity.py

@@ -1,7 +1,8 @@
-'''
+"""
 Remove Transfer db integrity problems
 ⚠️ long migration
-'''
+"""
+
 import logging
 
 import mongoengine
@@ -12,7 +13,7 @@ log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Processing Transfer objects.')
+    log.info("Processing Transfer objects.")
 
     transfers = Transfer.objects.no_cache().all()
 
@@ -26,4 +27,4 @@ def migrate(db):
             transfer.delete()
             count += 1
 
-    log.info(f'Completed, removed {count} Transfer objects')
+    log.info(f"Completed, removed {count} Transfer objects")

udata/migrations/2021-08-17-users-integrity.py

@@ -1,6 +1,7 @@
-'''
+"""
 Remove User db integrity problems, and Discussion.subject in the process
-'''
+"""
+
 import logging
 
 import mongoengine
@@ -11,7 +12,7 @@ log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Processing Discussion user references.')
+    log.info("Processing Discussion user references.")
 
     discussions = Discussion.objects(user__ne=None).no_cache().all()
     remove_count = 0
@@ -37,9 +38,9 @@ def migrate(db):
             discussion.save()
             modif_count += 1
 
-    log.info(f'Modified {modif_count} Discussion objects, deleted {remove_count}')
+    log.info(f"Modified {modif_count} Discussion objects, deleted {remove_count}")
 
-    log.info('Processing Badges user references.')
+    log.info("Processing Badges user references.")
 
     organizations = Organization.objects.filter(badges__0__exists=True)
     count = 0
@@ -52,9 +53,9 @@ def migrate(db):
                 badge.created_by = None
                 org.save()
 
-    log.info(f'Modified {count} badges')
+    log.info(f"Modified {count} badges")
 
-    log.info('Processing Request user references.')
+    log.info("Processing Request user references.")
 
     organizations = Organization.objects.filter(requests__0__exists=True)
     count = 0
@@ -67,4 +68,4 @@ def migrate(db):
                 request.handled_by = None
                 org.save()
 
-    log.info(f'Modified {count} requests')
+    log.info(f"Modified {count} requests")

udata/migrations/2021-12-14-reuse-topics.py

@@ -1,6 +1,7 @@
-'''
+"""
 Add a default topic to all reuses in db
-'''
+"""
+
 import logging
 
 from udata.models import Reuse
@@ -9,14 +10,14 @@ log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Processing Reuse.')
+    log.info("Processing Reuse.")
 
     reuses = Reuse.objects().no_cache().timeout(False)
     count = 0
     for reuse in reuses:
-        reuse.topic = 'others'
+        reuse.topic = "others"
         reuse.save()
         count += 1
 
-    log.info(f'Modified {count} Reuses objects')
-    log.info('Done')
+    log.info(f"Modified {count} Reuses objects")
+    log.info("Done")

udata/migrations/2022-04-21-improve-extension-detection.py

@@ -1,16 +1,17 @@
-'''
+"""
 Recompute resources extension with improved detection
-'''
+"""
+
 import logging
 
-from udata.models import Dataset
 from udata.core.storages.utils import extension
+from udata.models import Dataset
 
 log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Processing Datasets.')
+    log.info("Processing Datasets.")
 
     datasets = Dataset.objects().no_cache().timeout(False)
     count = 0
@@ -23,7 +24,7 @@ def migrate(db):
                 resource.format = detected_format
                 count += 1
         if save_dataset:
-            dataset.save(signal_kwargs={'ignores': ['post_save']})
+            dataset.save(signal_kwargs={"ignores": ["post_save"]})
 
-    log.info(f'Modified {count} resource objects')
-    log.info('Done')
+    log.info(f"Modified {count} resource objects")
+    log.info("Done")

udata/migrations/2022-09-22-clean-inactive-harvest-datasets.py

@@ -1,34 +1,36 @@
-'''
+"""
 Datasets linked to an inactive harvester are archived
-'''
+"""
+
 import logging
 
 from mongoengine.errors import ValidationError
 
-from udata.models import Dataset
 from udata.harvest.actions import archive_harvested_dataset
 from udata.harvest.models import HarvestSource
-
+from udata.models import Dataset
 
 log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Computing the list of datasets linked to an inactive harvester.')
+    log.info("Computing the list of datasets linked to an inactive harvester.")
 
     active_sources = [str(source.id) for source in HarvestSource.objects()]
-    dangling_datasets = Dataset.objects(**{
-        'extras__harvest:source_id__exists': True,
-        'extras__harvest:source_id__nin': active_sources,
-        'archived': None
-    })
+    dangling_datasets = Dataset.objects(
+        **{
+            "extras__harvest:source_id__exists": True,
+            "extras__harvest:source_id__nin": active_sources,
+            "archived": None,
+        }
+    )
 
-    log.info(f'{dangling_datasets.count()} datasets to archive.')
+    log.info(f"{dangling_datasets.count()} datasets to archive.")
 
     for dataset in dangling_datasets:
         try:
-            archive_harvested_dataset(dataset, reason='harvester-inactive', dryrun=False)
+            archive_harvested_dataset(dataset, reason="harvester-inactive", dryrun=False)
         except ValidationError as e:
-            log.error(f'Error on dataset {dataset.id}: {e}')
+            log.error(f"Error on dataset {dataset.id}: {e}")
 
-    log.info('Done')
+    log.info("Done")

udata/migrations/2022-10-10-add-fs_uniquifier-to-user-model.py

@@ -1,6 +1,7 @@
-'''
+"""
 Set fs_uniquifier field for every user due to Flask-Security-Too update > v4.0.0
-'''
+"""
+
 import logging
 import uuid
 
@@ -8,12 +9,11 @@ from mongoengine.errors import ValidationError
 
 from udata.models import User
 
-
 log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Processing Users fs_uniquifier attribute.')
+    log.info("Processing Users fs_uniquifier attribute.")
 
     users = User.objects().no_cache().timeout(False)
     for user in users:
@@ -21,6 +21,6 @@ def migrate(db):
         try:
             user.save()
         except ValidationError as e:
-            log.error(f'Error on user {user.id}: {e}')
+            log.error(f"Error on user {user.id}: {e}")
 
-    log.info('Completed.')
+    log.info("Completed.")

udata/migrations/2022-10-10-migrate-harvest-extras.py

@@ -1,57 +1,67 @@
-'''
+"""
 Move identifying harvest fields from extras to dedicated harvest metadata fields.
 Clean other extras set at harvest time, they will be re-created if needed.
-'''
+"""
+
 import logging
 
 from mongoengine.errors import ValidationError
-from udata.models import Dataset
+
 from udata.core.dataset.models import HarvestDatasetMetadata
+from udata.models import Dataset
 
 log = logging.getLogger(__name__)
 
 
 def should_keep(extra):
-    extras_to_remove = ['remote_url', 'dct:identifier', 'uri', 'ckan:name', 'ckan:source']
-    if extra.startswith('harvest:') or extra.startswith('ods:') or extra in extras_to_remove:
+    extras_to_remove = ["remote_url", "dct:identifier", "uri", "ckan:name", "ckan:source"]
+    if extra.startswith("harvest:") or extra.startswith("ods:") or extra in extras_to_remove:
         return False
     return True
 
 
 def migrate(db):
-    log.info('Processing Datasets.')
+    log.info("Processing Datasets.")
 
-    datasets = Dataset.objects(__raw__={
-            '$or': [
-                {'extras.harvest:remote_id': {'$exists': True}},
-                {'extras.harvest:domain': {'$exists': True}},
-                {'extras.harvest:source_id': {'$exists': True}},
-            ],
-        }).no_cache().timeout(False)
+    datasets = (
+        Dataset.objects(
+            __raw__={
+                "$or": [
+                    {"extras.harvest:remote_id": {"$exists": True}},
+                    {"extras.harvest:domain": {"$exists": True}},
+                    {"extras.harvest:source_id": {"$exists": True}},
+                ],
+            }
+        )
+        .no_cache()
+        .timeout(False)
+    )
 
     datasets_count = datasets.count()
     for dataset in datasets:
         if not dataset.harvest:
             dataset.harvest = HarvestDatasetMetadata()
-        dataset.harvest.remote_id = dataset.extras.pop('harvest:remote_id', None)
-        dataset.harvest.source_id = dataset.extras.pop('harvest:source_id', None)
-        dataset.harvest.domain = dataset.extras.pop('harvest:domain', None)
+        dataset.harvest.remote_id = dataset.extras.pop("harvest:remote_id", None)
+        dataset.harvest.source_id = dataset.extras.pop("harvest:source_id", None)
+        dataset.harvest.domain = dataset.extras.pop("harvest:domain", None)
         # Keep last_update for archiving reason
-        dataset.harvest.last_update = dataset.extras.pop('harvest:last_update', None)
+        dataset.harvest.last_update = dataset.extras.pop("harvest:last_update", None)
         # Keep any archived reason and date
-        dataset.harvest.archived = dataset.extras.pop('harvest:archived', None)
-        dataset.harvest.archived_at = dataset.extras.pop('harvest:archived_at', None)
+        dataset.harvest.archived = dataset.extras.pop("harvest:archived", None)
+        dataset.harvest.archived_at = dataset.extras.pop("harvest:archived_at", None)
 
-        dataset.extras = {extra: dataset.extras[extra] for extra in dataset.extras
-                          if should_keep(extra)}
+        dataset.extras = {
+            extra: dataset.extras[extra] for extra in dataset.extras if should_keep(extra)
+        }
         for resource in dataset.resources:
-            resource.extras = {extra: resource.extras[extra] for extra in resource.extras
-                               if should_keep(extra)}
+            resource.extras = {
+                extra: resource.extras[extra] for extra in resource.extras if should_keep(extra)
+            }
 
         try:
             dataset.save()
         except ValidationError as e:
-            log.error(f'Failed to save dataset {dataset.id}: {e}')
+            log.error(f"Failed to save dataset {dataset.id}: {e}")
 
-    log.info(f'Modified {datasets_count} datasets objects')
-    log.info('Done')
+    log.info(f"Modified {datasets_count} datasets objects")
+    log.info("Done")

udata/migrations/2023-02-08-rename-internal-dates.py

@@ -1,48 +1,66 @@
-'''
+"""
 The purpose here is to change the dates name in the dataset and resource model.
-'''
+"""
+
 import logging
 
 from mongoengine.connection import get_db
+
 from udata.models import Dataset
 
 log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Processing Dataset collection.')
+    log.info("Processing Dataset collection.")
 
     db = get_db()
     dataset_collection = db.dataset
 
-    result = dataset_collection.update_many({}, {'$rename': {
-        'created_at': 'created_at_internal',
-        'last_modified': 'last_modified_internal'}})
-    log.info(f'{result.modified_count} Datasets processed.')
+    result = dataset_collection.update_many(
+        {},
+        {
+            "$rename": {
+                "created_at": "created_at_internal",
+                "last_modified": "last_modified_internal",
+            }
+        },
+    )
+    log.info(f"{result.modified_count} Datasets processed.")
 
     community_resource_collection = db.community_resource
-    result = community_resource_collection.update_many({}, {'$rename': {
-        'created_at': 'created_at_internal',
-        'modified': 'last_modified_internal'}})
-    log.info(f'{result.modified_count} Community Resources processed for renaming.')
-    result = community_resource_collection.update_many({}, {'$unset': { 'published': "" }})
-    log.info(f'{result.modified_count} Community Resources processed for published removing.')
-
-    result = dataset_collection.update_many({}, [{
-        '$set': {
-            'resources': {
-                '$map': {
-                    'input': '$resources',
-                    'in': {
-                        '$mergeObjects': ['$$this', { 'created_at_internal': '$$this.created_at', 'last_modified_internal': '$$this.modified' }]
+    result = community_resource_collection.update_many(
+        {}, {"$rename": {"created_at": "created_at_internal", "modified": "last_modified_internal"}}
+    )
+    log.info(f"{result.modified_count} Community Resources processed for renaming.")
+    result = community_resource_collection.update_many({}, {"$unset": {"published": ""}})
+    log.info(f"{result.modified_count} Community Resources processed for published removing.")
+
+    result = dataset_collection.update_many(
+        {},
+        [
+            {
+                "$set": {
+                    "resources": {
+                        "$map": {
+                            "input": "$resources",
+                            "in": {
+                                "$mergeObjects": [
+                                    "$$this",
+                                    {
+                                        "created_at_internal": "$$this.created_at",
+                                        "last_modified_internal": "$$this.modified",
+                                    },
+                                ]
+                            },
                         }
                     }
-                }
+                },
             },
-        },
-        { '$unset': 'resources.created_at'},
-        { '$unset': 'resources.modified'},
-        { '$unset': 'resources.published'}
-    ])
+            {"$unset": "resources.created_at"},
+            {"$unset": "resources.modified"},
+            {"$unset": "resources.published"},
+        ],
+    )
 
-    log.info('Completed.')
+    log.info("Completed.")

udata/migrations/2024-01-29-fix-reuse-and-dataset-with-private-None.py

@@ -1,20 +1,22 @@
-'''
+"""
 Some old dataset/reuse don't have a `private` field. It should be False by default.
-'''
+"""
+
 import logging
 
-from udata.models import Reuse, Dataset
+from udata.models import Dataset, Reuse
 
 log = logging.getLogger(__name__)
 
+
 def migrate(db):
-    log.info('Processing Reuse…')
+    log.info("Processing Reuse…")
 
     count = Reuse.objects(private=None).update(private=False)
-    log.info(f'Fixed {count} Reuse objects from private None to private False.')
+    log.info(f"Fixed {count} Reuse objects from private None to private False.")
 
-    log.info('Processing Datasets…')
+    log.info("Processing Datasets…")
     count = Dataset.objects(private=None).update(private=False)
-    log.info(f'Fixed {count} Dataset objects from private None to private False.')
+    log.info(f"Fixed {count} Dataset objects from private None to private False.")
 
-    log.info('Done')
+    log.info("Done")

udata/migrations/2024-03-22-migrate-activity-kwargs-to-extras.py

@@ -1,16 +1,18 @@
 """
 Migrates `Activity.kwargs` to `Activity.extras`.
 """
+
 import logging
 
 from mongoengine.connection import get_db
 
 log = logging.getLogger(__name__)
 
+
 def migrate(db):
-    log.info('Processing activity objects')
+    log.info("Processing activity objects")
 
-    result = get_db().activity.update_many(filter={}, update={'$rename': {'kwargs': 'extras'}})
+    result = get_db().activity.update_many(filter={}, update={"$rename": {"kwargs": "extras"}})
 
-    log.info(f'{result.modified_count} activity object(s) migrated')
-    log.info('Done')
+    log.info(f"{result.modified_count} activity object(s) migrated")
+    log.info("Done")

udata/migrations/2024-06-11-fix-reuse-datasets-references.py

@@ -1,10 +1,11 @@
-'''
+"""
 Add a default topic to all reuses in db
-'''
+"""
+
 import logging
 
-from bson import DBRef
 import mongoengine
+from bson import DBRef
 
 from udata.models import Reuse
 
@@ -12,7 +13,7 @@ log = logging.getLogger(__name__)
 
 
 def migrate(db):
-    log.info('Processing Reuse.')
+    log.info("Processing Reuse.")
 
     reuses = Reuse.objects().no_cache().timeout(False)
     count = 0
@@ -31,5 +32,5 @@ def migrate(db):
             reuse.save()
             count += 1
 
-    log.info(f'Modified {count} Reuses objects (removed {errors} datasets)')
-    log.info('Done')
+    log.info(f"Modified {count} Reuses objects (removed {errors} datasets)")
+    log.info("Done")