udata 10.0.6.dev33079__py2.py3-none-any.whl → 10.0.6.dev33111__py2.py3-none-any.whl

udata/core/dataservices/tasks.py

@@ -1,8 +1,7 @@
 from celery.utils.log import get_task_logger
 
 from udata.core.dataservices.models import Dataservice
-
-# from udata.harvest.models import HarvestJob
+from udata.harvest.models import HarvestJob
 from udata.models import Discussion, Follow, Transfer
 from udata.tasks import job
 
@@ -18,8 +17,7 @@ def purge_dataservices(self):
         # Remove discussions
         Discussion.objects(subject=dataservice).delete()
         # Remove HarvestItem references
-        # TODO: uncomment when adding dataservice harvest
-        # HarvestJob.objects(items__dataservice=dataservice).update(set__items__S__dataservice=None)
+        HarvestJob.objects(items__dataservice=dataservice).update(set__items__S__dataservice=None)
         # Remove associated Transfers
         Transfer.objects(subject=dataservice).delete()
         # Remove dataservice

udata/core/dataset/tasks.py

@@ -48,6 +48,11 @@ def purge_datasets(self):
             datasets = topic.datasets
             datasets.remove(dataset)
             topic.update(datasets=datasets)
+        # Remove dataservices related dataset
+        for dataservice in Dataservice.objects(datasets=dataset):
+            datasets = dataservice.datasets
+            datasets.remove(dataset)
+            dataservice.update(datasets=datasets)
         # Remove HarvestItem references
         HarvestJob.objects(items__dataset=dataset).update(set__items__S__dataset=None)
         # Remove associated Transfers

udata/harvest/tasks.py

@@ -13,8 +13,9 @@ def harvest(self, ident):
     log.info('Launching harvest job for source "%s"', ident)
 
     source = HarvestSource.get(ident)
-    if source.deleted:
-        return  # Ignore deleted sources
+    if source.deleted or not source.active:
+        log.info('Ignoring inactive or deleted source "%s"', ident)
+        return  # Ignore deleted and inactive sources
     Backend = backends.get(current_app, source.backend)
     backend = Backend(source)
 

udata/tests/dataservice/test_dataservice_tasks.py

@@ -3,6 +3,8 @@ import pytest
 from udata.core.dataservices import tasks
 from udata.core.dataservices.models import Dataservice
 from udata.core.user.factories import UserFactory
+from udata.harvest.models import HarvestItem, HarvestJob
+from udata.harvest.tests.factories import HarvestJobFactory
 from udata.models import Discussion, Follow, Message, Transfer
 from udata.utils import faker
 
@@ -34,9 +36,12 @@ def test_purge_dataservices():
 
     follower = Follow.objects.create(follower=user, following=dataservices[0])
 
+    HarvestJobFactory(items=[HarvestItem(dataservice=dataservices[0])])
+
     tasks.purge_dataservices()
 
     assert Dataservice.objects.count() == 1
     assert Transfer.objects.filter(id=transfer.id).count() == 0
     assert Discussion.objects.filter(id=discussion.id).count() == 0
     assert Follow.objects.filter(id=follower.id).count() == 0
+    assert HarvestJob.objects.filter(items__dataservice=dataservices[0].id).count() == 0

udata/tests/dataset/test_dataset_tasks.py

@@ -1,17 +1,20 @@
 import pytest
 
-from udata.core.dataset import tasks
-
 # Those imports seem mandatory for the csv adapters to be registered.
 # This might be because of the decorator mechanism.
+from udata.core.dataservices.models import Dataservice
+from udata.core.dataset import tasks
 from udata.core.dataset.csv import DatasetCsvAdapter, ResourcesCsvAdapter  # noqa
 from udata.core.dataset.factories import CommunityResourceFactory, DatasetFactory
+from udata.core.discussions.factories import DiscussionFactory
 from udata.core.organization.csv import OrganizationCsvAdapter  # noqa
 from udata.core.reuse.csv import ReuseCsvAdapter  # noqa
 from udata.core.tags.csv import TagCsvAdapter  # noqa
 from udata.core.user.factories import UserFactory
 from udata.harvest.csv import HarvestSourceCsvAdapter  # noqa
-from udata.models import CommunityResource, Dataset, Topic, Transfer
+from udata.harvest.models import HarvestItem, HarvestJob
+from udata.harvest.tests.factories import HarvestJobFactory
+from udata.models import CommunityResource, Dataset, Discussion, Follow, Topic, Transfer
 
 pytestmark = pytest.mark.usefixtures("clean_db")
 
@@ -32,6 +35,14 @@ def test_purge_datasets():
         comment="comment",
     )
 
+    discussion = DiscussionFactory(subject=datasets[0])
+
+    follower = Follow.objects.create(follower=user, following=datasets[0])
+
+    HarvestJobFactory(items=[HarvestItem(dataset=datasets[0])])
+
+    Dataservice.objects.create(title="test", datasets=datasets)
+
     tasks.purge_datasets()
 
     assert Transfer.objects.filter(id=transfer.id).count() == 0
@@ -39,6 +50,11 @@ def test_purge_datasets():
     topic = Topic.objects(name="test topic").first()
     assert topic.datasets[0] == datasets[1]
 
+    assert Discussion.objects.filter(id=discussion.id).count() == 0
+    assert Follow.objects.filter(id=follower.id).count() == 0
+    assert HarvestJob.objects.filter(items__dataset=datasets[0].id).count() == 0
+    assert Dataservice.objects.filter(datasets=datasets[0].id).count() == 0
+
 
 def test_purge_datasets_community():
     dataset = Dataset.objects.create(title="delete me", deleted="2016-01-01")

{udata-10.0.6.dev33079.dist-info → udata-10.0.6.dev33111.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata
-Version: 10.0.6.dev33079
+Version: 10.0.6.dev33111
 Summary: Open data portal
 Home-page: https://github.com/opendatateam/udata
 Author: Opendata Team
@@ -140,10 +140,13 @@ It is collectively taken care of by members of the
 
 ## Current (in progress)
 
+- Ignore inactive sources at harvest time [#3226](https://github.com/opendatateam/udata/pull/3226)
+    - If some harvest sources were marked inactive by mistake, they won't get executed anymore
 - Organization url is now a URLField and should be a valid URL [#3227](https://github.com/opendatateam/udata/pull/3227)
 - Fix the `parse-url` command [#3225](https://github.com/opendatateam/udata/pull/3225)
 - Add `with_drafts` argument to posts API [#3229](https://github.com/opendatateam/udata/pull/3229)
 - Fix failing dataset save in update reuses metrics [#3230](https://github.com/opendatateam/udata/pull/3230)
+- Update purge tasks [#3167](https://github.com/opendatateam/udata/pull/3167)
 
 ## 10.0.5 (2024-12-09)
 

{udata-10.0.6.dev33079.dist-info → udata-10.0.6.dev33111.dist-info}/RECORD

@@ -88,7 +88,7 @@ udata/core/dataservices/models.py,sha256=lTcC0BqnF7bbFi1S2CZ8Qnk1Fk3TjWoHrvheFB5
 udata/core/dataservices/permissions.py,sha256=98zM_R4v2ZtRubflB7ajaVQz-DVc-pZBMgtKUYy34oI,169
 udata/core/dataservices/rdf.py,sha256=9dWSqONCD5m9HmAU_ndmEmG3F1IuRUOW6MWTmk2GEEI,6792
 udata/core/dataservices/search.py,sha256=htjh3sq3tDuD2qwgRgZsW6ClrHLF6hMwYO4nfBO0Hr4,4171
-udata/core/dataservices/tasks.py,sha256=a43x1oKNF-EYmzM3O-8GQPJvZsgJmQiXOZx8Qti3M4E,960
+udata/core/dataservices/tasks.py,sha256=d2tG1l6u8-eUKUYBOgnCsQLbLmLgJXU-DOzZWhhL6Qg,897
 udata/core/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/core/dataset/actions.py,sha256=mX6xox0PiMrbcAPZ3VZsI26rfM-ciYfEXxN6sqqImKA,1222
 udata/core/dataset/activities.py,sha256=v8k1jwhdx62Z2ARZq8Q-x86OWSsBK99hRloPl74OCgA,1502
@@ -108,7 +108,7 @@ udata/core/dataset/preview.py,sha256=IwCqiNTjjXbtA_SSKF52pwnzKKEz0GyYM95QNn2Dkog
 udata/core/dataset/rdf.py,sha256=TGpwjsFF9hpiC3fxcF7Rej97bJjlzRy2vXbisn-a90A,28271
 udata/core/dataset/search.py,sha256=E7LqHBnq3sMefvmLwTpiw-Ovem2a3NJswHesRjctboE,5627
 udata/core/dataset/signals.py,sha256=WN4sV-lJlNsRkhcnhoy0SYJvCoYmK_5QFYZd1u-h4gs,161
-udata/core/dataset/tasks.py,sha256=-8gasOP2mRu7PYUgxSglkgIW36Vo10eetiaN3pXdAFE,9766
+udata/core/dataset/tasks.py,sha256=jG6U7cwGywhLK-MqUrvbZ1WLl_NLApiThtxmsckGzP8,10009
 udata/core/discussions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/core/discussions/actions.py,sha256=kjdBLDIeu0yWTSxQGgOpN1WoxUqbMygn4SiBk_S9T5I,1051
 udata/core/discussions/api.py,sha256=5yMdKeWp0c9H2ipy5bXoMf82r3ULMIARhvuIdM-XJp0,9446
@@ -289,7 +289,7 @@ udata/harvest/forms.py,sha256=eDDyrK0VuYgCsYo0HY0P6--RDVpWBCuTwktjFiwPQhw,4252
 udata/harvest/models.py,sha256=QBDcdu3N5QILolmiisx9RN5xzfD9tPC06cXO8z5XF4A,6113
 udata/harvest/notifications.py,sha256=8WkHtD68v6rfZC6jCmAuuuRp4NN6q71ZkksZU5m9oJc,867
 udata/harvest/signals.py,sha256=3AhFHMPIFH5vz01NX5ycR_RWH14MXFWnCT6__LSa-QI,1338
-udata/harvest/tasks.py,sha256=uu2bbsZyrh1iya-ar9M5QNGUzDm9loYhaAypdj41_F4,1616
+udata/harvest/tasks.py,sha256=ddJtvE0s-kAYt27-rKH6n8U8vKD8qKczlJtdBJzMUns,1718
 udata/harvest/backends/__init__.py,sha256=QjoFfBJfpw_xgk5YYWI1SgKJOMEmTMlxSfW79GNkSTI,459
 udata/harvest/backends/base.py,sha256=gq39-0SN3DSXfXIv4qjhIkEmPkV-2g6z53V0FikooSs,16413
 udata/harvest/backends/dcat.py,sha256=wbznPWCS9UJzpCWPAnmeUhUCREJg-yoQAHtEfr88RqU,15963
@@ -637,7 +637,7 @@ udata/tests/data/image.jpg,sha256=hdmpaCjOhmAAfNGuTqWKEjv7IC4GXJx-nP_rT274hc8,33
 udata/tests/data/image.png,sha256=GAqXz7w_u7CapODIUF45UpVddmqelnGQkcrwKZq3448,266488
 udata/tests/dataservice/test_csv_adapter.py,sha256=HSnw7HTtzSj0UItKzSvKp3PbwGEni4LGpQ5T63w50MQ,1867
 udata/tests/dataservice/test_dataservice_rdf.py,sha256=VLFgdclFoIhmctfMEgTxvuNlRxcI1yDkZWE-lbEYGdQ,2984
-udata/tests/dataservice/test_dataservice_tasks.py,sha256=n1yTRpyv5DTkX1H0WaxC-T2xxDyN-Sk3bjUiv0C0Bww,1335
+udata/tests/dataservice/test_dataservice_tasks.py,sha256=9XUoYLb_p3dzmgLfswaXjMG2civP0i1kzzP4VdJWE0k,1614
 udata/tests/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/tests/dataset/test_csv_adapter.py,sha256=CSAHEur5-183fRsD_cBgsHnLuohn0DOb9zkVT1q0PNo,3866
 udata/tests/dataset/test_dataset_actions.py,sha256=bgDjVYjOvu3sX_FCTCzf2snZYSprsqor2nAhIVuokSs,722
@@ -645,7 +645,7 @@ udata/tests/dataset/test_dataset_commands.py,sha256=zMPJG2wYwKBee2zI65kmboxf59Zq
 udata/tests/dataset/test_dataset_events.py,sha256=hlrpoOiBbnX_COUI9Pzdqlp45GZZDqu5piwupbnPiTI,3601
 udata/tests/dataset/test_dataset_model.py,sha256=VPF17L5sMpKcFo7JNKmgXnKGAgzy1Ejo470avn3bdcQ,30874
 udata/tests/dataset/test_dataset_rdf.py,sha256=TuKomK0SWqO8R1hQkV8iA3w4YvReO0vVHPfBqKP_oaU,36750
-udata/tests/dataset/test_dataset_tasks.py,sha256=rSafDjCiOyEb2_tVUDN4wqGylF6Yf9VNB769SLmxlwI,2283
+udata/tests/dataset/test_dataset_tasks.py,sha256=n1W2Pg0ez02d66zQG3N93kh7dpR2yLMRDqUI6PnPaI0,3088
 udata/tests/dataset/test_resource_preview.py,sha256=fp9mSL7unhyM66GR0gwhgX3OGQ4TJt7G9xU-CjsL3HI,3908
 udata/tests/features/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/tests/features/territories/__init__.py,sha256=gMD73RL-ymcWvGPDPM0aPxz7WAfd1VEDL8YHRI7HT0Q,956
@@ -710,9 +710,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=18Y5YtzVKInDejw-R-45HNzsB3OVwJ
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=6IQvFk0NTDV5Jq-kLkkzpioWfrMaLDa1oQSevKFbxKQ,44943
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=LWzCNS68hS2lbeo1bncZw4ZKqV2Dk_JUC-4SiF3o994,28163
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=X1ieIsJDmk_A_YiblpmUBzWWrVHta9F8yzCZLrjeW-Y,51435
-udata-10.0.6.dev33079.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-10.0.6.dev33079.dist-info/METADATA,sha256=7lwAKUC5GkDPXlisMKYZRCMY5fb_u-2T0FzoS_cKmhE,137645
-udata-10.0.6.dev33079.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
-udata-10.0.6.dev33079.dist-info/entry_points.txt,sha256=3SKiqVy4HUqxf6iWspgMqH8d88Htk6KoLbG1BU-UddQ,451
-udata-10.0.6.dev33079.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
-udata-10.0.6.dev33079.dist-info/RECORD,,
+udata-10.0.6.dev33111.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-10.0.6.dev33111.dist-info/METADATA,sha256=jNpE_i0DJk2-zjpIJXA0Tv8uu3cv8aYqygdH5Jh68ZY,137917
+udata-10.0.6.dev33111.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
+udata-10.0.6.dev33111.dist-info/entry_points.txt,sha256=3SKiqVy4HUqxf6iWspgMqH8d88Htk6KoLbG1BU-UddQ,451
+udata-10.0.6.dev33111.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
+udata-10.0.6.dev33111.dist-info/RECORD,,