udata 9.1.2.dev30355__py2.py3-none-any.whl → 9.1.2.dev30382__py2.py3-none-any.whl

udata/harvest/tests/test_actions.py

@@ -1,37 +1,42 @@
 import csv
 import logging
-import pytest
-
 from datetime import datetime, timedelta
 from tempfile import NamedTemporaryFile
 
+import pytest
 from mock import patch
 
-from udata.models import Dataset, PeriodicTask
-
-from udata.core.organization.factories import OrganizationFactory
-from udata.core.user.factories import UserFactory
 from udata.core.dataset.factories import DatasetFactory
 from udata.core.dataset.models import HarvestDatasetMetadata
+from udata.core.organization.factories import OrganizationFactory
+from udata.core.user.factories import UserFactory
+from udata.models import Dataset, PeriodicTask
 from udata.tests.helpers import assert_emit, assert_equal_dates
-from udata.utils import faker, Paginable
+from udata.utils import Paginable, faker
 
-from .factories import (
-    HarvestSourceFactory, HarvestJobFactory, MockBackendsMixin,
-    mock_initialize, mock_process, DEFAULT_COUNT as COUNT
-)
+from .. import actions, signals
+from ..backends import BaseBackend
 from ..models import (
-    HarvestSource, HarvestJob, HarvestError,
-    VALIDATION_PENDING, VALIDATION_ACCEPTED, VALIDATION_REFUSED
+    VALIDATION_ACCEPTED,
+    VALIDATION_PENDING,
+    VALIDATION_REFUSED,
+    HarvestError,
+    HarvestJob,
+    HarvestSource,
+)
+from .factories import DEFAULT_COUNT as COUNT
+from .factories import (
+    HarvestJobFactory,
+    HarvestSourceFactory,
+    MockBackendsMixin,
+    mock_initialize,
+    mock_process,
 )
-from ..backends import BaseBackend
-from .. import actions, signals
-
 
 log = logging.getLogger(__name__)
 
 pytestmark = [
-    pytest.mark.usefixtures('clean_db'),
+    pytest.mark.usefixtures("clean_db"),
 ]
 
 
@@ -163,15 +168,13 @@ class HarvestActionsTest:
         source_url = faker.url()
 
         with assert_emit(signals.harvest_source_created):
-            source = actions.create_source('Test source',
-                                           source_url,
-                                           'factory')
+            source = actions.create_source("Test source", source_url, "factory")
 
-        assert source.name == 'Test source'
-        assert source.slug == 'test-source'
+        assert source.name == "Test source"
+        assert source.slug == "test-source"
         assert source.url == source_url
-        assert source.backend == 'factory'
-        assert source.frequency == 'manual'
+        assert source.backend == "factory"
+        assert source.frequency == "manual"
         assert source.active
         assert source.owner is None
         assert source.organization is None
@@ -184,15 +187,12 @@ class HarvestActionsTest:
     def test_create_source_with_config(self):
         source_url = faker.url()
         config = {
-            'filters': [{'key': 'test', 'value': 42}],
-            'features': {'key': True},
+            "filters": [{"key": "test", "value": 42}],
+            "features": {"key": True},
         }
 
         with assert_emit(signals.harvest_source_created):
-            source = actions.create_source('Test source',
-                                           source_url,
-                                           'factory',
-                                           config=config)
+            source = actions.create_source("Test source", source_url, "factory", config=config)
 
         assert source.config == config
 
@@ -200,7 +200,7 @@ class HarvestActionsTest:
         source = HarvestSourceFactory()
         data = source.to_dict()
         new_url = faker.url()
-        data['url'] = new_url
+        data["url"] = new_url
 
         with assert_emit(signals.harvest_source_updated):
             source = actions.update_source(source.id, data)
@@ -209,7 +209,7 @@ class HarvestActionsTest:
         source.reload()
         assert source.url == new_url
 
-    @patch('udata.harvest.actions.launch')
+    @patch("udata.harvest.actions.launch")
     def test_validate_source(self, mock):
         source = HarvestSourceFactory()
 
@@ -224,30 +224,30 @@ class HarvestActionsTest:
 
         assert source.periodic_task is not None
 
-    @patch('udata.harvest.actions.launch')
+    @patch("udata.harvest.actions.launch")
     def test_validate_source_with_comment(self, mock):
         source = HarvestSourceFactory()
 
-        actions.validate_source(source.id, 'comment')
+        actions.validate_source(source.id, "comment")
 
         source.reload()
 
         assert source.validation.state == VALIDATION_ACCEPTED
         assert source.validation.on is not None
         assert source.validation.by is None
-        assert source.validation.comment == 'comment'
+        assert source.validation.comment == "comment"
         mock.assert_called_once_with(source.id)
 
     def test_reject_source(self):
         source = HarvestSourceFactory()
 
-        actions.reject_source(source.id, 'comment')
+        actions.reject_source(source.id, "comment")
 
         source.reload()
         assert source.validation.state == VALIDATION_REFUSED
         assert source.validation.on is not None
         assert source.validation.by is None
-        assert source.validation.comment == 'comment'
+        assert source.validation.comment == "comment"
 
     def test_get_source_by_slug(self):
         source = HarvestSourceFactory()
@@ -291,7 +291,7 @@ class HarvestActionsTest:
         deleted_sources = HarvestSource.objects(deleted__exists=True)
         assert len(deleted_sources) == 1
 
-    @pytest.mark.parametrize('by_attr', ['source.id', 'str(source.id)', 'source.slug'])
+    @pytest.mark.parametrize("by_attr", ["source.id", "str(source.id)", "source.slug"])
     def test_clean_source(self, by_attr):
         source = HarvestSourceFactory()
         for _ in range(5):
@@ -319,30 +319,30 @@ class HarvestActionsTest:
         periodic_task = source.periodic_task
         assert periodic_task == PeriodicTask.objects.first()
         assert periodic_task.args == [str(source.id)]
-        assert periodic_task.crontab.hour == '0'
-        assert periodic_task.crontab.minute == '*'
-        assert periodic_task.crontab.day_of_week == '*'
-        assert periodic_task.crontab.day_of_month == '*'
-        assert periodic_task.crontab.month_of_year == '*'
+        assert periodic_task.crontab.hour == "0"
+        assert periodic_task.crontab.minute == "*"
+        assert periodic_task.crontab.day_of_week == "*"
+        assert periodic_task.crontab.day_of_month == "*"
+        assert periodic_task.crontab.month_of_year == "*"
         assert periodic_task.enabled
-        assert periodic_task.name == 'Harvest {0}'.format(source.name)
+        assert periodic_task.name == "Harvest {0}".format(source.name)
 
     def test_schedule_from_cron(self):
         source = HarvestSourceFactory()
         with assert_emit(signals.harvest_source_scheduled):
-            source = actions.schedule(str(source.id), '0 1 2 3 sunday')
+            source = actions.schedule(str(source.id), "0 1 2 3 sunday")
 
         assert len(PeriodicTask.objects) == 1
         periodic_task = source.periodic_task
         assert periodic_task == PeriodicTask.objects.first()
         assert periodic_task.args == [str(source.id)]
-        assert periodic_task.crontab.minute == '0'
-        assert periodic_task.crontab.hour == '1'
-        assert periodic_task.crontab.day_of_month == '2'
-        assert periodic_task.crontab.month_of_year == '3'
-        assert periodic_task.crontab.day_of_week == 'sunday'
+        assert periodic_task.crontab.minute == "0"
+        assert periodic_task.crontab.hour == "1"
+        assert periodic_task.crontab.day_of_month == "2"
+        assert periodic_task.crontab.month_of_year == "3"
+        assert periodic_task.crontab.day_of_week == "sunday"
         assert periodic_task.enabled
-        assert periodic_task.name == 'Harvest {0}'.format(source.name)
+        assert periodic_task.name == "Harvest {0}".format(source.name)
 
     def test_reschedule(self):
         source = HarvestSourceFactory()
@@ -356,21 +356,21 @@ class HarvestActionsTest:
         periodic_task = source.periodic_task
         assert periodic_task == PeriodicTask.objects.first()
         assert periodic_task.args == [str(source.id)]
-        assert periodic_task.crontab.hour == '*'
-        assert periodic_task.crontab.minute == '0'
-        assert periodic_task.crontab.day_of_week == '*'
-        assert periodic_task.crontab.day_of_month == '*'
-        assert periodic_task.crontab.month_of_year == '*'
+        assert periodic_task.crontab.hour == "*"
+        assert periodic_task.crontab.minute == "0"
+        assert periodic_task.crontab.day_of_week == "*"
+        assert periodic_task.crontab.day_of_month == "*"
+        assert periodic_task.crontab.month_of_year == "*"
         assert periodic_task.enabled
-        assert periodic_task.name == 'Harvest {0}'.format(source.name)
+        assert periodic_task.name == "Harvest {0}".format(source.name)
 
     def test_unschedule(self):
         periodic_task = PeriodicTask.objects.create(
-            task='harvest',
+            task="harvest",
             name=faker.name(),
             description=faker.sentence(),
             enabled=True,
-            crontab=PeriodicTask.Crontab()
+            crontab=PeriodicTask.Crontab(),
         )
         source = HarvestSourceFactory(periodic_task=periodic_task)
         with assert_emit(signals.harvest_source_unscheduled):
@@ -382,17 +382,15 @@ class HarvestActionsTest:
 
     def test_purge_sources(self):
         periodic_task = PeriodicTask.objects.create(
-            task='harvest',
+            task="harvest",
             name=faker.name(),
             description=faker.sentence(),
             enabled=True,
-            crontab=PeriodicTask.Crontab()
+            crontab=PeriodicTask.Crontab(),
         )
         now = datetime.utcnow()
         to_delete = HarvestSourceFactory.create_batch(2, deleted=now)
-        to_delete.append(
-            HarvestSourceFactory(periodic_task=periodic_task, deleted=now)
-        )
+        to_delete.append(HarvestSourceFactory(periodic_task=periodic_task, deleted=now))
         to_keep = HarvestSourceFactory.create_batch(2)
         harvest_job = HarvestJobFactory(source=to_delete[0])
         dataset_to_archive = DatasetFactory(
@@ -406,7 +404,7 @@ class HarvestActionsTest:
         assert len(HarvestSource.objects) == len(to_keep)
         assert PeriodicTask.objects.filter(id=periodic_task.id).count() == 0
         assert HarvestJob.objects(id=harvest_job.id).count() == 0
-        assert dataset_to_archive.harvest.archived == 'harvester-deleted'
+        assert dataset_to_archive.harvest.archived == "harvester-deleted"
         assert_equal_dates(dataset_to_archive.archived, now)
 
     @pytest.mark.options(HARVEST_JOBS_RETENTION_DAYS=2)
@@ -428,91 +426,71 @@ class HarvestActionsTest:
     def test_attach(self):
         datasets = DatasetFactory.create_batch(3)
 
-        with NamedTemporaryFile(mode='w') as csvfile:
-            writer = csv.DictWriter(csvfile,
-                                    fieldnames=['local', 'remote'],
-                                    delimiter=';',
-                                    quotechar='"')
+        with NamedTemporaryFile(mode="w") as csvfile:
+            writer = csv.DictWriter(
+                csvfile, fieldnames=["local", "remote"], delimiter=";", quotechar='"'
+            )
 
             writer.writeheader()
             for index, dataset in enumerate(datasets):
-                writer.writerow({
-                    'local': str(dataset.id),
-                    'remote': str(index)
-                })
+                writer.writerow({"local": str(dataset.id), "remote": str(index)})
             csvfile.flush()
 
-            result = actions.attach('test.org', csvfile.name)
+            result = actions.attach("test.org", csvfile.name)
 
         assert result.success == len(datasets)
         assert result.errors == 0
         for index, dataset in enumerate(datasets):
             dataset.reload()
-            assert dataset.harvest.domain == 'test.org'
+            assert dataset.harvest.domain == "test.org"
             assert dataset.harvest.remote_id == str(index)
 
     def test_attach_does_not_duplicate(self):
         attached_datasets = []
         for i in range(2):
             dataset = DatasetFactory.build()
-            dataset.harvest = HarvestDatasetMetadata(
-                domain='test.org',
-                remote_id=str(i)
-            )
+            dataset.harvest = HarvestDatasetMetadata(domain="test.org", remote_id=str(i))
             dataset.last_modified_internal = datetime.utcnow()
             dataset.save()
             attached_datasets.append(dataset)
 
         datasets = DatasetFactory.create_batch(3)
 
-        with NamedTemporaryFile(mode='w') as csvfile:
-            writer = csv.DictWriter(csvfile,
-                                    fieldnames=['local', 'remote'],
-                                    delimiter=';',
-                                    quotechar='"')
+        with NamedTemporaryFile(mode="w") as csvfile:
+            writer = csv.DictWriter(
+                csvfile, fieldnames=["local", "remote"], delimiter=";", quotechar='"'
+            )
 
             writer.writeheader()
             for index, dataset in enumerate(datasets):
-                writer.writerow({
-                    'local': str(dataset.id),
-                    'remote': str(index)
-                })
+                writer.writerow({"local": str(dataset.id), "remote": str(index)})
             csvfile.flush()
 
-            result = actions.attach('test.org', csvfile.name)
+            result = actions.attach("test.org", csvfile.name)
 
-        dbcount = Dataset.objects(**{
-            'harvest__remote_id__exists': True
-        }).count()
+        dbcount = Dataset.objects(**{"harvest__remote_id__exists": True}).count()
         assert result.success == len(datasets)
         assert dbcount == result.success
         for index, dataset in enumerate(datasets):
             dataset.reload()
-            assert dataset.harvest.domain == 'test.org'
+            assert dataset.harvest.domain == "test.org"
             assert dataset.harvest.remote_id == str(index)
 
     def test_attach_skip_not_found(self):
         datasets = DatasetFactory.create_batch(3)
 
-        with NamedTemporaryFile(mode='w') as csvfile:
-            writer = csv.DictWriter(csvfile,
-                                    fieldnames=['local', 'remote'],
-                                    delimiter=';',
-                                    quotechar='"')
+        with NamedTemporaryFile(mode="w") as csvfile:
+            writer = csv.DictWriter(
+                csvfile, fieldnames=["local", "remote"], delimiter=";", quotechar='"'
+            )
 
             writer.writeheader()
-            writer.writerow({
-                'local': 'not-found',
-                'remote': '42'
-            })
+            writer.writerow({"local": "not-found", "remote": "42"})
             for index, dataset in enumerate(datasets):
-                writer.writerow({
-                    'local': str(dataset.id),
-                    'remote': str(index)
-                })
+                writer.writerow({"local": str(dataset.id), "remote": str(index)})
             csvfile.flush()
 
-            result = actions.attach('test.org', csvfile.name)
+            result = actions.attach("test.org", csvfile.name)
 
         assert result.success == len(datasets)
         assert result.errors == 1
@@ -521,9 +499,8 @@ class HarvestActionsTest:
 class ExecutionTestMixin(MockBackendsMixin):
     def test_default(self):
         org = OrganizationFactory()
-        source = HarvestSourceFactory(backend='factory', organization=org)
-        with assert_emit(signals.before_harvest_job,
-                         signals.after_harvest_job):
+        source = HarvestSourceFactory(backend="factory", organization=org)
+        with assert_emit(signals.before_harvest_job, signals.after_harvest_job):
             self.action(source.slug)
 
         source.reload()
@@ -531,14 +508,14 @@ class ExecutionTestMixin(MockBackendsMixin):
 
         job = source.get_last_job()
 
-        assert job.status == 'done'
+        assert job.status == "done"
         assert job.errors == []
         assert job.started is not None
         assert job.ended is not None
         assert len(job.items) == COUNT
 
         for item in job.items:
-            assert item.status == 'done'
+            assert item.status == "done"
             assert item.errors == []
             assert item.started is not None
             assert item.ended is not None
@@ -547,27 +524,26 @@ class ExecutionTestMixin(MockBackendsMixin):
             dataset = item.dataset
             assert Dataset.objects(id=dataset.id).first() is not None
             assert dataset.organization == org
-            assert 'remote_id' in dataset.harvest
-            assert 'last_update' in dataset.harvest
-            assert 'source_id' in dataset.harvest
+            assert "remote_id" in dataset.harvest
+            assert "last_update" in dataset.harvest
+            assert "source_id" in dataset.harvest
 
         assert len(HarvestJob.objects) == 1
         assert len(Dataset.objects) == COUNT
 
     def test_error_on_initialize(self):
         def init(self):
-            raise ValueError('test')
+            raise ValueError("test")
 
-        source = HarvestSourceFactory(backend='factory')
-        with assert_emit(signals.before_harvest_job),\
-                mock_initialize.connected_to(init):
+        source = HarvestSourceFactory(backend="factory")
+        with assert_emit(signals.before_harvest_job), mock_initialize.connected_to(init):
             self.action(source.slug)
 
         source.reload()
         assert len(HarvestJob.objects(source=source)) == 1
 
         job = source.get_last_job()
-        assert job.status == 'failed'
+        assert job.status == "failed"
         assert len(job.errors) == 1
         error = job.errors[0]
         assert isinstance(error, HarvestError)
@@ -580,20 +556,21 @@ class ExecutionTestMixin(MockBackendsMixin):
 
     def test_error_on_item(self):
         def process(self, item):
-            if item == '1':
-                raise ValueError('test')
-
-        source = HarvestSourceFactory(backend='factory')
-        with assert_emit(signals.before_harvest_job,
-                         signals.after_harvest_job), \
-                mock_process.connected_to(process):
+            if item == "1":
+                raise ValueError("test")
+
+        source = HarvestSourceFactory(backend="factory")
+        with (
+            assert_emit(signals.before_harvest_job, signals.after_harvest_job),
+            mock_process.connected_to(process),
+        ):
             self.action(source.slug)
 
         source.reload()
         assert len(HarvestJob.objects(source=source)) == 1
 
         job = source.get_last_job()
-        assert job.status == 'done-errors'
+        assert job.status == "done-errors"
         assert job.started is not None
         assert job.ended is not None
         assert len(job.errors) == 0
@@ -605,13 +582,13 @@ class ExecutionTestMixin(MockBackendsMixin):
         for item in items_ok:
             assert item.started is not None
             assert item.ended is not None
-            assert item.status == 'done'
+            assert item.status == "done"
             assert item.errors == []
 
         item_ko = next(i for i in job.items if i.errors)
         assert item_ko.started is not None
         assert item_ko.ended is not None
-        assert item_ko.status == 'failed'
+        assert item_ko.status == "failed"
         assert len(item_ko.errors) == 1
 
         error = item_ko.errors[0]
@@ -621,9 +598,8 @@ class ExecutionTestMixin(MockBackendsMixin):
         assert len(Dataset.objects) == COUNT - 1
 
     def test_empty(self):
-        source = HarvestSourceFactory(backend='factory', config={'count': 0})
-        with assert_emit(signals.before_harvest_job,
-                         signals.after_harvest_job):
+        source = HarvestSourceFactory(backend="factory", config={"count": 0})
+        with assert_emit(signals.before_harvest_job, signals.after_harvest_job):
             self.action(source.slug)
 
         source.reload()
@@ -631,7 +607,7 @@ class ExecutionTestMixin(MockBackendsMixin):
 
         job = source.get_last_job()
 
-        assert job.status == 'done'
+        assert job.status == "done"
         assert job.errors == []
         assert job.started is not None
         assert job.ended is not None
@@ -643,9 +619,7 @@ class ExecutionTestMixin(MockBackendsMixin):
     @pytest.mark.options(HARVEST_MAX_ITEMS=5)
     def test_harvest_max_items(self):
         org = OrganizationFactory()
-        source = HarvestSourceFactory(backend='factory',
-                                      organization=org,
-                                      config={'count': 10})
+        source = HarvestSourceFactory(backend="factory", organization=org, config={"count": 10})
 
         self.action(source.slug)
         assert len(Dataset.objects) == 5
@@ -664,18 +638,18 @@ class HarvestRunTest(ExecutionTestMixin):
 class HarvestPreviewTest(MockBackendsMixin):
     def test_preview(self):
         org = OrganizationFactory()
-        source = HarvestSourceFactory(backend='factory', organization=org)
+        source = HarvestSourceFactory(backend="factory", organization=org)
 
         job = actions.preview(source.slug)
 
-        assert job.status == 'done'
+        assert job.status == "done"
         assert job.errors == []
         assert job.started is not None
         assert job.ended is not None
         assert len(job.items) == COUNT
 
         for item in job.items:
-            assert item.status == 'done'
+            assert item.status == "done"
             assert item.errors == []
             assert item.started is not None
             assert item.ended is not None
@@ -683,9 +657,9 @@ class HarvestPreviewTest(MockBackendsMixin):
 
             dataset = item.dataset
             assert dataset.organization == org
-            assert 'remote_id' in dataset.harvest
-            assert 'last_update' in dataset.harvest
-            assert 'source_id' in dataset.harvest
+            assert "remote_id" in dataset.harvest
+            assert "last_update" in dataset.harvest
+            assert "source_id" in dataset.harvest
 
         assert len(HarvestJob.objects) == 0
         assert len(Dataset.objects) == 0
@@ -693,9 +667,7 @@ class HarvestPreviewTest(MockBackendsMixin):
     @pytest.mark.options(HARVEST_PREVIEW_MAX_ITEMS=5)
     def test_preview_max_items(self):
         org = OrganizationFactory()
-        source = HarvestSourceFactory(backend='factory',
-                                      organization=org,
-                                      config={'count': 10})
+        source = HarvestSourceFactory(backend="factory", organization=org, config={"count": 10})
 
         job = actions.preview(source.slug)
 
@@ -703,14 +675,14 @@ class HarvestPreviewTest(MockBackendsMixin):
 
     def test_preview_with_error_on_initialize(self):
         def init(self):
-            raise ValueError('test')
+            raise ValueError("test")
 
-        source = HarvestSourceFactory(backend='factory')
+        source = HarvestSourceFactory(backend="factory")
 
         with mock_initialize.connected_to(init):
             job = actions.preview(source.slug)
 
-        assert job.status == 'failed'
+        assert job.status == "failed"
         assert len(job.errors) == 1
         error = job.errors[0]
         assert isinstance(error, HarvestError)
@@ -723,15 +695,15 @@ class HarvestPreviewTest(MockBackendsMixin):
 
     def test_preview_with_error_on_item(self):
         def process(self, item):
-            if item == '1':
-                raise ValueError('test')
+            if item == "1":
+                raise ValueError("test")
 
-        source = HarvestSourceFactory(backend='factory')
+        source = HarvestSourceFactory(backend="factory")
 
         with mock_process.connected_to(process):
             job = actions.preview(source.slug)
 
-        assert job.status == 'done-errors'
+        assert job.status == "done-errors"
         assert job.started is not None
         assert job.ended is not None
         assert len(job.errors) == 0
@@ -743,13 +715,13 @@ class HarvestPreviewTest(MockBackendsMixin):
         for item in items_ok:
             assert item.started is not None
             assert item.ended is not None
-            assert item.status == 'done'
+            assert item.status == "done"
             assert item.errors == []
 
         item_ko = next(i for i in job.items if i.errors)
         assert item_ko.started is not None
         assert item_ko.ended is not None
-        assert item_ko.status == 'failed'
+        assert item_ko.status == "failed"
         assert len(item_ko.errors) == 1
 
         error = item_ko.errors[0]
@@ -762,20 +734,18 @@ class HarvestPreviewTest(MockBackendsMixin):
         org = OrganizationFactory()
         source_url = faker.url()
         count = 10
-        job = actions.preview_from_config('Test source',
-                                          source_url,
-                                          'factory',
-                                          organization=org,
-                                          config={'count': count})
+        job = actions.preview_from_config(
+            "Test source", source_url, "factory", organization=org, config={"count": count}
+        )
 
-        assert job.status == 'done'
+        assert job.status == "done"
         assert job.errors == []
         assert job.started is not None
         assert job.ended is not None
         assert len(job.items) == count
 
         for item in job.items:
-            assert item.status == 'done'
+            assert item.status == "done"
             assert item.errors == []
             assert item.started is not None
             assert item.ended is not None
@@ -783,9 +753,9 @@ class HarvestPreviewTest(MockBackendsMixin):
 
             dataset = item.dataset
             assert dataset.organization == org
-            assert 'remote_id' in dataset.harvest
-            assert 'last_update' in dataset.harvest
-            assert 'source_id' in dataset.harvest
+            assert "remote_id" in dataset.harvest
+            assert "last_update" in dataset.harvest
+            assert "source_id" in dataset.harvest
 
         assert len(HarvestJob.objects) == 0
         assert len(Dataset.objects) == 0