udata 9.1.2.dev30355__py2.py3-none-any.whl → 9.1.2.dev30382__py2.py3-none-any.whl

udata/tests/test_storages.py

@@ -1,230 +1,241 @@
-import pytest
-
 from datetime import datetime, timedelta
 from io import BytesIO
+from os.path import basename
 from uuid import uuid4
 
-from flask import url_for, json
+import pytest
+from flask import json, url_for
+from werkzeug.test import EnvironBuilder
+from werkzeug.wrappers import Request
 
 from udata.core import storages
 from udata.core.storages import utils
-from udata.core.storages.api import chunk_filename, META
+from udata.core.storages.api import META, chunk_filename
 from udata.core.storages.tasks import purge_chunks
 from udata.utils import faker
 
 from .helpers import assert200, assert400
 
-from os.path import basename
-
-from werkzeug.test import EnvironBuilder
-from werkzeug.wrappers import Request
-
 
 class StorageUtilsTest:
-    '''
+    """
     Perform all tests on a file of size 2 * CHUNCK_SIZE = 2 * (2 ** 16).
     Expected values are precomputed with shell `md5sum`, `sha1sum`...
-    '''
+    """
+
     @pytest.fixture(autouse=True)
     def write_file(self, tmpdir):
-        tmpfile = tmpdir.join('test.txt')
-        tmpfile.write_binary(b'a' * 2 * (2 ** 16))
+        tmpfile = tmpdir.join("test.txt")
+        tmpfile.write_binary(b"a" * 2 * (2**16))
         self.file = self.filestorage(str(tmpfile))
 
     def filestorage(self, filename):
-        data = open(filename, 'rb')
-        builder = EnvironBuilder(method='POST', data={
-            'file': (data, basename(filename))
-        })
+        data = open(filename, "rb")
+        builder = EnvironBuilder(method="POST", data={"file": (data, basename(filename))})
         env = builder.get_environ()
         req = Request(env)
-        return req.files['file']
+        return req.files["file"]
 
     def test_sha1(self):
         # Output of sha1sum
-        expected = 'ce5653590804baa9369f72d483ed9eba72f04d29'
+        expected = "ce5653590804baa9369f72d483ed9eba72f04d29"
         assert utils.sha1(self.file) == expected
 
     def test_md5(self):
-        expected = '81615449a98aaaad8dc179b3bec87f38'  # Output of md5sum
+        expected = "81615449a98aaaad8dc179b3bec87f38"  # Output of md5sum
         assert utils.md5(self.file) == expected
 
     def test_crc32(self):
-        expected = 'CA975130'  # Output of cksfv
+        expected = "CA975130"  # Output of cksfv
         assert utils.crc32(self.file) == expected
 
     def test_mime(self):
-        assert utils.mime('test.txt') == 'text/plain'
-        assert utils.mime('test') is None
+        assert utils.mime("test.txt") == "text/plain"
+        assert utils.mime("test") is None
 
     def test_extension_default(self, app):
-        assert utils.extension('test.txt') == 'txt'
-        assert utils.extension('prefix/test.txt') == 'txt'
-        assert utils.extension('prefix.with.dot/test.txt') == 'txt'
+        assert utils.extension("test.txt") == "txt"
+        assert utils.extension("prefix/test.txt") == "txt"
+        assert utils.extension("prefix.with.dot/test.txt") == "txt"
 
     def test_extension_compound(self, app):
-        assert utils.extension('test.tar.gz') == 'tar.gz'
-        assert utils.extension('prefix.with.dot/test.tar.gz') == 'tar.gz'
+        assert utils.extension("test.tar.gz") == "tar.gz"
+        assert utils.extension("prefix.with.dot/test.tar.gz") == "tar.gz"
 
     def test_extension_compound_with_allowed_extension(self, app):
-        assert utils.extension('test.2022.csv.tar.gz') == 'csv.tar.gz'
-        assert utils.extension('prefix.with.dot/test.2022.csv.tar.gz') == 'csv.tar.gz'
+        assert utils.extension("test.2022.csv.tar.gz") == "csv.tar.gz"
+        assert utils.extension("prefix.with.dot/test.2022.csv.tar.gz") == "csv.tar.gz"
 
     def test_extension_compound_without_allowed_extension(self, app):
-        assert utils.extension('test.2022.tar.gz') == 'tar.gz'
-        assert utils.extension('prefix.with.dot/test.2022.tar.gz') == 'tar.gz'
+        assert utils.extension("test.2022.tar.gz") == "tar.gz"
+        assert utils.extension("prefix.with.dot/test.2022.tar.gz") == "tar.gz"
 
     def test_no_extension(self, app):
-        assert utils.extension('test') is None
-        assert utils.extension('prefix/test') is None
+        assert utils.extension("test") is None
+        assert utils.extension("prefix/test") is None
 
     def test_normalize_no_changes(self):
-        assert utils.normalize('test.txt') == 'test.txt'
+        assert utils.normalize("test.txt") == "test.txt"
 
     def test_normalize_spaces(self):
-        expected = 'test-with-spaces.txt'
-        assert utils.normalize('test with  spaces.txt') == expected
+        expected = "test-with-spaces.txt"
+        assert utils.normalize("test with  spaces.txt") == expected
 
     def test_normalize_to_lower(self):
-        assert utils.normalize('Test.TXT') == 'test.txt'
+        assert utils.normalize("Test.TXT") == "test.txt"
 
     def test_normalize_special_chars(self):
-        assert utils.normalize('éàü@€.txt') == 'eau-eur.txt'
+        assert utils.normalize("éàü@€.txt") == "eau-eur.txt"
 
 
-@pytest.mark.usefixtures('app')
+@pytest.mark.usefixtures("app")
 class ConfigurableAllowedExtensionsTest:
     def test_has_default(self):
-        assert 'csv' in storages.CONFIGURABLE_AUTHORIZED_TYPES
-        assert 'xml' in storages.CONFIGURABLE_AUTHORIZED_TYPES
-        assert 'json' in storages.CONFIGURABLE_AUTHORIZED_TYPES
-        assert 'exe' not in storages.CONFIGURABLE_AUTHORIZED_TYPES
-        assert 'bat' not in storages.CONFIGURABLE_AUTHORIZED_TYPES
+        assert "csv" in storages.CONFIGURABLE_AUTHORIZED_TYPES
+        assert "xml" in storages.CONFIGURABLE_AUTHORIZED_TYPES
+        assert "json" in storages.CONFIGURABLE_AUTHORIZED_TYPES
+        assert "exe" not in storages.CONFIGURABLE_AUTHORIZED_TYPES
+        assert "bat" not in storages.CONFIGURABLE_AUTHORIZED_TYPES
 
-    @pytest.mark.options(ALLOWED_RESOURCES_EXTENSIONS=['csv', 'json'])
+    @pytest.mark.options(ALLOWED_RESOURCES_EXTENSIONS=["csv", "json"])
     def test_with_config(self):
-        assert 'csv' in storages.CONFIGURABLE_AUTHORIZED_TYPES
-        assert 'json' in storages.CONFIGURABLE_AUTHORIZED_TYPES
-        assert 'xml' not in storages.CONFIGURABLE_AUTHORIZED_TYPES
-        assert 'exe' not in storages.CONFIGURABLE_AUTHORIZED_TYPES
-        assert 'bat' not in storages.CONFIGURABLE_AUTHORIZED_TYPES
+        assert "csv" in storages.CONFIGURABLE_AUTHORIZED_TYPES
+        assert "json" in storages.CONFIGURABLE_AUTHORIZED_TYPES
+        assert "xml" not in storages.CONFIGURABLE_AUTHORIZED_TYPES
+        assert "exe" not in storages.CONFIGURABLE_AUTHORIZED_TYPES
+        assert "bat" not in storages.CONFIGURABLE_AUTHORIZED_TYPES
 
 
-@pytest.mark.usefixtures('instance_path')
+@pytest.mark.usefixtures("instance_path")
 class StorageUploadViewTest:
     def test_standard_upload(self, client):
         client.login()
         response = client.post(
-            url_for('test-storage.upload', name='resources'),
-            {'file': (BytesIO(b'aaa'), 'Test with  spaces.TXT')})
+            url_for("test-storage.upload", name="resources"),
+            {"file": (BytesIO(b"aaa"), "Test with  spaces.TXT")},
+        )
 
         assert200(response)
-        assert response.json['success']
-        assert 'url' in response.json
-        assert 'size' in response.json
-        assert 'sha1' in response.json
-        assert 'filename' in response.json
-        filename = response.json['filename']
-        assert filename.endswith('test-with-spaces.txt')
+        assert response.json["success"]
+        assert "url" in response.json
+        assert "size" in response.json
+        assert "sha1" in response.json
+        assert "filename" in response.json
+        filename = response.json["filename"]
+        assert filename.endswith("test-with-spaces.txt")
         expected = storages.resources.url(filename, external=True)
-        assert response.json['url'] == expected
-        assert response.json['mime'] == 'text/plain'
+        assert response.json["url"] == expected
+        assert response.json["mime"] == "text/plain"
 
     def test_chunked_upload(self, client):
         client.login()
-        url = url_for('test-storage.upload', name='tmp')
+        url = url_for("test-storage.upload", name="tmp")
         uuid = str(uuid4())
         parts = 4
 
         for i in range(parts):
-            response = client.post(url, {
-                'file': (BytesIO(b'a'), 'blob'),
-                'uuid': uuid,
-                'filename': 'Test with  spaces.TXT',
-                'partindex': i,
-                'partbyteoffset': 0,
-                'totalfilesize': parts,
-                'totalparts': parts,
-                'chunksize': 1,
-            })
+            response = client.post(
+                url,
+                {
+                    "file": (BytesIO(b"a"), "blob"),
+                    "uuid": uuid,
+                    "filename": "Test with  spaces.TXT",
+                    "partindex": i,
+                    "partbyteoffset": 0,
+                    "totalfilesize": parts,
+                    "totalparts": parts,
+                    "chunksize": 1,
+                },
+            )
 
             assert200(response)
-            assert response.json['success']
-            assert 'filename' not in response.json
-            assert 'url' not in response.json
-            assert 'size' not in response.json
-            assert 'sha1' not in response.json
-            assert 'url' not in response.json
-
-        response = client.post(url, {
-            'uuid': uuid,
-            'filename': 'Test with  spaces.TXT',
-            'totalfilesize': parts,
-            'totalparts': parts,
-        })
-        assert 'filename' in response.json
-        assert 'url' in response.json
-        assert 'size' in response.json
-        assert response.json['size'] == parts
-        assert 'sha1' in response.json
-        expected_filename = 'test-with-spaces.txt'
-        filename = response.json['filename']
+            assert response.json["success"]
+            assert "filename" not in response.json
+            assert "url" not in response.json
+            assert "size" not in response.json
+            assert "sha1" not in response.json
+            assert "url" not in response.json
+
+        response = client.post(
+            url,
+            {
+                "uuid": uuid,
+                "filename": "Test with  spaces.TXT",
+                "totalfilesize": parts,
+                "totalparts": parts,
+            },
+        )
+        assert "filename" in response.json
+        assert "url" in response.json
+        assert "size" in response.json
+        assert response.json["size"] == parts
+        assert "sha1" in response.json
+        expected_filename = "test-with-spaces.txt"
+        filename = response.json["filename"]
         assert filename.endswith(expected_filename)
         expected_url = storages.tmp.url(filename, external=True)
-        assert response.json['url'] == expected_url
-        assert response.json['mime'] == 'text/plain'
-        assert storages.tmp.read(filename) == b'aaaa'
+        assert response.json["url"] == expected_url
+        assert response.json["mime"] == "text/plain"
+        assert storages.tmp.read(filename) == b"aaaa"
         assert list(storages.chunks.list_files()) == []
 
     def test_chunked_upload_bad_chunk(self, client):
         client.login()
-        url = url_for('test-storage.upload', name='tmp')
+        url = url_for("test-storage.upload", name="tmp")
         uuid = str(uuid4())
         parts = 4
 
-        response = client.post(url, {
-            'file': (BytesIO(b'a'), 'blob'),
-            'uuid': uuid,
-            'filename': 'test.txt',
-            'partindex': 0,
-            'partbyteoffset': 0,
-            'totalfilesize': parts,
-            'totalparts': parts,
-            'chunksize': 10,  # Does not match
-        })
+        response = client.post(
+            url,
+            {
+                "file": (BytesIO(b"a"), "blob"),
+                "uuid": uuid,
+                "filename": "test.txt",
+                "partindex": 0,
+                "partbyteoffset": 0,
+                "totalfilesize": parts,
+                "totalparts": parts,
+                "chunksize": 10,  # Does not match
+            },
+        )
 
         assert400(response)
-        assert not response.json['success']
-        assert 'filename' not in response.json
-        assert 'url' not in response.json
-        assert 'size' not in response.json
-        assert 'sha1' not in response.json
-        assert 'url' not in response.json
+        assert not response.json["success"]
+        assert "filename" not in response.json
+        assert "url" not in response.json
+        assert "size" not in response.json
+        assert "sha1" not in response.json
+        assert "url" not in response.json
 
         assert list(storages.chunks.list_files()) == []
 
     def test_upload_resource_bad_request(self, client):
         client.login()
         response = client.post(
-            url_for('test-storage.upload', name='tmp'),
-            {'bad': (BytesIO(b'aaa'), 'test.txt')})
+            url_for("test-storage.upload", name="tmp"), {"bad": (BytesIO(b"aaa"), "test.txt")}
+        )
 
         assert400(response)
-        assert not response.json['success']
-        assert 'error' in response.json
+        assert not response.json["success"]
+        assert "error" in response.json
 
 
-@pytest.mark.usefixtures('instance_path')
+@pytest.mark.usefixtures("instance_path")
 class ChunksRetentionTest:
     def create_chunks(self, uuid, nb=3, last=None):
         for i in range(nb):
             storages.chunks.write(chunk_filename(uuid, i), faker.word())
-        storages.chunks.write(chunk_filename(uuid, META), json.dumps({
-            'uuid': str(uuid),
-            'filename': faker.file_name(),
-            'totalparts': nb + 1,
-            'lastchunk': last or datetime.utcnow(),
-        }))
+        storages.chunks.write(
+            chunk_filename(uuid, META),
+            json.dumps(
+                {
+                    "uuid": str(uuid),
+                    "filename": faker.file_name(),
+                    "totalparts": nb + 1,
+                    "lastchunk": last or datetime.utcnow(),
+                }
+            ),
+        )
 
     @pytest.mark.options(UPLOAD_MAX_RETENTION=0)
     def test_chunks_cleanup_after_max_retention(self, client):
@@ -244,9 +255,7 @@ class ChunksRetentionTest:
         self.create_chunks(expired_uuid, nb=parts, last=expired)
         self.create_chunks(active_uuid, nb=parts, last=not_expired)
         purge_chunks.apply()
-        expected = set([
-            chunk_filename(active_uuid, i) for i in range(parts)
-        ])
+        expected = set([chunk_filename(active_uuid, i) for i in range(parts)])
         expected.add(chunk_filename(active_uuid, META))
         assert set(storages.chunks.list_files()) == expected
         assert not storages.chunks.exists(expired_uuid)  # Directory should be removed too

udata/tests/test_tags.py

@@ -1,17 +1,16 @@
 import logging
-import pytest
 from io import StringIO
 
+import pytest
 from flask import url_for
 
-from udata.frontend import csv
-
-from udata.tests.helpers import assert200
 from udata.core.dataset.factories import DatasetFactory
 from udata.core.reuse.factories import ReuseFactory
 from udata.core.tags.models import Tag
 from udata.core.tags.tasks import count_tags
-from udata.tags import tags_list, normalize, slug
+from udata.frontend import csv
+from udata.tags import normalize, slug, tags_list
+from udata.tests.helpers import assert200
 
 log = logging.getLogger(__name__)
 
@@ -21,42 +20,42 @@ MAX_TAG_LENGTH = 32
 
 @pytest.mark.frontend
 class TagsTests:
-    modules = ['core.tags']
+    modules = ["core.tags"]
 
     def test_csv(self, client):
-        Tag.objects.create(name='datasets-only', counts={'datasets': 15})
-        Tag.objects.create(name='reuses-only', counts={'reuses': 10})
-        Tag.objects.create(name='both', counts={'reuses': 10, 'datasets': 15})
+        Tag.objects.create(name="datasets-only", counts={"datasets": 15})
+        Tag.objects.create(name="reuses-only", counts={"reuses": 10})
+        Tag.objects.create(name="both", counts={"reuses": 10, "datasets": 15})
 
-        response = client.get(url_for('tags.csv'))
+        response = client.get(url_for("tags.csv"))
         assert200(response)
-        assert response.mimetype == 'text/csv'
-        assert response.charset == 'utf-8'
+        assert response.mimetype == "text/csv"
+        assert response.charset == "utf-8"
 
-        csvfile = StringIO(response.data.decode('utf8'))
+        csvfile = StringIO(response.data.decode("utf8"))
         reader = reader = csv.get_reader(csvfile)
         header = next(reader)
         rows = list(reader)
 
-        assert header == ['name', 'datasets', 'reuses', 'total']
+        assert header == ["name", "datasets", "reuses", "total"]
         assert len(rows) is 3
-        assert rows[0] == ['both', '15', '10', '25']
-        assert rows[1] == ['datasets-only', '15', '0', '15']
-        assert rows[2] == ['reuses-only', '0', '10', '10']
+        assert rows[0] == ["both", "15", "10", "25"]
+        assert rows[1] == ["datasets-only", "15", "0", "15"]
+        assert rows[2] == ["reuses-only", "0", "10", "10"]
 
     def test_count(self):
         for i in range(1, 4):
             # Tags should be normalized and deduplicated.
-            tags = ['Tag "{0}"'.format(j) for j in range(i)] + ['tag-0']
+            tags = ['Tag "{0}"'.format(j) for j in range(i)] + ["tag-0"]
             DatasetFactory(tags=tags, visible=True)
             ReuseFactory(tags=tags, visible=True)
 
         count_tags.run()
 
         expected = {
-            'tag-0': 3,
-            'tag-1': 2,
-            'tag-2': 1,
+            "tag-0": 3,
+            "tag-1": 2,
+            "tag-2": 1,
         }
 
         assert len(Tag.objects) == len(expected)
@@ -64,51 +63,50 @@ class TagsTests:
         for name, count in expected.items():
             tag = Tag.objects.get(name=name)
             assert tag.total == 2 * count
-            assert tag.counts['datasets'] == count
-            assert tag.counts['reuses'] == count
+            assert tag.counts["datasets"] == count
+            assert tag.counts["reuses"] == count
 
 
 class TagsUtilsTest:
-
     def test_tags_list(self):
-        assert tags_list('') == []
-        assert tags_list('a') == ['a']
-        assert sorted(tags_list('a, b, c')) == ['a', 'b', 'c']
-        assert sorted(tags_list('a b, c d, e')) == ['a-b', 'c-d', 'e']
+        assert tags_list("") == []
+        assert tags_list("a") == ["a"]
+        assert sorted(tags_list("a, b, c")) == ["a", "b", "c"]
+        assert sorted(tags_list("a b, c d, e")) == ["a-b", "c-d", "e"]
 
     def test_tags_list_strip(self):
-        assert sorted(tags_list('a, b ,  ,,, c')) == ['a', 'b', 'c']
-        assert sorted(tags_list('  a b ,  c   d, e ')) == ['a-b', 'c-d', 'e']
+        assert sorted(tags_list("a, b ,  ,,, c")) == ["a", "b", "c"]
+        assert sorted(tags_list("  a b ,  c   d, e ")) == ["a-b", "c-d", "e"]
 
     def test_tags_list_deduplication(self):
-        assert sorted(tags_list('a b, c d,  a  b , e')) == ['a-b', 'c-d', 'e']
+        assert sorted(tags_list("a b, c d,  a  b , e")) == ["a-b", "c-d", "e"]
 
     def test_slug_empty(self):
-        assert slug('') == ''
+        assert slug("") == ""
 
     def test_slug_several_words(self):
-        assert slug('la claire fontaine') == 'la-claire-fontaine'
+        assert slug("la claire fontaine") == "la-claire-fontaine"
 
     def test_slug_accents(self):
-        assert slug('école publique') == 'ecole-publique'
+        assert slug("école publique") == "ecole-publique"
 
     def test_slug_case(self):
-        assert slug('EcoLe publiquE') == 'ecole-publique'
+        assert slug("EcoLe publiquE") == "ecole-publique"
 
     def test_slug_consecutive_spaces(self):
-        assert slug('ecole  publique') == 'ecole-publique'
+        assert slug("ecole  publique") == "ecole-publique"
 
     def test_slug_special_characters(self):
-        assert slug('ecole-publique') == 'ecole-publique'
-        assert slug('ecole publique.') == 'ecole-publique'
-        assert slug('ecole publique-') == 'ecole-publique'
-        assert slug('ecole publique_') == 'ecole-publique'
+        assert slug("ecole-publique") == "ecole-publique"
+        assert slug("ecole publique.") == "ecole-publique"
+        assert slug("ecole publique-") == "ecole-publique"
+        assert slug("ecole publique_") == "ecole-publique"
 
     @pytest.mark.options(TAG_MAX_LENGTH=MAX_TAG_LENGTH)
     def test_normalize(self, app):
-        assert normalize('') == ''
-        assert normalize('a') == ''
-        assert normalize('aa') == ''
-        assert normalize('aaa') == 'aaa'
-        assert normalize('a' * (MAX_TAG_LENGTH + 1)) == 'a' * MAX_TAG_LENGTH
-        assert normalize('aAa a') == 'aaa-a'
+        assert normalize("") == ""
+        assert normalize("a") == ""
+        assert normalize("aa") == ""
+        assert normalize("aaa") == "aaa"
+        assert normalize("a" * (MAX_TAG_LENGTH + 1)) == "a" * MAX_TAG_LENGTH
+        assert normalize("aAa a") == "aaa-a"

udata/tests/test_topics.py

@@ -1,22 +1,23 @@
 import pytest
 
-from udata.search import reindex
-from udata.core.topic.factories import TopicFactory
 from udata.core.dataset.factories import DatasetFactory
+from udata.core.topic.factories import TopicFactory
+from udata.search import reindex
 
 
 @pytest.fixture
 def job_reindex(mocker):
-    return mocker.patch.object(reindex, 'delay')
+    return mocker.patch.object(reindex, "delay")
 
 
 @pytest.fixture
 def job_reindex_undelayed(mocker):
     """Mock the reindex.delay fn to access the underlying reindex fn"""
-    return mocker.patch.object(reindex, 'delay', side_effect=reindex)
+    return mocker.patch.object(reindex, "delay", side_effect=reindex)
+
 
+pytestmark = pytest.mark.usefixtures("clean_db")
 
-pytestmark = pytest.mark.usefixtures('clean_db')
 
 class TopicModelTest:
     # allows url_for with correct context when calling reindex
@@ -26,7 +27,7 @@ class TopicModelTest:
         topic = TopicFactory(datasets=[])
         dataset = DatasetFactory()
 
-        topic.name = 'new_name'
+        topic.name = "new_name"
         topic.save()
         job_reindex.assert_not_called()
 
@@ -44,4 +45,3 @@ class TopicModelTest:
         # creates a topic with datasets, thus calls reindex
         TopicFactory()
         job_reindex_undelayed.assert_called()
-