PyPI - udata - Versions diffs - 10.8.2.dev36743__py2.py3-none-any.whl → 10.8.2.dev36842__py2.py3-none-any.whl - Mend

udata 10.8.2.dev36743py2.py3-none-any.whl → 10.8.2.dev36842py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of udata might be problematic. Click here for more details.

Files changed (45) hide show

udata/harvest/tests/ckan/test_ckan_backend_errors.py ADDED Viewed

@@ -0,0 +1,140 @@
+import pytest
+from udata.harvest import actions
+from udata.harvest.tests.factories import HarvestSourceFactory
+pytestmark = [
+    pytest.mark.usefixtures("clean_db"),
+    pytest.mark.options(PLUGINS=["ckan"]),
+]
+CKAN_URL = "https://harvest.me/"
+API_URL = "{}api/3/action/package_list".format(CKAN_URL)
+# We test against success and error status code
+# because CKAN API always return 200
+# but some other cases may happen outside the API
+STATUS_CODE = (400, 500)
+@pytest.mark.parametrize("code", STATUS_CODE)
+def test_html_error(rmock, code):
+    # Happens with wrong source URL (html is returned instead of json)
+    html = "<html><body>Error</body></html>"
+    source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
+    rmock.get(API_URL, text=html, status_code=code, headers={"Content-Type": "text/html"})
+    actions.run(source.slug)
+    source.reload()
+    job = source.get_last_job()
+    assert len(job.items) == 0
+    assert len(job.errors) == 1
+    error = job.errors[0]
+    # HTML is detected and does not clutter the message
+    assert html not in error.message
+@pytest.mark.parametrize("code", STATUS_CODE)
+def test_plain_text_error(rmock, code):
+    source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
+    rmock.get(
+        API_URL, text='"Some error"', status_code=code, headers={"Content-Type": "text/plain"}
+    )
+    actions.run(source.slug)
+    source.reload()
+    job = source.get_last_job()
+    assert len(job.items) == 0
+    assert len(job.errors) == 1
+    error = job.errors[0]
+    # Raw quoted string is properly unquoted
+    http_message = "Server Error" if code == 500 else "Client Error"
+    assert (
+        error.message
+        == f"{code} {http_message}: None for url: https://harvest.me/api/3/action/package_list"
+    )
+def test_200_plain_text_error(rmock):
+    source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
+    rmock.get(API_URL, text='"Some error"', status_code=200, headers={"Content-Type": "text/plain"})
+    actions.run(source.slug)
+    source.reload()
+    job = source.get_last_job()
+    assert len(job.items) == 0
+    assert len(job.errors) == 1
+    error = job.errors[0]
+    # Raw quoted string is properly unquoted
+    assert error.message == "Some error"
+def test_standard_api_json_error(rmock):
+    json = {"success": False, "error": "an error"}
+    source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
+    rmock.get(API_URL, json=json, status_code=200, headers={"Content-Type": "application/json"})
+    actions.run(source.slug)
+    source.reload()
+    job = source.get_last_job()
+    assert len(job.items) == 0
+    assert len(job.errors) == 1
+    error = job.errors[0]
+    assert error.message == "an error"
+def test_standard_api_json_error_with_details(rmock):
+    json = {
+        "success": False,
+        "error": {
+            "message": "an error",
+        },
+    }
+    source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
+    rmock.get(API_URL, json=json, status_code=200, headers={"Content-Type": "application/json"})
+    actions.run(source.slug)
+    source.reload()
+    job = source.get_last_job()
+    assert len(job.items) == 0
+    assert len(job.errors) == 1
+    error = job.errors[0]
+    assert error.message == "an error"
+def test_standard_api_json_error_with_details_and_type(rmock):
+    json = {
+        "success": False,
+        "error": {
+            "message": "Access denied",
+            "__type": "Authorization Error",
+        },
+    }
+    source = HarvestSourceFactory(backend="ckan", url=CKAN_URL)
+    rmock.get(API_URL, json=json, status_code=200, headers={"Content-Type": "application/json"})
+    actions.run(source.slug)
+    source.reload()
+    job = source.get_last_job()
+    assert len(job.items) == 0
+    assert len(job.errors) == 1
+    error = job.errors[0]
+    assert error.message == "Authorization Error: Access denied"

udata/harvest/tests/ckan/test_ckan_backend_filters.py ADDED Viewed

@@ -0,0 +1,130 @@
+import urllib
+import pytest
+from udata.harvest import actions
+from udata.harvest.tests.factories import HarvestSourceFactory
+from udata.utils import faker
+pytestmark = [
+    pytest.mark.usefixtures("clean_db"),
+    pytest.mark.options(PLUGINS=["ckan"]),
+]
+def test_include_org_filter(ckan, rmock):
+    source = HarvestSourceFactory(
+        backend="ckan",
+        url=ckan.BASE_URL,
+        config={"filters": [{"key": "organization", "value": "organization_name"}]},
+    )
+    rmock.get(
+        ckan.PACKAGE_SEARCH_URL,
+        json={"success": True, "result": {"results": []}},
+        status_code=200,
+        headers={"Content-Type": "application/json"},
+    )
+    actions.run(source.slug)
+    source.reload()
+    assert rmock.call_count == 1
+    params = {"q": "organization:organization_name", "rows": 1000}
+    assert rmock.last_request.url == f"{ckan.PACKAGE_SEARCH_URL}?{urllib.parse.urlencode(params)}"
+def test_exclude_org_filter(ckan, rmock):
+    source = HarvestSourceFactory(
+        backend="ckan",
+        url=ckan.BASE_URL,
+        config={
+            "filters": [{"key": "organization", "value": "organization_name", "type": "exclude"}]
+        },
+    )
+    rmock.get(
+        ckan.PACKAGE_SEARCH_URL,
+        json={"success": True, "result": {"results": []}},
+        status_code=200,
+        headers={"Content-Type": "application/json"},
+    )
+    actions.run(source.slug)
+    source.reload()
+    assert rmock.call_count == 1
+    params = {"q": "-organization:organization_name", "rows": 1000}
+    assert rmock.last_request.url == f"{ckan.PACKAGE_SEARCH_URL}?{urllib.parse.urlencode(params)}"
+def test_tag_filter(ckan, rmock):
+    tag = faker.word()
+    source = HarvestSourceFactory(
+        backend="ckan", url=ckan.BASE_URL, config={"filters": [{"key": "tags", "value": tag}]}
+    )
+    rmock.get(
+        ckan.PACKAGE_SEARCH_URL,
+        json={"success": True, "result": {"results": []}},
+        status_code=200,
+        headers={"Content-Type": "application/json"},
+    )
+    actions.run(source.slug)
+    source.reload()
+    assert rmock.call_count == 1
+    params = {"q": f"tags:{tag}", "rows": 1000}
+    assert rmock.last_request.url == f"{ckan.PACKAGE_SEARCH_URL}?{urllib.parse.urlencode(params)}"
+def test_exclude_tag_filter(ckan, rmock):
+    tag = faker.word()
+    source = HarvestSourceFactory(
+        backend="ckan",
+        url=ckan.BASE_URL,
+        config={"filters": [{"key": "tags", "value": tag, "type": "exclude"}]},
+    )
+    rmock.get(
+        ckan.PACKAGE_SEARCH_URL,
+        json={"success": True, "result": {"results": []}},
+        status_code=200,
+        headers={"Content-Type": "application/json"},
+    )
+    actions.run(source.slug)
+    source.reload()
+    assert rmock.call_count == 1
+    params = {"q": f"-tags:{tag}", "rows": 1000}
+    assert rmock.last_request.url == f"{ckan.PACKAGE_SEARCH_URL}?{urllib.parse.urlencode(params)}"
+def test_can_have_multiple_filters(ckan, rmock):
+    source = HarvestSourceFactory(
+        backend="ckan",
+        url=ckan.BASE_URL,
+        config={
+            "filters": [
+                {"key": "organization", "value": "organization_name"},
+                {"key": "tags", "value": "tag-2", "type": "exclude"},
+            ]
+        },
+    )
+    rmock.get(
+        ckan.PACKAGE_SEARCH_URL,
+        json={"success": True, "result": {"results": []}},
+        status_code=200,
+        headers={"Content-Type": "application/json"},
+    )
+    actions.run(source.slug)
+    source.reload()
+    assert rmock.call_count == 1
+    params = {"q": "organization:organization_name AND -tags:tag-2", "rows": 1000}
+    assert rmock.last_request.url == f"{ckan.PACKAGE_SEARCH_URL}?{urllib.parse.urlencode(params)}"

udata/harvest/tests/ckan/test_dkan_backend.py ADDED Viewed

@@ -0,0 +1,68 @@
+import json
+import os
+from datetime import datetime
+import pytest
+from udata.app import create_app
+from udata.core.organization.factories import OrganizationFactory
+from udata.harvest import actions
+from udata.harvest.tests.factories import HarvestSourceFactory
+from udata.models import Dataset
+from udata.settings import Defaults, Testing
+from udata.tests.plugin import drop_db
+def data_path(filename):
+    """Get a test data path"""
+    return os.path.join(os.path.dirname(__file__), "data", filename)
+class DkanSettings(Testing):
+    PLUGINS = ["dkan"]
+@pytest.fixture(scope="module")
+def app(request):
+    """Create an udata app once for the module."""
+    app = create_app(Defaults, override=DkanSettings)
+    with app.app_context():
+        drop_db(app)
+    yield app
+    with app.app_context():
+        drop_db(app)
+def test_dkan_french_w_license(app, rmock):
+    """CKAN Harvester should accept the minimum dataset payload"""
+    DKAN_URL = "https://harvest.me/"
+    API_URL = "{}api/3/action/".format(DKAN_URL)
+    PACKAGE_LIST_URL = "{}package_list".format(API_URL)
+    PACKAGE_SHOW_URL = "{}package_show".format(API_URL)
+    with open(data_path("dkan-french-w-license.json")) as ifile:
+        data = json.loads(ifile.read())
+    org = OrganizationFactory()
+    source = HarvestSourceFactory(backend="dkan", url=DKAN_URL, organization=org)
+    rmock.get(
+        PACKAGE_LIST_URL,
+        json={"success": True, "result": ["fake-name"]},
+        status_code=200,
+        headers={"Content-Type": "application/json"},
+    )
+    rmock.get(
+        PACKAGE_SHOW_URL, json=data, status_code=200, headers={"Content-Type": "application/json"}
+    )
+    actions.run(source.slug)
+    source.reload()
+    assert source.get_last_job().status == "done"
+    datasets = Dataset.objects.filter(organization=org)
+    assert len(datasets) > 0
+    dataset = datasets.get(**{"harvest__remote_id": "04be6288-696d-4331-850d-a144871a7e3a"})
+    assert dataset.harvest.created_at == datetime(2019, 12, 10, 0, 0)
+    assert dataset.harvest.modified_at == datetime(2019, 9, 30, 0, 0)
+    assert len(dataset.resources) == 2
+    assert "xlsx" in [r.format for r in dataset.resources]

udata 10.8.2.dev36743__py2.py3-none-any.whl → 10.8.2.dev36842__py2.py3-none-any.whl

Potentially problematic release.

udata 10.8.2.dev36743py2.py3-none-any.whl → 10.8.2.dev36842py2.py3-none-any.whl