udata 10.3.2.dev34939__py2.py3-none-any.whl → 10.3.2.dev34982__py2.py3-none-any.whl

udata/{frontend → core}/csv.py

@@ -240,7 +240,7 @@ def stream(queryset_or_adapter, basename=None):
         cls = _adapters.get(queryset_or_adapter._document)
         adapter = cls(queryset_or_adapter)
     else:
-        raise ValueError("Unsupported object type")
+        raise ValueError(f"Unsupported object type {queryset_or_adapter}")
 
     timestamp = datetime.utcnow().strftime("%Y-%m-%d-%H-%M")
     headers = {

udata/core/dataservices/csv.py

@@ -1,4 +1,4 @@
-from udata.frontend import csv
+from udata.core import csv
 
 from .models import Dataservice
 

udata/core/dataset/api.py

@@ -38,6 +38,7 @@ from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.models import CHECKSUM_TYPES
 from udata.core.followers.api import FollowAPI
 from udata.core.organization.models import Organization
+from udata.core.reuse.models import Reuse
 from udata.core.storages.api import handle_upload, upload_parser
 from udata.core.topic.models import Topic
 from udata.linkchecker.checker import check_resource
@@ -122,6 +123,7 @@ class DatasetApiParser(ModelApiParser):
         self.parser.add_argument("topic", type=str, location="args")
         self.parser.add_argument("credit", type=str, location="args")
         self.parser.add_argument("dataservice", type=str, location="args")
+        self.parser.add_argument("reuse", type=str, location="args")
         self.parser.add_argument(
             "archived",
             type=boolean,
@@ -200,6 +202,15 @@ class DatasetApiParser(ModelApiParser):
                 pass
             else:
                 datasets = datasets.filter(id__in=[d.id for d in dataservice.datasets])
+        if args.get("reuse"):
+            if not ObjectId.is_valid(args["reuse"]):
+                api.abort(400, "Reuse arg must be an identifier")
+            try:
+                reuse = Reuse.objects.get(id=args["reuse"])
+            except Reuse.DoesNotExist:
+                pass
+            else:
+                datasets = datasets.filter(id__in=[d.id for d in reuse.datasets])
         if args.get("archived") is not None:
             if current_user.is_anonymous:
                 abort(401)

udata/core/dataset/csv.py

@@ -1,8 +1,8 @@
 # for backwards compatibility (see https://github.com/opendatateam/udata/pull/3152)
 import json
 
+from udata.core import csv
 from udata.core.discussions.csv import DiscussionCsvAdapter  # noqa: F401
-from udata.frontend import csv
 
 from .models import Dataset, Resource
 

udata/core/dataset/tasks.py

@@ -9,9 +9,8 @@ from mongoengine import ValidationError
 
 from udata import mail
 from udata import models as udata_models
-from udata.core import storages
+from udata.core import csv, storages
 from udata.core.dataservices.models import Dataservice
-from udata.frontend import csv
 from udata.harvest.models import HarvestJob
 from udata.i18n import lazy_gettext as _
 from udata.models import Activity, Discussion, Follow, Organization, Topic, Transfer, db

udata/core/discussions/csv.py

@@ -1,4 +1,4 @@
-from udata.frontend import csv
+from udata.core import csv
 
 from .models import Discussion
 

udata/core/organization/api.py

@@ -1,3 +1,4 @@
+import itertools
 from datetime import datetime
 
 from flask import make_response, redirect, request, url_for
@@ -6,6 +7,7 @@ from mongoengine.queryset.visitor import Q
 from udata.api import API, api, errors
 from udata.api.parsers import ModelApiParser
 from udata.auth import admin_permission, current_user
+from udata.core import csv
 from udata.core.badges import api as badges_api
 from udata.core.badges.fields import badge_fields
 from udata.core.contact_point.api import ContactPointApiParser
@@ -13,8 +15,10 @@ from udata.core.contact_point.api_fields import contact_point_page_fields
 from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.api import DatasetApiParser
 from udata.core.dataset.api_fields import dataset_page_fields
+from udata.core.dataset.csv import DatasetCsvAdapter, ResourcesCsvAdapter
 from udata.core.dataset.models import Dataset
 from udata.core.discussions.api import discussion_fields
+from udata.core.discussions.csv import DiscussionCsvAdapter
 from udata.core.discussions.models import Discussion
 from udata.core.followers.api import FollowAPI
 from udata.core.reuse.models import Reuse
@@ -164,6 +168,42 @@ class OrganizationAPI(API):
         return "", 204
 
 
+@ns.route("/<org:org>/datasets.csv", endpoint="organization_datasets_csv", doc=common_doc)
+@api.response(404, "Organization not found")
+@api.response(410, "Organization has been deleted")
+class DatasetsCsvAPI(API):
+    def get(self, org):
+        datasets = Dataset.objects(organization=str(org.id)).visible()
+        adapter = DatasetCsvAdapter(datasets)
+        return csv.stream(adapter, "{0}-datasets".format(org.slug))
+
+
+@ns.route("/<org:org>/discussions.csv", endpoint="organization_discussions_csv", doc=common_doc)
+@api.response(404, "Organization not found")
+@api.response(410, "Organization has been deleted")
+class DiscussionsCsvAPI(API):
+    def get(self, org):
+        datasets = Dataset.objects.filter(organization=str(org.id))
+        discussions = [Discussion.objects.filter(subject=dataset) for dataset in datasets]
+        # Turns a list of lists into a flat list.
+        adapter = DiscussionCsvAdapter(itertools.chain(*discussions))
+        return csv.stream(adapter, "{0}-discussions".format(org.slug))
+
+
+@ns.route(
+    "/<org:org>/datasets-resources.csv",
+    endpoint="organization_datasets_resources_csv",
+    doc=common_doc,
+)
+@api.response(404, "Organization not found")
+@api.response(410, "Organization has been deleted")
+class DatasetsResourcesCsvAPI(API):
+    def get(self, org):
+        datasets = Dataset.objects(organization=str(org.id)).visible()
+        adapter = ResourcesCsvAdapter(datasets)
+        return csv.stream(adapter, "{0}-datasets-resources".format(org.slug))
+
+
 @ns.route("/<org:org>/catalog", endpoint="organization_rdf", doc=common_doc)
 @api.response(404, "Organization not found")
 @api.response(410, "Organization has been deleted")

udata/core/organization/csv.py

@@ -1,5 +1,5 @@
+from udata.core import csv
 from udata.core.dataset.models import Dataset
-from udata.frontend import csv
 
 from .models import Organization
 

udata/core/reuse/csv.py

@@ -1,4 +1,4 @@
-from udata.frontend import csv
+from udata.core import csv
 
 from .models import Reuse
 

udata/core/site/api.py

@@ -1,10 +1,23 @@
 from bson import ObjectId
-from flask import json, make_response, redirect, request, url_for
+from flask import current_app, json, make_response, redirect, request, url_for
 
 from udata.api import API, api, fields
 from udata.auth import admin_permission
+from udata.core import csv
+from udata.core.dataservices.csv import DataserviceCsvAdapter
 from udata.core.dataservices.models import Dataservice
+from udata.core.dataset.api import DatasetApiParser
 from udata.core.dataset.api_fields import dataset_fields
+from udata.core.dataset.csv import ResourcesCsvAdapter
+from udata.core.dataset.search import DatasetSearch
+from udata.core.dataset.tasks import get_queryset as get_csv_queryset
+from udata.core.organization.api import OrgApiParser
+from udata.core.organization.csv import OrganizationCsvAdapter
+from udata.core.organization.models import Organization
+from udata.core.reuse.api import ReuseApiParser
+from udata.core.reuse.csv import ReuseCsvAdapter
+from udata.harvest.csv import HarvestSourceCsvAdapter
+from udata.harvest.models import HarvestSource
 from udata.models import Dataset, Reuse
 from udata.rdf import CONTEXT, RDF_EXTENSIONS, graph_response, negociate_content
 from udata.utils import multi_to_dict
@@ -114,9 +127,101 @@ class SiteRdfCatalogFormat(API):
         return make_response(*graph_response(catalog, format))
 
 
+@api.route("/site/datasets.csv", endpoint="site_datasets_csv")
+class SiteDatasetsCsv(API):
+    def get(self):
+        # redirect to EXPORT_CSV dataset if feature is enabled and no filter is set
+        exported_models = current_app.config.get("EXPORT_CSV_MODELS", [])
+        if not request.args and "dataset" in exported_models:
+            return redirect(get_export_url("dataset"))
+        search_parser = DatasetSearch.as_request_parser(store_missing=False)
+        params = search_parser.parse_args()
+        params["facets"] = False
+        datasets = DatasetApiParser.parse_filters(get_csv_queryset(Dataset), params)
+        adapter = csv.get_adapter(Dataset)
+        return csv.stream(adapter(datasets), "datasets")
+
+
+@api.route("/site/resources.csv", endpoint="site_datasets_resources_csv")
+class SiteResourcesCsv(API):
+    def get(self):
+        # redirect to EXPORT_CSV dataset if feature is enabled and no filter is set
+        exported_models = current_app.config.get("EXPORT_CSV_MODELS", [])
+        if not request.args and "resource" in exported_models:
+            return redirect(get_export_url("resource"))
+        search_parser = DatasetSearch.as_request_parser(store_missing=False)
+        params = search_parser.parse_args()
+        params["facets"] = False
+        datasets = DatasetApiParser.parse_filters(get_csv_queryset(Dataset), params)
+        return csv.stream(ResourcesCsvAdapter(datasets), "resources")
+
+
+@api.route("/site/organizations.csv", endpoint="site_organizations_csv")
+class SiteOrganizationsCsv(API):
+    def get(self):
+        params = multi_to_dict(request.args)
+        # redirect to EXPORT_CSV dataset if feature is enabled and no filter is set
+        exported_models = current_app.config.get("EXPORT_CSV_MODELS", [])
+        if not params and "organization" in exported_models:
+            return redirect(get_export_url("organization"))
+        params["facets"] = False
+        organizations = OrgApiParser.parse_filters(get_csv_queryset(Organization), params)
+        return csv.stream(OrganizationCsvAdapter(organizations), "organizations")
+
+
+@api.route("/site/reuses.csv", endpoint="site_reuses_csv")
+class SiteReusesCsv(API):
+    def get(self):
+        params = multi_to_dict(request.args)
+        # redirect to EXPORT_CSV dataset if feature is enabled and no filter is set
+        exported_models = current_app.config.get("EXPORT_CSV_MODELS", [])
+        if not params and "reuse" in exported_models:
+            return redirect(get_export_url("reuse"))
+        params["facets"] = False
+        reuses = ReuseApiParser.parse_filters(get_csv_queryset(Reuse), params)
+        return csv.stream(ReuseCsvAdapter(reuses), "reuses")
+
+
+@api.route("/site/dataservices.csv", endpoint="site_dataservices_csv")
+class SiteDataservicesCsv(API):
+    def get(self):
+        params = multi_to_dict(request.args)
+        # redirect to EXPORT_CSV dataset if feature is enabled and no filter is set
+        exported_models = current_app.config.get("EXPORT_CSV_MODELS", [])
+        if not params and "dataservice" in exported_models:
+            return redirect(get_export_url("dataservice"))
+        params["facets"] = False
+        dataservices = Dataservice.apply_sort_filters(get_csv_queryset(Dataservice))
+        return csv.stream(DataserviceCsvAdapter(dataservices), "dataservices")
+
+
+@api.route("/site/harvests.csv", endpoint="site_harvests_csv")
+class SiteHarvestsCsv(API):
+    def get(self):
+        # redirect to EXPORT_CSV dataset if feature is enabled
+        exported_models = current_app.config.get("EXPORT_CSV_MODELS", [])
+        if "harvest" in exported_models:
+            return redirect(get_export_url("harvest"))
+        adapter = HarvestSourceCsvAdapter(get_csv_queryset(HarvestSource).order_by("created_at"))
+        return csv.stream(adapter, "harvest")
+
+
 @api.route("/site/context.jsonld", endpoint="site_jsonld_context")
 class SiteJsonLdContext(API):
     def get(self):
         response = make_response(json.dumps(CONTEXT))
         response.headers["Content-Type"] = "application/ld+json"
         return response
+
+
+def get_export_url(model):
+    did = current_app.config["EXPORT_CSV_DATASET_ID"]
+    dataset = Dataset.objects.get_or_404(id=did)
+    resource = None
+    for r in dataset.resources:
+        if r.extras.get("csv-export:model", "") == model:
+            resource = r
+            break
+    if not resource:
+        api.abort(404)
+    return resource.url

udata/core/tags/csv.py

@@ -1,4 +1,4 @@
-from udata.frontend import csv
+from udata.core import csv
 
 from .models import Tag
 

udata/core/tags/views.py

@@ -1,6 +1,6 @@
 import logging
 
-from udata.frontend import csv
+from udata.core import csv
 from udata.i18n import I18nBlueprint
 
 from .csv import TagCsvAdapter

udata/harvest/csv.py

@@ -1,4 +1,4 @@
-from udata.frontend import csv
+from udata.core import csv
 
 from .models import HarvestSource