ckanapi-harvesters 0.0.0__py3-none-any.whl → 0.0.3__py3-none-any.whl

ckanapi_harvesters/ckan_api/ckan_api_params.py

@@ -0,0 +1,192 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Basic parameters for the CkanApi class
+"""
+from typing import Union, Tuple
+import copy
+from warnings import warn
+import argparse
+
+import requests
+from requests.auth import AuthBase
+
+from ckanapi_harvesters.auxiliary.proxy_config import ProxyConfig
+from ckanapi_harvesters.auxiliary.ckan_auxiliary import CkanIdFieldTreatment
+from ckanapi_harvesters.auxiliary.ckan_configuration import allow_no_ca
+from ckanapi_harvesters.auxiliary.ckan_errors import NoCAVerificationError
+from ckanapi_harvesters.auxiliary.path import path_rel_to_dir
+
+default_df_download_id_field_treatment: CkanIdFieldTreatment = CkanIdFieldTreatment.SetIndex
+
+
+class CkanApiParamsBasic:
+    def __init__(self, *, proxies:Union[str,dict,ProxyConfig]=None,
+                 ckan_headers:dict=None, http_headers:dict=None):
+        """
+        :param proxies: proxies to use for requests
+        :param ckan_headers: headers to use for requests, only to the CKAN server
+        :param http_headers: headers to use for requests, for all requests, including external requests and to the CKAN server
+        """
+        if ckan_headers is None: ckan_headers = {}
+        if http_headers is None: http_headers = {}
+        # HTTP parameters
+        self._proxy_config: ProxyConfig = ProxyConfig.from_str_or_config(proxies)
+        self.user_agent: Union[str,None] = None
+        self.http_headers: dict = http_headers
+        self.ckan_headers: dict = ckan_headers
+        self._ckan_ca: Union[bool, str, None] = True  # use to specify a path to a custom CA certificate for the CKAN server (see also environment variable REQUESTS_CA_BUNDLE)
+        self._extern_ca: Union[bool, str, None] = True  # use to specify a path to a custom CA certificate for external connexions (see also environment variable REQUESTS_CA_BUNDLE)
+        # debug parameters
+        self.store_last_response: bool = True
+        self.store_last_response_debug_info: bool = True
+        # modes
+        self.dry_run: bool = False  # if True, no requests are sent to the server (for debugging purposes)
+        # limits
+        self.default_limit_list:Union[int,None] = 100   # limit the number of entries per list response (used as default value)
+        self.default_limit_read:Union[int,None] = 5000  # limit the number of entries per response (used as default value)
+        self.max_requests_count:int = 1000  # when automating multiple requests, the number of requests is limited by this parameter
+        # timeouts
+        self.multi_requests_timeout:float = 60  # when automating multiple requests, the total time elapsed is limited by this parameter (evaluated between each request)
+        self.multi_requests_time_between_requests:float = 0  # when automating multiple requests, wait this additional time (in seconds) between each request
+        self.requests_timeout:Union[float,None] = 100  # timeout per request sent to the requests module
+        # verbosity
+        self.verbose_multi_requests:bool = False
+        self.verbose_request:bool = False
+        self.verbose_request_error:bool = True
+        self.verbose_extra:bool = True
+
+    def copy(self, *, dest=None):
+        if dest is None:
+            dest = CkanApiParamsBasic()
+        dest._proxy_config = self._proxy_config.copy()
+        dest.user_agent = copy.deepcopy(self.user_agent)
+        dest.http_headers = copy.deepcopy(self.http_headers)
+        dest.ckan_headers = copy.deepcopy(self.ckan_headers)
+        dest._ckan_ca = self._ckan_ca
+        dest._extern_ca = self._extern_ca
+        dest.dry_run = self.dry_run
+        dest.store_last_response = self.store_last_response
+        dest.store_last_response_debug_info = self.store_last_response_debug_info
+        dest.default_limit_list = self.default_limit_list
+        dest.default_limit_read = self.default_limit_read
+        dest.max_requests_count = self.max_requests_count
+        dest.multi_requests_timeout = self.multi_requests_timeout
+        dest.multi_requests_time_between_requests = self.multi_requests_time_between_requests
+        dest.requests_timeout = self.requests_timeout
+        dest.verbose_multi_requests = self.verbose_multi_requests
+        dest.verbose_request = self.verbose_request
+        dest.verbose_request_error = self.verbose_request_error
+        dest.verbose_extra = self.verbose_extra
+        return dest
+
+    @property
+    def proxies(self) -> dict:
+        return self._proxy_config.proxies
+    @proxies.setter
+    def proxies(self, proxies:dict) -> None:
+        self._proxy_config.proxies = proxies
+    @property
+    def proxy_string(self) -> str:
+        return self._proxy_config.proxy_string
+    @proxy_string.setter
+    def proxy_string(self, proxies:str) -> None:
+        self._proxy_config.proxy_string = proxies
+    @property
+    def proxy_auth(self) -> Union[AuthBase, Tuple[str,str]]:
+        return self._proxy_config.proxy_auth
+    @proxy_auth.setter
+    def proxy_auth(self, proxy_auth:Union[AuthBase, Tuple[str,str]]) -> None:
+        self._proxy_config.proxy_auth = proxy_auth
+    @property
+    def ckan_ca(self) -> Union[bool,str,None]:
+        return self._ckan_ca
+    @ckan_ca.setter
+    def ckan_ca(self, ca_cert:Union[bool,str,None]) -> None:
+        if ca_cert is not None and isinstance(ca_cert, bool) and not ca_cert:
+            if not allow_no_ca:
+                raise NoCAVerificationError()
+            else:
+                msg = "CA verification has been disabled. Only allow in a local environment!"
+                warn(msg)
+        self._ckan_ca = ca_cert
+    @property
+    def extern_ca(self) -> Union[bool,str,None]:
+        return self._extern_ca
+    @extern_ca.setter
+    def extern_ca(self, ca_cert:Union[bool,str,None]) -> None:
+        if ca_cert is not None and isinstance(ca_cert, bool) and not ca_cert:
+            if not allow_no_ca:
+                raise NoCAVerificationError()
+            else:
+                msg = "CA verification has been disabled. Only allow in a local environment!"
+                warn(msg)
+        self._extern_ca = ca_cert
+
+    @staticmethod
+    def _setup_cli_ckan_parser__params(parser:argparse.ArgumentParser=None) -> argparse.ArgumentParser:
+        """
+        Define or add CLI arguments to initialize a CKAN API connection
+        parser help message:
+
+        CKAN API connection parameters initialization
+
+        :param parser: option to provide an existing parser to add the specific fields needed to initialize a CKAN API connection
+        :return:
+        """
+        if parser is None:
+            parser = argparse.ArgumentParser(description="CKAN API connection parameters initialization")
+        ProxyConfig._setup_cli_proxy_parser(parser)  # add arguments --proxy --http-proxy --https-proxy --no-proxy --proxy-auth-file
+        parser.add_argument("--ckan-ca", type=str,
+                            help="CKAN CA certificate location (.pem file)")
+        parser.add_argument("--extern-ca", type=str,
+                            help="CA certificate location for extern connexions (.pem file)")
+        parser.add_argument("--user-agent", type=str,
+                            help="User agent for HTTP requests")
+        parser.add_argument("-l", "--default-limit", type=int,
+                            help="Default number of rows per request")
+        parser.add_argument("-v", "--verbose",
+                            help="Option to set verbosity", action="store_true", default=False)
+        # parser.add_argument("--external-code", action="store_true",
+        #                     help="Enable external code execution for builder (only enable for trusted sources)")
+        return parser
+
+    def _cli_ckan_args_apply(self, args: argparse.Namespace, *, base_dir:str=None, error_not_found:bool=True,
+                             default_proxies:dict=None, proxy_headers:dict=None) -> None:
+        """
+        Apply the arguments parsed by the argument parser defined by _setup_cli_ckan_parser
+
+        :param args:
+        :param base_dir: base directory to find the CKAN API key file, if a relative path is provided
+        (recommended: leave None to use cwd)
+        :param error_not_found: option to raise an exception if the CKAN API key file is not found
+        :param default_proxies: proxies used if proxies="default"
+        :param proxy_headers: headers used to access the proxies, generally for authentication
+        :return:
+        """
+        proxy_config = ProxyConfig.from_cli_args(args, base_dir=base_dir, error_not_found=error_not_found,
+                                                 default_proxies=default_proxies, proxy_headers=proxy_headers)
+        if proxy_config is not None:
+            self._proxy_config = proxy_config
+        if args.ckan_ca is not None:
+            self.ckan_ca = path_rel_to_dir(args.ckan_ca, base_dir=base_dir)
+        if args.extern_ca is not None:
+            self.extern_ca = path_rel_to_dir(args.extern_ca, base_dir=base_dir)
+        if args.user_agent is not None:
+            self.user_agent = args.user_agent
+        # if args.default_limit is not None:
+        #     self.set_limits(args.default_limit)
+        # if args.verbose is not None:
+        #     self.set_verbosity(args.verbose)
+        # if args.external_code:
+        #     unlock_external_code_execution()
+        print(args)
+
+class CkanApiDebug:
+    def __init__(self):
+        self.ckan_request_counter: int = 0
+        self.extern_request_counter: int = 0
+        self.last_response: Union[requests.Response, None] = None  # field containing the last response, for debug purposes
+        self.last_response_request_count: int = 0
+        self.multi_requests_last_successful_offset: int = 0  # last used offset when multiple queries are performed. This can be used in order to restart an update/download sequence in case of an error.
+

ckanapi_harvesters/ckan_api/deprecated/__init__.py

@@ -0,0 +1,9 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Deprecated code
+"""
+
+from . import ckan_api_deprecated
+from . import ckan_api_deprecated_vocabularies
+

ckanapi_harvesters/ckan_api/deprecated/ckan_api_deprecated.py

@@ -0,0 +1,267 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+
+"""
+from typing import List
+import copy
+from warnings import warn
+
+from ckanapi_harvesters.auxiliary.ckan_model import CkanPackageInfo, CkanResourceInfo, CkanViewInfo
+from ckanapi_harvesters.auxiliary.ckan_auxiliary import RequestType, assert_or_raise
+from ckanapi_harvesters.auxiliary.ckan_action import CkanNotFoundError
+from ckanapi_harvesters.auxiliary.ckan_errors import ReadOnlyError
+from ckanapi_harvesters.ckan_api.ckan_api_1_map import use_ckan_owner_org_as_default
+
+from ckanapi_harvesters.ckan_api.ckan_api_5_manage import CkanApiManage
+
+
+
+class CkanApiDeprecated(CkanApiManage):
+    """
+    CKAN Database API interface to CKAN server with helper functions using pandas DataFrames.
+    This class implements API calls which are not recommended to use.
+    """
+
+    def copy(self, new_identifier: str = None, *, dest=None):
+        if dest is None:
+            dest = CkanApiDeprecated()
+        super().copy(new_identifier=new_identifier, dest=dest)
+        return dest
+
+    ## Not recommended mapping functions ------------------v
+    def _api_package_list(self, *, params:dict=None, owner_org:str=None, limit:int=None, offset:int=None) -> List[str]:
+        """
+        __Not recommended__
+        API call to package_list.
+        :param params: typically, the request can be limited to an organization with the owner_org parameter
+        :return:
+        """
+        msg = DeprecationWarning("Prefer using package_search rather than package_list because this API does not list private packages")
+        warn(msg)
+        if params is None: params = {}
+        if limit is None: limit = self.params.default_limit_list
+        if limit is not None:
+            params["limit"] = limit
+        if offset is not None:
+            params["offset"] = offset
+        if owner_org is None and use_ckan_owner_org_as_default:
+            owner_org = self.owner_org
+        if owner_org is not None:
+            params["owner_org"] = owner_org
+        response = self._api_action_request("package_list", method=RequestType.Get, params=params)
+        if response.success:
+            return response.result
+        else:
+            raise response.default_error(self)
+
+    def _api_package_list_all(self, *, params:dict=None, owner_org:str=None, limit:int=None, offset:int=None) -> List[str]:
+        """
+        __Not recommended__
+        API call to package_list until an empty list is received.
+        :see: api_package_list()
+        :param params:
+        :return:
+        """
+        msg = DeprecationWarning("Prefer using package_search rather than package_list because this API does not list private packages")
+        warn(msg)
+        if params is None: params = {}
+        responses = self._request_all_results_list(self._api_package_list, params=params, owner_org=owner_org, limit=limit, offset=offset)
+        return sum(responses, [])
+
+    package_list_all = _api_package_list_all  # function alias
+
+
+    def _api_resource_search(self, query:str=None, *, order_by:str=None, limit:int=None, offset:int=None,
+                             resource_name:str=None,
+                             datastore_info:bool=None, resource_view_list:bool=None,
+                             params:dict=None) -> List[CkanResourceInfo]:
+        """
+        __Not recommended__
+        API call to resource_search. It is more recommended to use the package_show API because it is not possible to
+        filter the resources by package name here. Moreover, it does not return information on private resources.
+        :see: map_resources()
+        :param query: (string or list of strings of the form {field}:{term1}) – The search criteria. See above for description.
+        :param order_by: A field on the Resource model that orders the results.
+        :param limit:
+        :param offset:
+        :param resource_name: a shortcut to add the filter "name:{resource_name}"
+        :param datastore_info: an option to query the datastore info for all the resources found.
+        If not provided, the last value for this option used with map_resources will be used.
+        :param resource_view_list: an option to query the resource views list for all the resources found.
+        If not provided, the last value for this option used with map_resources will be used.
+        :param params: additional parameters to pass to resource_search
+        :return:
+        """
+        msg = DeprecationWarning("Prefer using package_search rather than resource_search because resource_search cannot filter per package")
+        warn(msg)
+        if datastore_info is None:
+            datastore_info = self.map._mapping_query_datastore_info
+        if resource_view_list is None:
+            resource_view_list = self.map._mapping_query_resource_view_list
+        if params is None: params = {}
+        if limit is None: limit = self.params.default_limit_list
+        if limit is not None:
+            params["limit"] = limit
+        if offset is not None:
+            params["offset"] = offset
+        if query is None:
+            query = []
+        elif isinstance(query, str):
+            query = [query]
+        if resource_name is not None:
+            query.append(f"name:{resource_name}")
+        if query is not None:
+            params["query"] = query
+        if order_by is not None:
+            params["order_by"] = order_by
+        response = self._api_action_request("resource_search", method=RequestType.Get, params=params)
+        if response.success:
+            resource_info_list = [CkanResourceInfo(e) for e in response.result["results"]]
+            for resource_info in resource_info_list:
+                self._enrich_resource_info(resource_info, datastore_info=datastore_info,
+                                           resource_view_list=resource_view_list)
+            self.map._update_resource_info(resource_info_list)
+            return copy.deepcopy(resource_info_list)
+        else:
+            raise response.default_error(self)
+
+    def _api_resource_search_all(self, query: str = None, *, order_by: str = None, limit: int = None, offset: int = None,
+                                 resource_name: str = None,
+                                 datastore_info: bool = None, resource_view_list: bool = None,
+                                 params: dict = None) -> List[CkanResourceInfo]:
+        """
+        __Not recommended__
+        API call to resource_search until an empty list is received. It is more recommended to use the package_show API because it is not possible to
+        filter the resources by package name here. Moreover, it does not return information on private resources.
+        :see: map_resources()
+        :see: _api_resource_search()
+        :param query: (string or list of strings of the form {field}:{term1}) – The search criteria. See above for description.
+        :param order_by: A field on the Resource model that orders the results.
+        :param limit: maximum number of results to return.
+        :param offset: the offset in the complete result for where the set of returned datasets should begin.
+        :param resource_name: a shortcut to add the filter "name:{resource_name}"
+        :param datastore_info: an option to query the datastore info for all the resources found.
+        If not provided, the last value for this option used with map_resources will be used.
+        :param resource_view_list: an option to query the resource views list for all the resources found.
+        If not provided, the last value for this option used with map_resources will be used.
+        :param params: additional parameters to pass to resource_search
+        :return:
+        """
+        msg = DeprecationWarning("Prefer using package_search rather than resource_search because resource_search cannot filter per package")
+        warn(msg)
+        if params is None: params = {}
+        responses = self._request_all_results_list(self._api_resource_search, params=params, limit=limit, offset=offset,
+                                                   query=query, order_by=order_by,
+                                                   resource_name=resource_name,
+                                                   datastore_info=datastore_info, resource_view_list=resource_view_list)
+        return sum(responses, [])
+
+    resource_search_all = _api_resource_search_all  # function alias
+
+
+    def _api_group_package_show(self, group_name: str, *, params:dict=None, owner_org:str=None,
+                                include_private:bool=True, include_drafts:bool=False, sort:str=None,
+                                limit:int=None, offset:int=None) -> List[CkanPackageInfo]:
+        """
+        __Not recommended__
+        API call to group_package_show. Return the datasets (packages) of a group.
+        :param group_name: group name or id
+        :param owner_org: ability to filter packages by owner_org
+        :param include_private: if True, private datasets will be included in the results. Only private datasets from the user’s organizations will be returned and sysadmins will be returned all private datasets. Optional, the default is False in the API
+        :param include_drafts:  if True, draft datasets will be included in the results. A user will only be returned their own draft datasets, and a sysadmin will be returned all draft datasets. Optional, the default is False.
+        :param sort: sorting of the search results. Optional. Default: 'score desc, metadata_modified desc'. As per the solr documentation, this is a comma-separated string of field names and sort-orderings.
+        :param limit: maximum number of results to return. Translatees to the API rows argument.
+        :param offset: the offset in the complete result for where the set of returned datasets should begin. Translatees to the API start argument.
+        :param params: other parameters to pass to package_search
+        :return:
+        """
+        msg = DeprecationWarning("Prefer using package_search rather than group_package_show knowing the name of the package because this API does not list private packages")
+        warn(msg)
+        if params is None: params = {}
+        params["id"] = group_name
+        if limit is None: limit = self.params.default_limit_list
+        if limit is not None:
+            params["limit"] = limit
+        if offset is not None:
+            params["offset"] = offset
+        if owner_org is None and use_ckan_owner_org_as_default:
+            owner_org = self.owner_org
+        if owner_org is not None:
+            owner_org_info = self.get_organization_info_or_request(owner_org)
+            owner_org = owner_org_info.id
+            params["owner_org"] = owner_org
+        if sort is not None:
+            params["sort"] = sort
+        if include_private is not None:
+            params["include_private"] = include_private
+        if include_drafts is not None:
+            params["include_drafts"] = include_drafts
+        response = self._api_action_request("group_package_show", method=RequestType.Get, params=params)
+        if response.success:
+            package_info_list = [CkanPackageInfo(e) for e in response.result]
+            self.map._update_package_info(package_info_list)
+            return package_info_list
+        elif response.status_code == 404 and response.success_json_loads and response.error_message["__type"] == "Not Found Error":
+            raise CkanNotFoundError(self, "Group", response)
+        else:
+            raise response.default_error(self)
+
+    def _api_group_package_show_all(self, group_name: str, *, params:dict=None, owner_org:str=None,
+                                    include_private:bool=True, include_drafts:bool=False, sort:str=None,
+                                    limit:int=None, offset:int=None) -> List[CkanPackageInfo]:
+        """
+        __Not recommended__
+        API call to group_package_show until an empty list is received.
+        :see: _api_group_package_show()
+        :param group_name: group name or id
+        :param owner_org: ability to filter packages by owner_org
+        :param include_private: if True, private datasets will be included in the results. Only private datasets from the user’s organizations will be returned and sysadmins will be returned all private datasets. Optional, the default is False in the API
+        :param include_drafts:  if True, draft datasets will be included in the results. A user will only be returned their own draft datasets, and a sysadmin will be returned all draft datasets. Optional, the default is False.
+        :param sort: sorting of the search results. Optional. Default: 'score desc, metadata_modified desc'. As per the solr documentation, this is a comma-separated string of field names and sort-orderings.
+        :param limit: maximum number of results to return. Translatees to the API rows argument.
+        :param offset: the offset in the complete result for where the set of returned datasets should begin. Translatees to the API start argument.
+        :param params: other parameters to pass to API
+        :return:
+        """
+        msg = DeprecationWarning("Prefer using package_search rather than group_package_show knowing the name of the package because this API does not list private packages")
+        warn(msg)
+        if params is None: params = {}
+        responses = self._request_all_results_list(self._api_group_package_show, params=params, limit=limit, offset=offset,
+                                                   group_name=group_name, owner_org=owner_org,
+                                                   include_private=include_private, include_drafts=include_drafts)
+        return sum(responses, [])
+
+    group_package_show_all = _api_group_package_show_all  # function alias
+
+    ## resource view
+    def _api_resource_create_default_resource_views(self, resource_id:str, *, create_datastore_views:bool=None,
+                                                    params:dict=None) -> List[CkanViewInfo]:
+        """
+        API call to resource_create_default_resource_views
+        :param resource_id: resource id
+        :param create_datastore_views: whether to create views that rely on data being on the DataStore (optional, API defaults to False)
+        :param params:
+        :return:
+        """
+        msg = DeprecationWarning("Prefer using resource_view_create rather than resource_create_default_resource_views")
+        warn(msg)
+        assert_or_raise(not self.params.read_only, ReadOnlyError())
+        if params is None: params = {}
+        resource_info = self.resource_show(resource_id)
+        resource_dict = resource_info.details
+        if create_datastore_views is None:
+            create_datastore_views = self.resource_is_datastore(resource_id)
+        params["resource"] = resource_dict
+        params["create_datastore_views"] = create_datastore_views
+        response = self._api_action_request(f"resource_create_default_resource_views", method=RequestType.Post, json=params)
+        if response.success:
+            view_info_list = [CkanViewInfo(view_dict) for view_dict in response.result]
+            self.map._update_view_info(view_info_list)
+            return copy.deepcopy(view_info_list)
+        else:
+            raise response.default_error(self)
+
+    resource_create_default_resource_views = _api_resource_create_default_resource_views  # function alias
+
+

ckanapi_harvesters/ckan_api/deprecated/ckan_api_deprecated_vocabularies.py

@@ -0,0 +1,189 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+
+"""
+from typing import List, Dict, Union
+import copy
+from warnings import warn
+
+from ckanapi_harvesters.auxiliary.proxy_config import ProxyConfig
+from ckanapi_harvesters.auxiliary.ckan_auxiliary import RequestType, assert_or_raise
+from ckanapi_harvesters.auxiliary.ckan_map import CkanMap
+from ckanapi_harvesters.auxiliary.ckan_errors import MandatoryAttributeError
+from ckanapi_harvesters.auxiliary.ckan_vocabulary_deprecated import CkanTagVocabularyInfo, CkanVocabularyMap
+from ckanapi_harvesters.policies.data_format_policy import CkanPackageDataFormatPolicy
+from ckanapi_harvesters.policies.data_format_policy_tag_groups import TagListPolicy
+
+from ckanapi_harvesters.ckan_api.deprecated.ckan_api_deprecated import CkanApiDeprecated
+
+
+class CkanApiVocabulariesDeprecated(CkanApiDeprecated):
+    def __init__(self, url:str=None, *, proxies:Union[str,dict,ProxyConfig]=None,
+                 ckan_headers:dict=None, http_headers:dict=None,
+                 apikey:str=None, apikey_file:str=None,
+                 owner_org:str=None,
+                 policy:CkanPackageDataFormatPolicy=None, policy_file:str=None,
+                 identifier=None):
+        """
+        CKAN Database API interface to CKAN server with helper functions using pandas DataFrames.
+
+        :param url: url of the CKAN server
+        :param proxies: proxies to use for requests
+        :param ckan_headers: headers to use for requests, only to the CKAN server
+        :param http_headers: headers to use for requests, for all requests, including external requests and to the CKAN server
+        :param apikey: way to provide the API key directly (optional)
+        :param apikey_file: path to a file containing a valid API key in the first line of text (optional)
+        :param policy: data format policy to use with policy_check function
+        :param policy_file: path to a JSON file containing the data format policy to use with policy_check function
+        :param owner_org: name of the organization to limit package_search (optional)
+        """
+        msg = DeprecationWarning("Vocabularies are used to define custom fields which accept specific values and require to implement an IDatasetForm extension. This is not covered in this package.")
+        warn(msg)
+        super().__init__(url=url, proxies=proxies, apikey=apikey, apikey_file=apikey_file,
+                         ckan_headers=ckan_headers, http_headers=http_headers,
+                         owner_org=owner_org, policy=policy, policy_file=policy_file, identifier=identifier)
+        self.map_vocabulary: CkanVocabularyMap = CkanVocabularyMap()
+
+    def copy(self, new_identifier: str = None, *, dest=None):
+        if dest is None:
+            dest = CkanApiVocabulariesDeprecated()
+        super().copy(new_identifier=new_identifier, dest=dest)
+        dest.map_vocabulary = self.map_vocabulary.copy()
+        return dest
+
+    def set_default_map_mode(self, datastore_info:bool=None, resource_view_list:bool=None,
+                             organization_info:bool=None, license_list:bool=None,
+                             load_policy:bool=None, vocabulary_list:bool=None) -> None:
+        super().set_default_map_mode(datastore_info=datastore_info, resource_view_list=resource_view_list,
+                                     organization_info=organization_info, license_list=license_list,
+                                     load_policy=load_policy)
+        if vocabulary_list is None:
+            vocabulary_list = self.map_vocabulary._mapping_query_vocabulary_list
+        self.map_vocabulary._mapping_query_vocabulary_list = vocabulary_list
+
+    def map_resources(self, package_list:Union[str, List[str]]=None, *, params:dict=None,
+                      datastore_info:bool=None, resource_view_list:bool=None, organization_info:bool=None, license_list:bool=None,
+                      only_missing:bool=True, error_not_found:bool=True,
+                      owner_org:str=None, load_policy:bool=None, vocabulary_list:bool=None) -> CkanMap:
+        # overload including a call to load the default data format policy
+        self.set_default_map_mode(vocabulary_list=vocabulary_list)
+        map = super().map_resources(package_list=package_list, params=params, datastore_info=datastore_info,
+                              resource_view_list=resource_view_list, organization_info=organization_info,
+                              license_list=license_list, only_missing=only_missing, error_not_found=error_not_found,
+                              owner_org=owner_org, load_policy=load_policy)
+        vocabulary_list = self.map_vocabulary._mapping_query_vocabulary_list
+        if vocabulary_list:
+            self.vocabulary_list(cancel_if_present=True)
+        return map
+
+
+    ## Vocabulary management (requires sysadmin rights) --------------
+    def _api_vocabulary_list(self, *, params:dict=None) -> List[CkanTagVocabularyInfo]:
+        """
+        API call to vocabulary_list.
+
+        :return: a list of vocabulary info objects
+        """
+        msg = DeprecationWarning("Vocabulary functions did not work when tested")
+        warn(msg)
+        response = self._api_action_request(f"vocabulary_list", method=RequestType.Post, json=params)
+        if response.success:
+            vocabulary_list = [CkanTagVocabularyInfo(vocabulary_dict) for vocabulary_dict in response.result]
+            self.map_vocabulary._update_vocabulary_info(vocabulary_list, vocabularies_listed=True)  # update map
+            return copy.deepcopy(vocabulary_list)
+        else:
+            raise response.default_error(self)
+
+    def vocabulary_list(self, cancel_if_present:bool=True) -> List[CkanTagVocabularyInfo]:
+        if self.map_vocabulary.vocabularies_listed and cancel_if_present:
+            return list(self.map_vocabulary.vocabularies.values())
+        else:
+            return self._api_vocabulary_list()
+
+    def _api_vocabulary_create(self, vocabulary_name: str, tags_list_dict: List[Dict[str, str]], *, params:dict=None) -> CkanTagVocabularyInfo:
+        """
+        API call to vocabulary_create.
+
+        :return: a
+        """
+        msg = DeprecationWarning("Vocabulary functions did not work when tested")
+        warn(msg)
+        if params is None: params = {}
+        params["name"] = vocabulary_name
+        params["tags"] = tags_list_dict
+        response = self._api_action_request(f"vocabulary_create", method=RequestType.Post, json=params)
+        if response.success:
+            vocabulary_info = CkanTagVocabularyInfo(response.result)
+            self.map_vocabulary._update_vocabulary_info(vocabulary_info)
+            return copy.deepcopy(vocabulary_info)
+        else:
+            raise response.default_error(self)
+
+    def _api_vocabulary_update(self, vocabulary_id: str, tags_list_dict: List[Dict[str, str]], *, params:dict=None) -> CkanTagVocabularyInfo:
+        """
+        API call to vocabulary_update.
+
+        :return: a
+        """
+        msg = DeprecationWarning("Vocabulary functions did not work when tested")
+        warn(msg)
+        if params is None: params = {}
+        params["id"] = vocabulary_id
+        params["tags"] = tags_list_dict
+        response = self._api_action_request(f"vocabulary_update", method=RequestType.Post, json=params)
+        if response.success:
+            vocabulary_info = CkanTagVocabularyInfo(response.result)
+            self.map_vocabulary._update_vocabulary_info(vocabulary_info)
+            return copy.deepcopy(vocabulary_info)
+        else:
+            raise response.default_error(self)
+
+    def vocabulary_update(self, vocabulary_name: str, tags_list_dict: List[Dict[str, str]]):
+        vocabulary_id = self.map_vocabulary.get_vocabulary_id(vocabulary_name, error_not_mapped=False)
+        if vocabulary_id is None:
+            self._api_vocabulary_create(vocabulary_name=vocabulary_name, tags_list_dict=tags_list_dict)
+        else:
+            self._api_vocabulary_update(vocabulary_id, tags_list_dict=tags_list_dict)
+
+    def _api_vocabulary_delete(self, vocabulary_id: str, *, params:dict=None) -> bool:
+        """
+        API call to vocabulary_delete.
+
+        :return: True if success
+        """
+        msg = DeprecationWarning("Vocabulary functions did not work when tested")
+        warn(msg)
+        if params is None: params = {}
+        params["id"] = vocabulary_id
+        response = self._api_action_request(f"vocabulary_delete", method=RequestType.Post, json=params)
+        if response.success:
+            return True
+        else:
+            raise response.default_error(self)
+
+    def vocabulary_delete(self, vocabulary_id: str) -> bool:
+        return self._api_vocabulary_delete(vocabulary_id)
+
+    def vocabularies_clear(self):
+        self.vocabulary_list(cancel_if_present=True)
+        vocabulary_ids = list(self.map_vocabulary.vocabularies.keys())
+        for vocabulary_id in vocabulary_ids:
+            self._api_vocabulary_delete(vocabulary_id)
+
+    def initiate_vocabularies_from_policy(self, policy:CkanPackageDataFormatPolicy, *, remove_others:bool=False):
+        vocabulary_policy = policy.package_tags
+        vocabulary_list: TagListPolicy
+        vocabulary_names = {vocabulary_list.group_name for vocabulary_list in vocabulary_policy.value_group_specs}
+        if remove_others:
+            self.vocabulary_list(cancel_if_present=True)
+            current_vocabularies = set(self.map_vocabulary.vocabulary_id_index.keys())
+            extra_vocabularies = current_vocabularies - vocabulary_names
+            for vocabulary_name in extra_vocabularies:
+                vocabulary_id = self.map_vocabulary.vocabulary_id_index[vocabulary_name]
+                self._api_vocabulary_delete(vocabulary_id)
+        for vocabulary_list in vocabulary_policy.value_group_specs:
+            vocabulary_name = vocabulary_list.group_name
+            tags_list_dict = vocabulary_list.get_tags_list_dict()
+            assert_or_raise(vocabulary_name is not None, MandatoryAttributeError("Tag vocabulary", "vocabulary_name"))
+            self.vocabulary_update(vocabulary_name=vocabulary_name, tags_list_dict=tags_list_dict)