ckanapi-harvesters 0.0.0__py3-none-any.whl → 0.0.3__py3-none-any.whl

ckanapi_harvesters/policies/data_format_policy_errors.py

@@ -0,0 +1,79 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Data format policy representation and enforcing
+"""
+from typing import List, Tuple
+from warnings import warn
+from collections import OrderedDict
+
+from ckanapi_harvesters.auxiliary.error_level_message import ErrorLevelMessage, ErrorLevel
+
+
+class DataPolicyError(ErrorLevelMessage):
+    def __init__(self, context:str, error_level:ErrorLevel, policy_message: str):
+        message = f"In {context} / Data format policy {error_level.name}: {policy_message}"
+        super().__init__(error_level, message)
+        self.context: str = context
+        self.specific_message: str = policy_message
+
+    def to_dict(self) -> dict:
+        return OrderedDict([
+            ("level", str(self.error_level)),
+            ("context", self.context),
+            ("message", self.specific_message),
+        ])
+
+
+class UnsupportedPolicyVersionError(Exception):
+    def __init__(self, file_version):
+        super().__init__(f"Version error: policy file version {file_version} is not supported")
+
+class UrlPolicyLockedError(Exception):
+    def __init__(self, url):
+        super().__init__(f"Url is not allowed a policy definition - feature locked (url: {url})")
+
+def _policy_msg(msg:DataPolicyError, *, error_level:ErrorLevel, buffer:List[DataPolicyError], verbose:bool) -> None:
+    if buffer is not None:
+        buffer.append(msg)
+    elif error_level == ErrorLevel.Information and verbose:
+        print(str(msg))
+    elif error_level == ErrorLevel.Warning:
+        msg = str(msg)
+        warn(msg)
+    elif error_level == ErrorLevel.Error:
+        raise msg
+
+
+class ErrorCount:
+    def __init__(self, messages_list:List[DataPolicyError]):
+        self.messages_list:List[DataPolicyError] = messages_list
+        self.information:int = 0
+        self.warning:int = 0
+        self.error:int = 0
+        self.total:int = len(messages_list)
+        for message in messages_list:
+            if message.error_level == ErrorLevel.Information:
+                self.information += 1
+            elif message.error_level == ErrorLevel.Warning:
+                self.warning += 1
+            elif message.error_level == ErrorLevel.Error:
+                self.error += 1
+
+    def error_count_message(self) -> str:
+        if self.total == 0:
+            return "All tests passed"
+        else:
+            return f"{self.error} errors, {self.warning} warnings, {self.information} messages"
+
+    def __str__(self) -> str:
+        return "ErrorCount: " + self.error_count_message()
+
+    def __add__(self, other):
+        return ErrorCount(self.messages_list + other.messages_list)
+
+    def to_tuple(self) -> Tuple[int, int, int]:
+        return (self.error, self.warning, self.information)
+
+    def to_dict(self) -> dict[str,int]:
+        return OrderedDict([("errors", self.error), ("warnings", self.warning), ("information", self.information)])

ckanapi_harvesters/policies/data_format_policy_lists.py

@@ -0,0 +1,234 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Data format policy representation and enforcing for lists of values such as tags
+"""
+from typing import List, Any, Iterable, Union, Dict, Set
+from abc import ABC, abstractmethod
+from warnings import warn
+import re
+
+from ckanapi_harvesters.auxiliary.ckan_auxiliary import _string_from_element, assert_or_raise
+from ckanapi_harvesters.auxiliary.ckan_defs import ckan_tags_sep
+from ckanapi_harvesters.policies.data_format_policy_errors import DataPolicyError, ErrorLevel, _policy_msg
+from ckanapi_harvesters.policies.data_format_policy_defs import ListChoiceMode
+from ckanapi_harvesters.policies.data_format_policy_defs import StringValueSpecification
+from ckanapi_harvesters.policies.data_format_policy_abc import DataPolicyElementABC
+
+
+extra_group_name = "extra"
+
+
+class ValueListPolicy(DataPolicyElementABC):
+    _group_type_str = "group"
+
+    def __init__(self, list_specs:List[StringValueSpecification]=None, group_name:str=None,
+                 value_select:ListChoiceMode=ListChoiceMode.Any,
+                 mandatory:bool=False, error_level:ErrorLevel=ErrorLevel.Information):
+        super().__init__(mandatory=mandatory, error_level=error_level)
+        if list_specs is None:
+            list_specs = []
+        self.list_specs:List[StringValueSpecification] = list_specs
+        self.value_select: ListChoiceMode = value_select
+        self.group_name: str = group_name
+
+    def to_dict(self) -> dict:
+        d = {}
+        if self.group_name:
+            d["group_name"] = self.group_name
+        d.update(super().to_dict())
+        d.update({"values": [spec.to_dict() for spec in self.list_specs],
+                  "value_select": self.value_select.name})
+        return d
+
+    def list_specs_str(self) -> List[str]:
+        return [value_spec.value for value_spec in self.list_specs]
+
+    @staticmethod
+    def from_dict(d:dict) -> "ValueListPolicy":
+        obj = ValueListPolicy()
+        obj._load_from_dict(d)
+        return obj
+
+    def _load_from_dict(self, d:dict):
+        super()._load_from_dict(d)
+        self.group_name = d["group_name"] if len(d["group_name"]) > 0 else None
+        self.list_specs = [StringValueSpecification.from_dict(value)  for value in d["values"]]
+        self.value_select = ListChoiceMode.from_str(d["value_select"]) if "value_select" in d.keys() else ListChoiceMode.Any
+
+    def enforce(self, values: Union[str, List[str]], *, context:str=None,
+                verbose: bool = True, buffer:List[DataPolicyError]=None) -> bool:
+        if self.group_name is not None:
+            context = context + " / " + self._group_type_str + " " + self.group_name
+        success = True
+        spec = [tag_spec.value for tag_spec in self.list_specs]
+        if values is None:
+            values = []
+        elif isinstance(values, str):
+            values = values.split(ckan_tags_sep)
+        values = list(set(values).intersection(set(spec)))
+        msg = None
+        value_context = context + " / value '" + ','.join(values).join(values) + "'"
+        if (self.value_select == ListChoiceMode.MaxOne and len(values) > 1):
+            success = False
+            msg = DataPolicyError(value_context, self.error_level, f"Too many values for value list group '{self.group_name}'. Max one value is admitted within {spec}.")
+        if (self.value_select == ListChoiceMode.NoExtra and len(values) > 0):
+            success = False
+            msg = DataPolicyError(value_context, self.error_level, f"Too many values for value list group '{self.group_name}'. No values can be selected for this group ({spec}).")
+        if (self.value_select == ListChoiceMode.MandatoryOne and not len(values) == 1):
+            success = False
+            msg = DataPolicyError(value_context, self.error_level, f"Exactly one value must be present for value list group '{self.group_name}' ({spec}).")
+        if (self.value_select == ListChoiceMode.MandatoryMulti and not len(values) < 1):
+            success = False
+            msg = DataPolicyError(value_context, self.error_level, f"At least one value must be present for value list group '{self.group_name}' ({spec}).")
+        if not success:
+            _policy_msg(msg, error_level=self.error_level, buffer=buffer, verbose=verbose)
+        if len(spec) > 0:
+            for tag in values:
+                success &= self._enforce_unit_string(tag, spec, context=context, verbose=verbose, buffer=buffer)
+        return success
+
+
+class ExtraValueListPolicy(ValueListPolicy):
+    def __init__(self, list_specs:List[StringValueSpecification]=None,
+                 value_select:ListChoiceMode=ListChoiceMode.Any,
+                 mandatory:bool=False, error_level:ErrorLevel=ErrorLevel.Information):
+        super().__init__(list_specs=list_specs, group_name=extra_group_name,
+                         value_select=value_select, mandatory=mandatory, error_level=error_level)
+
+    @staticmethod
+    def from_ValueListPolicy(value: ValueListPolicy) -> "ExtraValueListPolicy":
+        obj = ExtraValueListPolicy()
+        obj.__dict__.update(value.__dict__)
+        return obj
+
+    @staticmethod
+    def from_dict(d:dict) -> "ExtraValueListPolicy":
+        obj = ExtraValueListPolicy()
+        obj._load_from_dict(d)
+        return obj
+
+    def enforce(self, values: Union[str, List[str]], *, context:str=None,
+                verbose: bool = True, buffer:List[DataPolicyError]=None, extra_spec_rm:Set[str]=None) -> bool:
+        if self.group_name is not None:
+            context = context + " / group " + self.group_name
+        success = True
+        spec = [tag_spec.value for tag_spec in self.list_specs]
+        if values is None:
+            values = []
+        elif isinstance(values, str):
+            values = values.split(ckan_tags_sep)
+        values = list(set(values) - extra_spec_rm)
+        msg = None
+        value_context = context + " / value '" + ','.join(values).join(values) + "'"
+        if (self.value_select == ListChoiceMode.MaxOne and len(values) > 1):
+            success = False
+            msg = DataPolicyError(value_context, self.error_level, f"Too many values for value list group '{self.group_name}'. Max one value is admitted within {spec}.")
+        if (self.value_select == ListChoiceMode.NoExtra and len(values) > 0):
+            success = False
+            msg = DataPolicyError(value_context, self.error_level, f"Too many values for value list group '{self.group_name}'. No values can be selected for this group ({spec}). Admitted values from other groups: {extra_spec_rm}")
+        if (self.value_select == ListChoiceMode.MandatoryOne and not len(values) == 1):
+            success = False
+            msg = DataPolicyError(value_context, self.error_level, f"Exactly one value must be present for value list group '{self.group_name}' ({spec}).")
+        if (self.value_select == ListChoiceMode.MandatoryMulti and not len(values) < 1):
+            success = False
+            msg = DataPolicyError(value_context, self.error_level, f"At least one value must be present for value list group '{self.group_name}' ({spec}).")
+        if not success:
+            _policy_msg(msg, error_level=self.error_level, buffer=buffer, verbose=verbose)
+        if len(spec) > 0:
+            for tag in values:
+                success &= self._enforce_unit_string(tag, spec, context=context, verbose=verbose, buffer=buffer)
+        return success
+
+
+class GroupedValueListPolicy(DataPolicyElementABC):
+    def __init__(self, value_group_specs:List[ValueListPolicy]=None,
+                 extra_values:ExtraValueListPolicy=None,
+                 mandatory:bool=False, error_level:ErrorLevel=ErrorLevel.Information):
+        super().__init__(mandatory=mandatory, error_level=error_level)
+        if value_group_specs is None:
+            value_group_specs = []
+        self.value_group_specs:List[ValueListPolicy] = value_group_specs
+        self.extra_values_spec:ExtraValueListPolicy = extra_values
+        self._extract_extra_values()
+
+    def _extract_extra_values(self):
+        i_rm = []
+        extra_values = self.extra_values_spec
+        for i, value_group_spec in enumerate(self.value_group_specs):
+            if value_group_spec.group_name.lower() == extra_group_name.lower():
+                assert(extra_values is None)
+                extra_values = ExtraValueListPolicy.from_ValueListPolicy(value_group_spec)
+                i_rm.append(i)
+        for i in reversed(i_rm):
+            self.value_group_specs.pop(i)
+        if extra_values is not None:
+            self.extra_values_spec:ExtraValueListPolicy = extra_values
+
+    def to_dict(self) -> dict:
+        d = super().to_dict()
+        if self.extra_values_spec is not None:
+            self.extra_values_spec.group_name = extra_group_name
+            extra_values_dict = [self.extra_values_spec.to_dict()]
+        else:
+            extra_values_dict = []
+        d.update({"groups": [spec.to_dict() for spec in self.value_group_specs] + extra_values_dict})
+        return d
+
+    @staticmethod
+    def from_dict(d:dict) -> "GroupedValueListPolicy":
+        obj = GroupedValueListPolicy()
+        obj._load_from_dict(d)
+        return obj
+
+    def _load_from_dict(self, d:dict, child_cls:type=None):
+        super()._load_from_dict(d)
+        if child_cls is None:
+            child_cls = ValueListPolicy
+        self.value_group_specs = [child_cls.from_dict(group_spec) for group_spec in d["groups"]]
+        self.extra_values_spec = None
+        self._extract_extra_values()
+
+    def enforce(self, values: Union[str, List[str]], *, context:str=None, verbose: bool = True, buffer:List[DataPolicyError]=None) -> bool:
+        success = True
+        extra_spec_rm = set()
+        for value_group_spec in self.value_group_specs:
+            if not value_group_spec.group_name == extra_group_name.lower():
+                extra_spec_rm = extra_spec_rm.union({tag_spec.value for tag_spec in value_group_spec.list_specs})
+        for value_group_spec in self.value_group_specs:
+            success &= value_group_spec.enforce(values, context=context, verbose=verbose, buffer=buffer)
+        if self.extra_values_spec is not None:
+            self.extra_values_spec.group_name = extra_group_name
+            success &= self.extra_values_spec.enforce(values, context=context, verbose=verbose, buffer=buffer, extra_spec_rm=extra_spec_rm)
+        return success
+
+
+class SingleValueListPolicy(DataPolicyElementABC):
+    def __init__(self, base_list:ValueListPolicy=None, extra_values:ListChoiceMode=ListChoiceMode.Any, mandatory:bool=False):
+        super().__init__(mandatory=mandatory)
+        self.base_list: GroupedValueListPolicy = GroupedValueListPolicy(extra_values=ExtraValueListPolicy(value_select=extra_values))
+        self.update_base_list(base_list)
+
+    def to_dict(self) -> dict:
+        return self.base_list.to_dict()
+
+    @staticmethod
+    def from_dict(d:dict) -> "SingleValueListPolicy":
+        obj = SingleValueListPolicy()
+        obj._load_from_dict(d)
+        return obj
+
+    def _load_from_dict(self, d:dict):
+        # super()._load_from_dict(d)
+        self.base_list._load_from_dict(d)
+
+    def update_base_list(self, base_list:ValueListPolicy):
+        if base_list is not None:
+            base_list.group_name = "base"
+            self.base_list.value_group_specs = [base_list]
+        else:
+            self.base_list.value_group_specs = []
+
+    def enforce(self, values: Union[str, List[str]], *, context:str=None, verbose: bool = True, buffer:List[DataPolicyError]=None) -> bool:
+        success = self.base_list.enforce(values, context=context, verbose=verbose, buffer=buffer)
+        return success

ckanapi_harvesters/policies/data_format_policy_tag_groups.py

@@ -0,0 +1,35 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Data format policy representation and enforcing for lists of tags grouped in vocabularies
+"""
+from typing import List, Dict
+
+from ckanapi_harvesters.auxiliary.error_level_message import ErrorLevel
+from ckanapi_harvesters.auxiliary.ckan_auxiliary import assert_or_raise
+from ckanapi_harvesters.auxiliary.ckan_errors import MandatoryAttributeError
+from ckanapi_harvesters.policies.data_format_policy_lists import ValueListPolicy, GroupedValueListPolicy, ExtraValueListPolicy
+from ckanapi_harvesters.policies.data_format_policy_defs import ListChoiceMode
+from ckanapi_harvesters.policies.data_format_policy_defs import StringValueSpecification
+
+tag_subs_re = r"[^a-zA-Z0-9_\-\.]"
+
+
+class TagListPolicy(ValueListPolicy):
+    def get_tags_list_dict(self, vocabulary_id: str=None) -> List[Dict[str, str]]:
+        """
+        Generate tags dictionary to initiate a vocabulary using the CKAN API.
+        :param vocabulary_id:
+        :return:
+        """
+        if vocabulary_id is not None:
+            tags_list_dict = [{"name": tag_spec.value, "vocabulary_id": vocabulary_id} for tag_spec in self.list_specs]
+        else:
+            tags_list_dict = [{"name": tag_spec.value} for tag_spec in self.list_specs]
+        return tags_list_dict
+
+
+class TagGroupsListPolicy(GroupedValueListPolicy):
+    pass
+
+

ckanapi_harvesters/reports/__init__.py

@@ -0,0 +1,11 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Package to make reports on the CKAN database.
+"""
+
+from . import admin_report
+
+# usage shortcuts
+
+

ckanapi_harvesters/reports/admin_report.py

@@ -0,0 +1,292 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Detailed report on package resources: size, access rights and data format policy scores
+"""
+from typing import List, Union, Dict
+from collections import OrderedDict
+import time
+import datetime
+import os
+from warnings import warn
+
+from ckanapi_harvesters.ckan_api import CkanApi
+from ckanapi_harvesters.auxiliary.ckan_auxiliary import to_jsons_indent_lists_single_line
+from ckanapi_harvesters.auxiliary.ckan_model import CkanVisibility, CkanUserInfo
+from ckanapi_harvesters.policies.data_format_policy_errors import ErrorCount, DataPolicyError
+
+
+def round_size(value_mb:float) -> float:
+    return round(value_mb, 2)
+
+
+class CkanAdminReport:
+    def __init__(self, package_list:List[str]=None, cancel_if_present:bool=True,
+                 package_custom_fields:List[str]=None, ckan:CkanApi=None, full_report:bool=False):
+        if package_custom_fields is None:
+            package_custom_fields = []  # option to include specific custom fields in the report e.g. a end of license date
+        if isinstance(package_list, str):
+            package_list = [package_list]
+        self.package_list: Union[List[str],None] = package_list
+        self.cancel_if_present: bool = cancel_if_present
+        self.include_package_custom_fields: List[str] = package_custom_fields
+        self.include_resources_detail: bool = True
+        self.include_policy_messages: bool = full_report
+        self.include_group_report: bool = full_report
+        self.date_format:Union[str,None] = '%d/%m/%Y %H:%M'
+        self._connected_user: Union[CkanUserInfo, None] = None
+        self.report_date: Union[datetime.datetime, None] = None
+        self._elapsed_time_requests: Union[float,None] = None
+        self._request_count: Union[int,None] = None
+        self.report: Union[dict,None] = None  # report output
+        if ckan is not None:
+            self.execute(ckan)
+
+    def _date_format_str(self, date:datetime.datetime) -> str:
+        if self.date_format is not None:
+            return date.strftime(self.date_format)
+        else:
+            return date.isoformat()
+
+    def _perform_requests(self, ckan: CkanApi) -> None:
+        if not self.cancel_if_present:
+            ckan.purge(purge_map=True)
+        start = time.time()
+        self.report_date = datetime.datetime.now()
+        request_count_init = ckan.debug.ckan_request_counter
+        self._connected_user = ckan.query_current_user()
+        if not self._connected_user.sysadmin:
+            msg = f"It is recommended to run the report with a user with sysadmin rights. Current user: {self._connected_user.name}"
+            warn(msg)
+        ckan.map_resources(self.package_list, datastore_info=True, only_missing=self.cancel_if_present)
+        ckan.organization_list_all(cancel_if_present=False, include_users=True)
+        ckan.license_list(cancel_if_present=self.cancel_if_present)
+        ckan.map_file_resource_sizes(cancel_if_present=self.cancel_if_present)
+        ckan.map_user_rights(cancel_if_present=self.cancel_if_present)
+        self._elapsed_time_requests = time.time() - start
+        self._request_count = ckan.debug.ckan_request_counter - request_count_init
+
+    def _consolidate(self, ckan: CkanApi) -> None:
+        for user_info in ckan.map.users.values():
+            user_info.organizations = []
+        for organization_info in ckan.map.organizations.values():
+            for user_id in organization_info.user_members.keys():
+                ckan.map.users[user_id].organizations.append(organization_info.name)
+
+    def _create_report(self, ckan: CkanApi) -> None:
+        policy_messages: Dict[str, List[DataPolicyError]] = {}
+        ckan.policy_check(buffer=policy_messages)
+
+        report_header = OrderedDict([
+            ("title", "Admin report on packages and resources"),
+            ("date", self._date_format_str(self.report_date)),
+            ("ckan", ckan.url),
+            ("user", self._connected_user.name),
+            ("user_sysadmin", self._connected_user.sysadmin),
+            ("package_selection", self.package_list if self.package_list is not None else "All"),
+        ])
+        packages_report = {}
+        total_policy_errors = ErrorCount([])
+        total_filestore_size_mb = 0.
+        total_external_size_mb = 0.
+        total_datastore_size_mb = 0.
+        total_resource_count = 0
+        total_external_resource_count = 0
+        total_datastore_count = 0
+        total_datastore_lines = 0
+        global_last_modified_resources = None
+        global_last_modified_metadata = None
+        for package_id, package_info in ckan.map.packages.items():
+            package_name = package_info.name
+            package_data_format_messages = policy_messages.get(package_name, [])
+            data_format_policy_scores = ErrorCount(package_data_format_messages)
+            total_policy_errors += data_format_policy_scores
+            resources_report = []
+            last_modified_resource = None
+            last_modified_resource_metadata = None
+            package_resource_count = len(package_info.package_resources)
+            package_external_resource_count = 0
+            package_datastore_count = 0
+            package_filestore_size_mb = 0.
+            package_external_size_mb = 0.
+            package_datastore_size_mb = 0.
+            package_datastore_lines = 0
+            for resource_id, resource_info in package_info.package_resources.items():
+                resource_modified = resource_info.last_modified if resource_info.last_modified is not None else resource_info.created
+                internal_filestore = ckan.is_url_internal(resource_info.download_url)
+                resource_report = OrderedDict([
+                    ("resource_name", resource_info.name),
+                    ("id", resource_id),
+                    ("state", str(resource_info.state)),
+                    ("external_url", resource_info.download_url if resource_info.download_url and not internal_filestore else None),
+                    ("filestore_size_mb", resource_info.download_size_mb if internal_filestore else None),
+                    ("external_size_mb", resource_info.download_size_mb if not internal_filestore else None),
+                    ("datastore_size_mb", 0),
+                    ("datastore_active", resource_info.datastore_active),
+                    ("datastore_lines", None),
+                    ("date_modified", self._date_format_str(resource_modified) if resource_modified is not None else None),
+                    ("metadata_modified", self._date_format_str(resource_info.metadata_modified) if resource_info.metadata_modified is not None else None),
+                    ("datastore_aliases", None),
+                ])
+                if resource_modified is not None:
+                    last_modified_resource = max(last_modified_resource, resource_modified) \
+                        if last_modified_resource else resource_modified
+                    global_last_modified_resources = max(global_last_modified_resources, resource_modified) \
+                        if global_last_modified_resources else resource_modified
+                if resource_info.metadata_modified is not None:
+                    last_modified_resource_metadata = max(last_modified_resource_metadata, resource_info.metadata_modified) \
+                        if last_modified_resource_metadata else resource_info.metadata_modified
+                    global_last_modified_metadata = max(global_last_modified_metadata, resource_modified) \
+                        if global_last_modified_metadata else resource_modified
+                if resource_info.download_url:
+                    if internal_filestore:
+                        package_filestore_size_mb += resource_info.download_size_mb
+                    else:
+                        package_external_size_mb += resource_info.download_size_mb
+                        package_external_resource_count += 1
+                if resource_info.datastore_info is not None:
+                    datastore_size = round_size(resource_info.datastore_info.table_size_mb + resource_info.datastore_info.index_size_mb)
+                    resource_report["datastore_aliases"] = resource_info.datastore_info.aliases
+                    resource_report["datastore_size_mb"] = datastore_size
+                    package_datastore_size_mb += datastore_size
+                    resource_report["datastore_lines"] = resource_info.datastore_info.row_count
+                    package_datastore_lines += resource_info.datastore_info.row_count
+                    package_datastore_count += 1
+                resources_report.append(resource_report)
+            package_report = OrderedDict([
+                ("package_title", package_info.title),
+                ("state", str(package_info.state)),
+                ("organization", package_info.organization_info.name if package_info.organization_info else None),
+                ("version", package_info.version),
+                ("license", ckan.map.licenses[package_info.license_id].title if package_info.license_id else None),
+                ("license_domain", ckan.map.licenses[package_info.license_id].domain.to_dict() if package_info.license_id else None),
+                ("author", package_info.author),
+                ("maintainer", package_info.maintainer),
+                ("metadata_modified", self._date_format_str(package_info.metadata_modified)),
+                ("resources_modified", self._date_format_str(last_modified_resource) if last_modified_resource is not None else None),
+                ("resources_metadata_modified", self._date_format_str(last_modified_resource_metadata) if last_modified_resource_metadata is not None else None),
+                ("visibility", str(CkanVisibility.from_bool_is_private(package_info.private))),
+                ("filestore_total_size_mb", round_size(package_filestore_size_mb)),
+                ("external_total_size_mb", round_size(package_external_size_mb)),
+                ("datastore_total_size_mb", round_size(package_datastore_size_mb)),
+                ("datastore_total_lines", package_datastore_lines),
+                ("resource_count", package_resource_count),
+                ("among_resources_external", package_external_resource_count),
+                ("among_resources_datastore", package_datastore_count),
+                ("data_format_policy_scores", data_format_policy_scores.to_dict()),
+                ("tags", package_info.tags),
+            ])
+            for custom_field in self.include_package_custom_fields:
+                package_report[custom_field] = package_info.custom_fields.get(custom_field, None)
+            package_report["users"] = []
+            package_report["groups"] = []
+            if self.include_resources_detail:
+                package_report["resources"] = resources_report
+            if package_info.private:
+                users_dict = {ckan.map.users[user_id].name: collaboration.to_dict(ckan.map.users[user_id], ckan.map.groups, self.date_format)
+                              for user_id, collaboration in package_info.user_access.items()}
+                package_report["users"] = OrderedDict(sorted(users_dict.items()))
+            else:
+                # TODO: do all users have write access if package is Public
+                package_report["users"] = "all (Public)"
+            package_report["groups"] = sorted([group_info.name for group_info in package_info.groups])
+            if self.include_policy_messages:
+                package_report["policy_messages"] = [message.to_dict() for message in package_data_format_messages]
+            total_filestore_size_mb += package_filestore_size_mb
+            total_external_size_mb += package_external_size_mb
+            total_datastore_size_mb += package_datastore_size_mb
+            total_resource_count += package_resource_count
+            total_external_resource_count += package_external_resource_count
+            total_datastore_count += package_datastore_count
+            total_datastore_lines += package_datastore_lines
+            global_last_modified_metadata = max(global_last_modified_metadata, package_info.metadata_modified) \
+                if global_last_modified_metadata else package_info.metadata_modified
+            packages_report[package_name] = package_report
+        packages_report = OrderedDict(sorted(packages_report.items()))
+        report_totals = OrderedDict([
+            ("total_filestore_size_mb", round_size(total_filestore_size_mb)),
+            ("total_datastore_size_mb", round_size(total_datastore_size_mb)),
+            ("total_external_size_mb", round_size(total_external_size_mb)),
+            ("total_datastore_lines", total_datastore_lines),
+            ("num_packages", len(packages_report)),
+            ("total_resource_count", total_resource_count),
+            ("among_resources_external", total_external_resource_count),
+            ("among_resources_datastore", total_datastore_count),
+            ("last_modified_data", self._date_format_str(global_last_modified_resources) if global_last_modified_resources else None),
+            ("last_modified_metadata", self._date_format_str(global_last_modified_metadata) if global_last_modified_metadata else None),
+            ("total_policy_errors", total_policy_errors.to_dict()),
+        ])
+        sysadmin_report = {user_info.name: OrderedDict([
+            ("fullname", user_info.fullname),
+            ("last_active", self._date_format_str(user_info.last_active) if user_info.last_active is not None else None),
+            ("organizations", user_info.organizations),
+        ]) for user_info in ckan.map.users.values() if user_info.sysadmin}
+        sysadmin_report = OrderedDict(sorted(sysadmin_report.items()))
+        users_report = {user_info.name: OrderedDict([
+            ("fullname", user_info.fullname),
+            ("last_active", self._date_format_str(user_info.last_active) if user_info.last_active is not None else None),
+            ("organizations", user_info.organizations),
+         ]) for user_info in ckan.map.users.values() if not user_info.sysadmin}
+        users_report = OrderedDict(sorted(users_report.items()))
+        groups_report = {group_info.name: OrderedDict([
+            ("group_title", group_info.title),
+            ("package_count", group_info.package_count),
+            ("users_count", len(group_info.user_members)),
+            ("users", OrderedDict(sorted({ckan.map.users[user_id].name: str(capacity) for user_id, capacity in group_info.user_members.items()}.items())) if group_info.user_members is not None else None),
+        ]) for group_info in ckan.map.groups.values()}
+        groups_report = OrderedDict(sorted(groups_report.items()))
+        report_footer = OrderedDict([
+            ("requests_count", self._request_count),
+            ("time_elapsed_seconds", self._elapsed_time_requests),
+        ])
+        report = OrderedDict([
+            ("header", report_header),
+            ("totals", report_totals),
+            ("packages", packages_report),
+            ("users", OrderedDict([
+                ("sysadmins", sysadmin_report),
+                ("other", users_report),
+            ])),
+        ])
+        if self.include_group_report:
+            report["groups"] = groups_report
+        report["footer"] = report_footer
+        self.report = report
+
+    def execute(self, ckan: CkanApi) -> dict:
+        self._perform_requests(ckan)
+        self._consolidate(ckan)
+        self._create_report(ckan)
+        return self.report
+
+    def refresh_report(self, ckan: CkanApi) -> dict:
+        self._create_report(ckan)
+        return self.report
+
+    def to_jsons(self) -> str:
+        return to_jsons_indent_lists_single_line(self.report)
+
+    def to_json(self, file_path:str) -> None:
+        with open(file_path, "w", encoding="utf8") as f:
+            f.write(self.to_jsons())
+
+
+if __name__ == '__main__':
+    ckan = CkanApi()
+    ckan.initialize_from_cli_args()
+    ckan.input_missing_info(input_args_if_necessary=True, input_owner_org=True)
+
+    package_list = None  # use this argument or no argument to make a full report
+    # package_list = ["builder-example-py"]  # limit to the example package
+
+    report = CkanAdminReport(ckan=ckan, package_list=package_list, full_report=True)
+    print(report.to_jsons())
+
+    self_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+    top_dir = os.path.abspath(os.path.join(self_dir, "..", ".."))
+    tests_dir = os.path.abspath(os.path.join(top_dir, "..", "tests"))
+    out_file = os.path.join(tests_dir, "admin_report.json")
+    # out_file = os.path.join(tests_dir, f"admin_report_{report.report_date.strftime('%Y%m%dT%H%M')}.json")
+    report.to_json(out_file)
+
+    print(f"Done. Saved report to {out_file}")