ckanapi-harvesters 0.0.0__py3-none-any.whl → 0.0.2__py3-none-any.whl

ckanapi_harvesters/auxiliary/ckan_model.py

@@ -0,0 +1,992 @@
+#!python3
+# -*- coding: utf-8 -*-
+"""
+Data model to represent a CKAN database architecture
+"""
+import datetime
+import re
+from abc import ABC, abstractmethod
+from enum import IntEnum, IntFlag
+from typing import List, Dict, Union, Tuple
+from warnings import warn
+import copy
+from collections import OrderedDict
+
+import numpy as np
+
+from ckanapi_harvesters.auxiliary.ckan_auxiliary import assert_or_raise, _bool_from_string, bytes_to_megabytes
+from ckanapi_harvesters.auxiliary.ckan_auxiliary import CkanFieldInternalAttrs
+from ckanapi_harvesters.auxiliary.ckan_errors import IntegrityError, MissingIdError
+from ckanapi_harvesters.auxiliary.ckan_auxiliary import dict_recursive_update
+
+ckan_package_name_re = "^[0-9a-z-_]*$"
+
+## Enumerations ------------------
+class UpsertChoice(IntEnum):
+    Insert = 1
+    Update = 2
+    Upsert = 3
+
+    def __str__(self):
+        return self.name.lower()
+
+
+class CkanFieldTypeABC(ABC):
+    @staticmethod
+    @abstractmethod
+    def from_str(s):
+        raise NotImplementedError()
+
+class CkanFieldType(str, CkanFieldTypeABC):
+    """
+    Role previously managed by CkanFieldTypeEnum, but accepts any string
+    """
+    @staticmethod
+    def from_str(s):
+        return CkanFieldType(s)
+
+
+class CkanFieldTypeEnum(IntEnum):  #, CkanFieldTypeABC):
+    """
+    Enumeration of types encountered during development + documentation
+    """
+    # CKAN web interface
+    Text = 1
+    Numeric = 2
+    TimeStamp = 3
+    # [CKAN documentation](https://docs.ckan.org/en/2.9/maintaining/datastore.html#field-types)
+    json = 30
+    date = 11  # This type is used to represent a calendar date (year, month, day). The oldest date that can be represented is 4713 BC and the latest date is 5874897 AD. The resolution is 1 day.
+    time = 12  # This type is used to represent a time of day without time zone. The low value is 00:00:00 and the high value is 24:00:00 with a resolution of 1 microsecond.
+    int = 13
+    float = 14
+    bool = 15
+    # [Postgre documentation](https://www.postgresql.org/docs/9.1/datatype.html)
+    timetz = 20  # time of day, including time zone
+    timestamptz = 21  # date and time, including time zone
+    # structured data
+    jsonb = 31
+    xml = 32
+    # numeric types
+    int2 = 101
+    integer = int  # alias
+    int4 = int  # alias
+    int8 = 102
+    bigint = int8  # alias
+    int16 = 103
+    int32 = 104
+    int64 = 105
+    float4 = 106  # real
+    float8 = 107  # double
+    money = 108
+    # decimal [ (p, s) ] : exact numeric of selectable precision
+    # other types
+    bit = 200  # fixed-length bit string
+    char = 201  # This type is used to represent fixed-length, space padded strings of the specified width. Storing character strings longer than the specified length will result in an error unless the excess characters are spaces, in which case the string will be truncated to the maximum length. If the string to be stored is shorter than the declared length, the value will be space padded.
+    varbit = 202  # variable-length
+    varchar = 203  # variable-length
+    bytea = 204  # This type is used to represent binary strings (a “byte array”). A binary string is a sequence of octets (or bytes). Unlike character strings, binary strings allow storing octets of value zero and other non-printable octets (outside the range 32 to 126).
+    # identifiers
+    serial4 = 150  # This type is used to represent an auto incrementing four-byte integer. The range is 1 to 2147483647. This is similar to specifying an integer column that has default values to be assigned from a sequence generator. It also has a NOT NULL constraint applied to it.
+    serial = serial4  # alias
+    serial8 = 151
+    bigserial = serial8  # alias
+    uuid = 152  # This type is used to represent Universally Unique Identifiers (UUIDs). These identifiers are 128-bit values generated by an algorithm. A UUID is a sequence of lower-case hexadecimal digits in several groups separated by hyphens. Specifically, it is a group of 8 digits, followed by three groups of 4 digits, followed by a group of 12 digits. An example of a UUID in this standard form is a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11.
+    oid = 153  #  numeric object identifier. It is currently implemented as an unsigned four-byte integer. Its use as a primary key in a user-created table is discouraged. OIDs are best used only for references to system tables.
+    macaddr = 154
+    inet = 155
+    cidr = 156
+    # geometries, on a plane
+    point = 220
+    path = 221
+    polygon = 222
+    box = 223
+    circle = 224
+    lseg = 225  # line segment
+    line = 226
+    # PostGRE extensions:
+    geometry = 228  # PostGIS
+    bson = 230
+
+    def __str__(self):
+        return self.name.lower()
+
+    @staticmethod
+    def from_str(s):
+        s = s.lower().strip()
+        if s == "text":
+            return CkanFieldTypeEnum.Text
+        elif s == "numeric":
+            return CkanFieldTypeEnum.Numeric
+        elif s == "timestamp":
+            return CkanFieldTypeEnum.TimeStamp
+        # elif s == "" or ((not isinstance(s, str)) and np.isnan(s)):
+        #     return CkanFieldTypeEnum.Default
+        elif hasattr(CkanFieldTypeEnum, s):  # other attributes are lower case
+            return getattr(CkanFieldTypeEnum, s)
+        else:
+            raise ValueError(s)
+
+
+class CkanState(IntEnum):
+    Draft = 0
+    Active = 1
+    Deleted = 2
+
+    def __str__(self):
+        return self.name.lower()
+
+    @staticmethod
+    def from_str(s):
+        s = s.lower().strip()
+        if s == "active":
+            return CkanState.Active
+        elif s == "draft":
+            return CkanState.Draft
+        elif s == "deleted":
+            return CkanState.Deleted
+        else:
+            raise ValueError(s)
+
+
+class CkanVisibility(IntEnum):
+    Private = 0
+    Public = 1
+
+    def __str__(self):
+        return self.name.lower()
+
+    @staticmethod
+    def from_str(s):
+        s = s.lower().strip()
+        if s == "private":
+            return CkanVisibility.Private
+        elif s == "public":
+            return CkanVisibility.Public
+        else:
+            raise ValueError(s)
+
+    @staticmethod
+    def from_bool_is_private(value):
+        if value:
+            return CkanVisibility.Private
+        else:
+            return CkanVisibility.Public
+
+    def to_bool_is_private(self):
+        return CkanVisibility.Private == self.value
+
+
+class CkanLicenseDomain(IntFlag):
+    NoDomain = 0
+    Software = 1
+    Data = 2
+    Content = 4
+
+    @staticmethod
+    def from_bool(*, domain_software:bool=False, domain_data:bool=False, domain_content:bool=False) -> "CkanLicenseDomain":
+        flag = CkanLicenseDomain.NoDomain
+        if domain_software:
+            flag = flag | CkanLicenseDomain.Software
+        if domain_data:
+            flag = flag | CkanLicenseDomain.Data
+        if domain_content:
+            flag = flag | CkanLicenseDomain.Content
+        return flag
+
+    def to_dict(self) -> dict:
+        return OrderedDict([
+                ("domain_software", self.value & CkanLicenseDomain.Software > 0),
+                ("domain_data", self.value & CkanLicenseDomain.Data > 0),
+                ("domain_content", self.value & CkanLicenseDomain.Content > 0),
+            ])
+
+    @staticmethod
+    def from_dict(d: dict) -> "CkanLicenseDomain":
+        return CkanLicenseDomain.from_bool(domain_software=_bool_from_string(d["domain_software"]),
+                                           domain_data=_bool_from_string(d["domain_data"]),
+                                           domain_content=_bool_from_string(d["domain_content"]))
+
+class CkanCapacity(IntEnum):
+    Excluded = 0
+    Member = 1
+    Editor = 2  # only for collaborators of a package
+    Admin = 3   # only for members of a group
+    SysAdmin = 4
+    Public = 5  # to notify access is publicly available
+
+    def __str__(self):
+        return self.name.lower()
+
+    @staticmethod
+    def from_str(s):
+        s = s.lower().strip()
+        if s == "excluded":
+            return CkanCapacity.Excluded
+        elif s == "member":
+            return CkanCapacity.Member
+        elif s == "editor":
+            return CkanCapacity.Editor
+        elif s == "admin":
+            return CkanCapacity.Admin
+        elif s == "sysadmin":
+            return CkanCapacity.SysAdmin
+        elif s == "public":
+            return CkanCapacity.Public
+        else:
+            raise ValueError(s)
+
+class CkanConfigurableObjectABC(ABC):
+    mandatory_attributes: set = None
+    configurable_attributes: set = None
+    extra_attributes: set = set()
+
+    @staticmethod
+    @abstractmethod
+    def get_resource_type() -> str:
+        raise NotImplementedError()
+
+
+## Field class ------------------
+class CkanField(CkanConfigurableObjectABC):
+    """
+    Object representation of a CKAN Field configuration
+    """
+    mandatory_attributes = {"name"}
+    configurable_attributes = {"name", "notes", "label"}
+
+    # TODO: implement schema part of dict? e.g. {'index_name': None, 'is_index': False, 'native_type': 'numeric', 'notnull': False, 'uniquekey': False}
+    def __init__(self, name:str, data_type:str, *, native_type:str=None, notes:str=None,
+                 type_override:bool=False, label:str=None):
+        if native_type is None:
+            native_type = data_type
+        self.name:str = name
+        self.data_type:Union[CkanFieldType,None] = CkanFieldType.from_str(data_type)
+        self.type_override:Union[bool,None] = type_override
+        self.label:Union[str,None] = label
+        self.notes:Union[str,None] = notes
+        self.is_index:Union[bool,None] = None
+        self.uniquekey:Union[bool,None] = None
+        self.notnull:Union[bool,None] = None
+        self.internal_attrs: CkanFieldInternalAttrs = CkanFieldInternalAttrs()
+        self.details:dict = {}
+        self.details = self.to_ckan_dict()
+        self.internal_attrs.init_from_native_type(self.data_type)
+
+    def __str__(self):
+        return f"Field '{self.name}' <{self.data_type}>"
+
+    def copy(self) -> "CkanField":
+        return copy.deepcopy(self)
+
+    def merge(self, new_values):
+        dest = self.copy()
+        if new_values.name is not None:
+            dest.name = new_values.name
+        if new_values.data_type is not None:
+            dest.data_type = new_values.data_type
+        if new_values.type_override is not None:
+            dest.type_override = new_values.type_override
+        if new_values.label is not None:
+            dest.label = new_values.label
+        if new_values.notes is not None:
+            dest.notes = new_values.notes
+        if new_values.is_index is not None:
+            dest.is_index = new_values.is_index
+        if new_values.uniquekey is not None:
+            dest.uniquekey = new_values.uniquekey
+        if new_values.notnull is not None:
+            dest.notnull = new_values.notnull
+        dest.internal_attrs = self.internal_attrs.merge(new_values.internal_attrs)
+        dest.details = dict_recursive_update(self.details, new_values.details)
+        return dest
+
+    def __eq__(self, other) -> bool:
+        equality = True
+        equality &= self.name == other.name
+        equality &= self.data_type == other.data_type
+        equality &= self.type_override == other.type_override
+        equality &= self.label == other.label
+        equality &= self.notes == other.notes
+        equality &= self.is_index == other.is_index
+        equality &= self.uniquekey == other.uniquekey
+        equality &= self.notnull == other.notnull
+        equality &= self.internal_attrs == other.internal_attrs
+        return equality
+
+    @staticmethod
+    def get_resource_type() -> str:
+        return "Field"
+
+    def to_ckan_dict(self, include_details:bool=True) -> dict:
+        d = dict()
+        if self.details is not None and include_details:
+            d.update(self.details)
+        d["id"] = self.name
+        if self.data_type is not None:
+            d["type"] = str(self.data_type)
+        field_info = self.details["info"] if include_details and self.details is not None and "info" in self.details.keys() else {}
+        if self.type_override:
+            field_info["type_override"] = str(self.data_type)
+        if self.label is not None:
+            field_info["label"] = self.label
+        if self.notes is not None:
+            field_info["notes"] = self.notes
+        if len(field_info) > 0:
+            d["info"] = field_info
+        schema_info = self.details["schema"] if include_details and self.details is not None and "schema" in self.details.keys() else {}
+        if self.is_index is not None:
+            schema_info["is_index"] = self.is_index
+        if self.uniquekey is not None:
+            schema_info["uniquekey"] = self.uniquekey
+        if self.notnull is not None:
+            schema_info["notnull"] = self.notnull
+        if self.data_type is not None:
+            schema_info["native_type"] = str(self.data_type)
+        if len(schema_info) > 0:
+            d["schema"] = schema_info
+        return d
+
+    @staticmethod
+    def from_ckan_dict(d:dict) -> "CkanField":
+        obj = CkanField(d["id"], d["type"])
+        obj.details = d
+        if "info" in d.keys():
+            field_info = d["info"]
+            if "type_override" in field_info.keys():
+                if isinstance(field_info["type_override"], str):
+                    if len(field_info["type_override"]) > 0:
+                        # the API usually returns a string representing the type override, equal to d["type"]
+                        if not d["type"] == field_info["type_override"]:
+                            obj.data_type = CkanFieldType.from_str(field_info["type_override"])
+                            msg = f"Inconsistency between data type and type override for field {obj.name} (data: {d['type']} vs. override: {field_info['type_override']})"
+                            warn(msg)
+                        obj.type_override = True
+                elif isinstance(field_info["type_override"], bool):
+                    obj.type_override = field_info["type_override"]
+                else:
+                    obj.type_override = field_info["type_override"] > 0
+            if "label" in field_info.keys():
+                obj.label = field_info["label"]
+            if "notes" in field_info.keys():
+                obj.notes = field_info["notes"]
+        if "schema" in d.keys():
+            schema_info = d["schema"]
+            if "is_index" in schema_info.keys():
+                obj.is_index = schema_info["is_index"]
+            if "uniquekey" in schema_info.keys():
+                obj.uniquekey = schema_info["uniquekey"]
+            if "notnull" in schema_info.keys():
+                obj.notnull = schema_info["notnull"]
+            if "native_type" in schema_info.keys():
+                obj.data_type = CkanFieldType.from_str(schema_info["native_type"])
+        obj.internal_attrs.init_from_native_type(obj.data_type)
+        return obj
+
+    def to_dict(self, include_details:bool=True) -> dict:
+        return self.to_ckan_dict(include_details=include_details)
+
+    @staticmethod
+    def from_dict(d:dict) -> "CkanField":
+        return CkanField.from_ckan_dict(d)
+
+
+class CkanAliasInfo:
+    def __init__(self, d:dict=None):
+        self.id: Union[str,None] = None
+        self.name: str = ""
+        self.alias_of: Union[str,None] = None
+        self.details:dict = d
+        if d is not None:
+            self.id: Union[str, None] = d["id"] if "id" in d.keys() else None
+            self.name: str = d["name"]
+            self.alias_of: Union[str, None] = d["alias_of"]
+
+    def __str__(self):
+        return f"Alias {self.name} of id {self.alias_of}"
+
+    def copy(self) -> "CkanAliasInfo":
+        return copy.deepcopy(self)
+
+    @staticmethod
+    def from_dict(d:dict) -> "CkanAliasInfo":
+        return CkanAliasInfo(d)
+
+    def to_dict(self, include_details:bool=True) -> dict:
+        d = dict()
+        if self.details is not None and include_details:
+            d.update(self.details)
+        d.update({"name": self.name, "alias_of": self.alias_of})
+        if id is not None:
+            d["id"] = id
+        return d
+
+## Users and groups ------------------
+class CkanUserInfo:
+    def __init__(self, d: dict):
+        self.id: str = d["id"]
+        self.name: str = d["name"]
+        self.display_name: str = d["display_name"]
+        self.fullname: str = d["fullname"]
+        self.about: str = d["about"]
+        self.sysadmin: bool = d["sysadmin"]
+        self.state: CkanState = CkanState.from_str(d["state"])
+        self.created: Union[datetime.datetime, None] = datetime.datetime.fromisoformat(
+            d["created"]) if "created" in d.keys() else None
+        self.last_active: Union[datetime.datetime, None] = datetime.datetime.fromisoformat(
+            d["last_active"]) if "last_active" in d.keys() else None
+        self.organizations: Union[None,List[str]] = None  # used by consolidate (detailed_report)
+        self.details: dict = d
+
+    def __str__(self):
+        return f"User '{self.name}' ({self.id})"
+
+    @staticmethod
+    def get_resource_type() -> str:
+        return "User"
+
+    def to_dict(self, include_details: bool = True) -> dict:
+        d = dict()
+        if self.details is not None and include_details:
+            d.update(self.details)
+        d.update({"id": self.id, "name": self.name, "display_name": self.display_name, "fullname": self.fullname,
+                  "about": self.about, "sysadmin": self.sysadmin, "state": str(self.state),
+                  "created": self.created.isoformat() if self.created is not None else None,
+                  "last_active": self.last_active.isoformat() if self.last_active is not None else None})
+        return d
+
+    @staticmethod
+    def from_dict(d: dict) -> "CkanUserInfo":
+        return CkanUserInfo(d)
+
+    def copy(self) -> "CkanUserInfo":
+        return copy.deepcopy(self)
+
+
+class CkanGroupInfo:
+    def __init__(self, d:dict):
+        self.id:str = d["id"]
+        self.name:str = d["name"]
+        self.title:str = d["title"]
+        self.description:str = d["description"]
+        self.package_count:Union[None,int] = d.get("package_count")
+        self.details:dict = d
+        # to be initialized with specific requests:
+        self.user_members:Union[dict[str,CkanCapacity],None] = None
+        self.package_members:Union[dict[str,CkanCapacity],None] = None
+
+    def __str__(self):
+        return f"Group '{self.title}' ({self.id})"
+
+    @staticmethod
+    def get_resource_type() -> str:
+        return "Group"
+
+    def to_dict(self, include_details:bool=True) -> dict:
+        d = dict()
+        if self.details is not None and include_details:
+            d.update(self.details)
+        d.update({"id": self.id, "name": self.name, "title": self.title, "description": self.description})
+        return d
+
+    @staticmethod
+    def from_dict(d:dict) -> "CkanGroupInfo":
+        return CkanGroupInfo(d)
+
+    def copy(self) -> "CkanGroupInfo":
+        return copy.deepcopy(self)
+
+
+## Package, resources and views map class ------------------
+class CkanDataStoreInfo:
+    def __init__(self, d:dict=None):
+        self.resource_id:Union[str,None] = None
+        self.row_count: Union[int,None] = None
+        self.fields_id_list:Union[List[str],None] = None
+        self.fields_dict:Union[OrderedDict[str,CkanField],None] = None
+        self.index_fields:Union[List[str],None] = None
+        self.aliases:Union[List[str],None] = None
+        self.table_size_mb:Union[float,None] = None
+        self.index_size_mb:Union[float,None] = None
+        self.details:dict = d
+        if d is not None:
+            self.resource_id:str = d["meta"]["id"]
+            if "aliases" in d["meta"].keys():
+                self.aliases = d["meta"]["aliases"]
+            if "count" in d["meta"].keys():
+                self.row_count:int = d["meta"]["count"]
+            self.table_size_mb = bytes_to_megabytes(d["meta"]["size"])
+            self.index_size_mb = bytes_to_megabytes(d["meta"]["idx_size"])
+            # what does the field meta.db_size represent?
+            if "fields" in d.keys():
+                self.fields_id_list:List[str] = [e["id"] for e in d["fields"]]
+                self.fields_dict = OrderedDict()
+                for e in d["fields"]:
+                    self.fields_dict[e["id"]] = CkanField.from_ckan_dict(d=e)
+                self.index_fields:List[str] = [e.name for e in self.fields_dict.values() if e.is_index]
+
+    def __str__(self):
+        return f"DataStore of resource id {self.resource_id}"
+
+    def get_basic_field_list_dict(self):
+        return [{"id": id} for id in self.fields_id_list]
+
+    def get_recomp_field_list_dict(self):
+        return [self.fields_dict[id].to_ckan_dict() for id in self.fields_id_list]
+
+    def get_original_field_list_dict(self):
+        return [self.fields_dict[id].details for id in self.fields_id_list]
+
+    def copy(self) -> "CkanDataStoreInfo":
+        return copy.deepcopy(self)
+
+    def to_dict(self, include_details:bool=True) -> dict:
+        d = dict()
+        if self.details is not None and include_details:
+            d.update(self.details)
+        if self.fields_dict is not None:
+            d["fields"] = [field.to_dict(include_details=include_details) for field in self.fields_dict.values()]
+        if "meta" not in d.keys():
+            d["meta"] = {}
+        d["meta"]["id"] = self.resource_id
+        if self.row_count is not None:
+            d["meta"]["count"] = self.row_count
+        return d
+
+    @staticmethod
+    def from_dict(d:dict) -> "CkanDataStoreInfo":
+        return CkanDataStoreInfo(d)
+
+
+class CkanViewInfo:
+    def __init__(self, d:dict):
+        self.id:str = d["id"]
+        self.title:str = d["title"]
+        self.view_type:str = d["view_type"]
+        self.resource_id:str = d["resource_id"]
+        self.package_id:str = d["package_id"]
+        self.details:dict = d
+
+    def __str__(self):
+        return f"View '{self.title}' ({self.id} of resource id {self.resource_id})"
+
+    def to_dict(self, include_details:bool=True) -> dict:
+        d = dict()
+        if self.details is not None and include_details:
+            d.update(self.details)
+        d.update({"id": self.id, "title": self.title, "view_type": self.view_type, "resource_id": self.resource_id, "package_id": self.package_id})
+        return d
+
+    @staticmethod
+    def from_dict(d:dict) -> "CkanViewInfo":
+        return CkanViewInfo(d)
+
+    def copy(self) -> "CkanViewInfo":
+        return copy.deepcopy(self)
+
+
+class CkanLicenseInfo:
+    def __init__(self, d:dict):
+        self.id:str = d["id"]
+        self.title:str = d["title"]
+        self.state:CkanState = CkanState.from_str(d["status"])
+        self.family:str = d["family"]
+        self.domain:CkanLicenseDomain = CkanLicenseDomain.from_bool(domain_software=_bool_from_string(d["domain_software"]),
+                                                        domain_data=_bool_from_string(d["domain_data"]), domain_content=_bool_from_string(d["domain_content"]))
+        self.is_generic:bool = _bool_from_string(d["is_generic"])
+        self.url:str = d["url"]
+        self.details:dict = d
+
+    def __str__(self):
+        return f"License '{self.title}' ({self.id}) [{str(self.domain)}]"
+
+    def to_dict(self, include_details:bool=True) -> dict:
+        d = dict()
+        if self.details is not None and include_details:
+            d.update(self.details)
+        d.update({"id": self.id, "title": self.title, "state": str(self.state), "family": self.family})
+        d.update(self.domain.to_dict())
+        return d
+
+    @staticmethod
+    def from_dict(d:dict) -> "CkanLicenseInfo":
+        return CkanLicenseInfo(d)
+
+
+class CkanTagInfo:
+    def __init__(self, d:dict):
+        self.id:str = d["id"]
+        self.name:str = d["name"]
+        self.display_name:str = d["display_name"]
+        self.state: Union[CkanState,None] = None
+        if "state" in d.keys():
+            self.state = CkanState.from_str(d["state"])
+        self.vocabulary_id:Union[str,None] = d["vocabulary_id"]
+        self.details:dict = d
+
+    def __str__(self):
+        if self.vocabulary_id is None:
+            return f"Tag '{self.name}' ({self.id})"
+        else:
+            return f"Tag '{self.name}' ({self.id}) [vocabulary {self.vocabulary_id}]"
+
+    def to_dict(self, include_details:bool=True) -> dict:
+        d = dict()
+        if self.details is not None and include_details:
+            d.update(self.details)
+        d.update({"id": self.id, "name": self.name, "display_name": self.display_name,
+                  "vocabulary_id": self.vocabulary_id})
+        if self.state is not None:
+            d["state"] = self.state
+        return d
+
+    @staticmethod
+    def from_dict(d:dict) -> "CkanTagInfo":
+        return CkanTagInfo(d)
+
+
+class CkanResourceInfo(CkanConfigurableObjectABC):
+    mandatory_attributes = {"name"}
+    configurable_attributes = {"name", "state", "format", "description"}
+    extra_attributes = {"download_url"}
+
+    def __init__(self, d:dict=None, name:str=None, format:str=None, description:str=None, state:CkanState=None):
+        self.id:Union[str,None] = None
+        self.name:Union[str,None] = name
+        self.package_id:Union[str,None] = None
+        self.state:Union[CkanState,None] = state
+        self.datastore_active:Union[bool,None] = None
+        self.download_url:Union[str,None] = None
+        self.format:Union[str,None] = format
+        self.description:Union[str,None] = description
+        self.datastore_info:Union[CkanDataStoreInfo,None] = None
+        self.datastore_info_error:Union[dict,None] = None
+        self.views:Union[OrderedDict[str,CkanViewInfo],None] = None  # dict id -> view info (list of known views - full list not guaranteed)
+        self.view_is_full_list:bool = False
+        self.created: Union[datetime.datetime,None] = None
+        self.last_modified: Union[datetime.datetime,None] = None
+        self.metadata_modified: Union[datetime.datetime,None] = None
+        self.download_size_mb:Union[None,float] = None  # obtained through a HEAD request
+        if d is not None:
+            self.id = d["id"]
+            self.name = d["name"]
+            self.package_id = d["package_id"]
+            if "state" in d.keys():
+                self.state = CkanState.from_str(d["state"])
+            self.datastore_active = d["datastore_active"]
+            self.download_url = d["url"]
+            self.format = d["format"]
+            self.description = d["description"]
+            if "datastore_info" in d.keys():
+                self.datastore_info = CkanDataStoreInfo.from_dict(d["datastore_info"])
+            if "datastore_info_error" in d.keys():
+                self.datastore_info_error = d["datastore_info_error"]
+            if "views" in d.keys():
+                self.views = OrderedDict()
+                for view_dict in d["views"]:
+                    self.views[view_dict["id"]] = CkanViewInfo.from_dict(view_dict)
+            self.created = datetime.datetime.fromisoformat(d["created"]) if "created" in d.keys() else None
+            self.last_modified = datetime.datetime.fromisoformat(d["last_modified"]) if "last_modified" in d.keys() and d["last_modified"] is not None else None
+            self.metadata_modified = datetime.datetime.fromisoformat(d["metadata_modified"]) if "metadata_modified" in d.keys() else None
+        self.details:dict = d
+        self.index_in_package:Union[int,None] = -1
+        self.newly_created:bool = False
+        self.newly_updated:bool = False
+
+    def __str__(self):
+        if self.datastore_info is not None:
+            datastore_str = f"DataStore info"
+        elif self.datastore_active:
+            datastore_str = f"DataStore active"
+        else:
+            datastore_str = f"no DataStore"
+        return f"Resource '{self.name}' ({self.id}) [{self.state}, {datastore_str}]"
+
+    @staticmethod
+    def get_resource_type() -> str:
+        return "Resource"
+
+    def copy(self) -> "CkanResourceInfo":
+        return copy.deepcopy(self)
+
+    def datastore_queried(self) -> bool:
+        return self.datastore_info is not None or self.datastore_info_error is not None
+
+    def update_view(self, view_info: Union[CkanViewInfo, List[CkanViewInfo]], view_list:bool=False) -> None:
+        if isinstance(view_info, CkanViewInfo):
+            view_info = [view_info]
+        if self.views is None:
+            self.views = OrderedDict()
+        for view_info_update in view_info:
+            self.views[view_info_update.id] = view_info_update
+        self.view_is_full_list = self.view_is_full_list or view_list  # bool indicating if the list comes from full view list API
+
+    def update(self, refresh) -> None:
+        refresh: CkanResourceInfo
+        self.id = refresh.id
+        self.name = refresh.name
+        self.state = refresh.state
+        self.package_id = refresh.package_id
+        self.datastore_active = refresh.datastore_active
+        self.details = refresh.details
+
+    def to_dict(self, include_details:bool=True) -> dict:
+        d = dict()
+        if self.details is not None and include_details:
+            d.update(self.details)
+        d.update({"id": self.id, "name": self.name, "package_id": self.package_id,
+                 "datastore_active": self.datastore_active, "url": self.download_url,
+                 "format": self.format, "description": self.description,
+                  })
+        if self.state is not None:
+            d["state"] = str(self.state)
+        if self.datastore_info is not None:
+            d["datastore_info"] = self.datastore_info.to_dict(include_details=include_details)
+        if self.datastore_info_error is not None:
+            d["datastore_info_error"] = self.datastore_info_error
+        if self.views is not None:
+            d["views"] = [view.to_dict(include_details=include_details) for view in self.views.values()]
+        if self.created is not None:
+            d["created"] = self.created.isoformat()
+        if self.last_modified is not None:
+            d["last_modified"] = self.last_modified.isoformat()
+        if self.metadata_modified is not None:
+            d["metadata_modified"] = self.metadata_modified.isoformat()
+        return d
+
+    @staticmethod
+    def from_dict(d:dict) -> "CkanResourceInfo":
+        return CkanResourceInfo(d)
+
+
+class CkanCollaboration:
+    def __init__(self, capacity:CkanCapacity=None, modified:datetime.datetime=None, group_id:str=None, d:dict=None):
+        self.capacity:CkanCapacity = capacity
+        self.group_id:Union[str,None] = group_id
+        self.modified: Union[datetime.datetime,None] = modified
+        if d is not None:
+            self.capacity = CkanCapacity.from_str(d["capacity"])
+            self.modified:datetime.datetime = datetime.datetime.fromisoformat(d["modified"])
+
+    def __str__(self):
+        return str(self.capacity)
+
+    def copy(self) -> "CkanCollaboration":
+        return copy.deepcopy(self)
+
+    def to_dict(self, user_info: CkanUserInfo, group_table: Dict[str,CkanGroupInfo], date_format:str) -> dict:
+        d = OrderedDict([
+            ("full_name", user_info.fullname),
+            ("capacity", str(self.capacity)),
+        ])
+        if user_info.organizations is not None:
+            d["organizations"] = sorted(user_info.organizations)
+        if self.modified is not None:
+            if date_format is None:
+                d["date_modified"] = self.modified.isoformat()
+            else:
+                d["date_modified"] = self.modified.strftime(date_format)
+        if self.group_id is not None:
+            d["from_group"] = group_table[self.group_id].name
+        return d
+
+
+class CkanPackageInfo(CkanConfigurableObjectABC):
+    mandatory_attributes = {"name"}
+    configurable_attributes = {"name", "state", "title", "description", "private", "version",
+                             "author", "author_email", "maintainer", "maintainer_email"}
+    extra_attributes = {"tags", "custom_fields"}
+
+    def __init__(self, d:dict=None, *, package_name:str=None, package_id:str=None,
+                 title:str=None, description:str=None, private:bool=None, state:CkanState=None, version:str=None,
+                 url:str=None, tags:List[str]=None):
+        self.id:Union[str, None] = package_id
+        self.name:Union[str, None] = package_name
+        self.title:Union[str, None] = title
+        self.description:Union[str, None] = description
+        self.private:Union[bool, None] = private
+        self.state:Union[CkanState, None] = state
+        self.version:Union[str, None] = version
+        self.custom_fields:Dict[str,str] = {}  # key, value pairs
+        self.details:dict = {}
+        self.package_resources:OrderedDict[str,CkanResourceInfo] = OrderedDict()  # resource id -> info
+        self.resources_id_index:Dict[str,str] = {}  # resource name -> id
+        self.resources_id_index_counts:Dict[str,int] = {}  # resource name -> counter
+        self.organization_info: Union[CkanOrganizationInfo, None] = None
+        self.groups:List[CkanGroupInfo] = []
+        self.license_id:Union[str, None] = None
+        self.author:Union[str, None] = None
+        self.author_email:Union[str, None] = None
+        self.maintainer:Union[str, None] = None
+        self.maintainer_email:Union[str, None] = None
+        self.url:Union[str, None] = url
+        self.tags:Union[List[str],None] = tags
+        self.tags_info:Union[Dict[str, CkanTagInfo],None] = None  # dict tag name -> tag info
+        self.metadata_created: Union[datetime.datetime,None] = None
+        self.metadata_modified: Union[datetime.datetime,None] = None
+        self.requested_datastore_info:bool = False
+        self.newly_created:bool = False
+        self.collaborators:Union[None,Dict[str,CkanCollaboration]] = None  # given by API package_collaborator_list
+        self.user_access:Union[None,Dict[str,CkanCollaboration]] = None  # given by function map_user_rights
+
+        if d is not None:
+            self.id = d["id"]
+            self.name = d["name"]
+            self.title = d["title"]
+            self.description = d["notes"]
+            self.private = d["private"]
+            if "state" in d.keys():
+                self.state = CkanState.from_str(d["state"])
+            self.version = d["version"]
+            self.custom_fields = {field["key"]: field["value"] for field in d["extras"]}
+            self.details = d
+            self.package_resources = OrderedDict()
+            for resource_info_dict in d["resources"]:
+                self.package_resources[resource_info_dict["id"]] = CkanResourceInfo(resource_info_dict)
+            self.resources_id_index = {resource_info.name: resource_info.id for resource_info in self.package_resources.values()}  # resource name -> id
+            self.resources_id_index_counts = {}  # resource name -> counter
+            for resource_info in self.package_resources.values():
+                if resource_info.name not in self.resources_id_index_counts.keys():
+                    self.resources_id_index_counts[resource_info.name] = 1
+                else:
+                    self.resources_id_index_counts[resource_info.name] += 1
+            self.organization_info = None
+            if "organization" in d.keys():
+                self.organization_info = CkanOrganizationInfo(d["organization"])
+                assert_or_raise(self.organization_info.id == d["owner_org"], IntegrityError("Unexpected: organization != owner_org"))
+            else:
+                assert_or_raise("owner_org" not in d.keys() or d["owner_org"] == "", IntegrityError("Unexpected: organization is not present but owner_org was found"))
+            self.groups = [CkanGroupInfo(info) for info in d["groups"]]
+            self.license_id = d["license_id"]
+            self.author = d["author"]
+            self.author_email = d["author_email"]
+            self.maintainer = d["maintainer"]
+            self.maintainer_email = d["maintainer_email"]
+            self.metadata_created = datetime.datetime.fromisoformat(d["metadata_created"]) if "metadata_created" in d.keys() else None
+            self.metadata_modified = datetime.datetime.fromisoformat(d["metadata_modified"]) if "metadata_modified" in d.keys() else None
+            self.url = d["url"]
+            self.tags_info = {tag_dict["name"]: CkanTagInfo(tag_dict) for tag_dict in d["tags"]} if d["tags"] is not None else None
+            self.tags = list(self.tags_info.keys()) if self.tags_info is not None else None
+
+    def __str__(self):
+        return f"Package '{self.name}' ({self.id}) [{self.state}]"
+
+    @staticmethod
+    def get_resource_type() -> str:
+        return "Package"
+
+    def copy(self) -> "CkanPackageInfo":
+        return copy.deepcopy(self)
+
+    def update(self, refresh: "CkanPackageInfo"):
+        refresh: CkanPackageInfo
+        self.id = refresh.id
+        self.name = refresh.name
+        self.state = refresh.state
+        self.details = refresh.details
+        self.package_resources = refresh.package_resources
+        self.resources_id_index = refresh.resources_id_index
+        self.resources_id_index_counts = refresh.resources_id_index_counts
+
+    def get_resource_index(self, resource_id:str) -> int:
+        i_found = None
+        for i, res_info in enumerate(self.package_resources.values()):
+            if res_info.id == resource_id:
+                i_found = i
+                break
+        return i_found
+
+    def update_resource(self, resource_info: CkanResourceInfo) -> int:
+        if resource_info.id is None:
+            raise MissingIdError("Resource", resource_info.name)
+        resource_id = resource_info.id
+        i_update = self.get_resource_index(resource_id)
+        if i_update is not None:
+            resource_info.index_in_package = i_update
+            self.package_resources[resource_id] = resource_info
+        else:
+            i_update = len(self.package_resources) - 1
+            resource_info.index_in_package = i_update
+            self.package_resources[resource_id] = resource_info
+            self.resources_id_index_counts[resource_info.name] = 1
+        self.resources_id_index[resource_info.name] = resource_id
+        return i_update
+
+    def to_dict(self, include_details:bool=True) -> dict:
+        d = dict()
+        if self.details is not None and include_details:
+            d.update(self.details)
+        d.update({"id": self.id, "name": self.name, "title": self.title,
+                 "notes": self.description, "private": self.private, "version": self.version,
+                 "tags": self.tags, "url": self.url,
+                 "author": self.author, "author_email": self.author_email,
+                 "maintainer": self.maintainer, "maintainer_email": self.maintainer_email,
+                 "groups": [group.to_dict(include_details=include_details) for group in self.groups],
+                 "license_id": self.license_id,
+                 "resources": [resource.to_dict(include_details=include_details) for resource in self.package_resources.values()],
+                 })
+        if self.metadata_created is not None:
+            d["metadata_created"] = self.metadata_created.isoformat()
+        if self.metadata_modified is not None:
+            d["metadata_modified"] = self.metadata_modified.isoformat()
+        if self.state is not None:
+            d["state"] = str(self.state)
+        if self.custom_fields is not None:
+            d["extras"] = [{"key": key, "value": value} for key, value in self.custom_fields.items()]
+        if self.organization_info is not None:
+            d["owner_org"] = self.organization_info.id
+            d["organization"] = self.organization_info.to_dict(include_details=include_details)
+        return d
+
+    @staticmethod
+    def from_dict(d:dict) -> "CkanPackageInfo":
+        return CkanPackageInfo(d)
+
+
+class CkanOrganizationInfo:
+    def __init__(self, d:dict):
+        self.id = d["id"]
+        self.name = d["name"]
+        self.state = CkanState.from_str(d["state"])
+        self.title = d["title"]
+        self.user_members: Union[None,Dict[str,CkanCapacity]] = None
+        self.details = d
+        if "users" in d:
+            self.user_members = {user_dict["id"]: CkanCapacity.from_str(user_dict["capacity"]) for user_dict in d["users"]}
+
+    def __str__(self):
+        return f"Organization '{self.name}' ({self.id}) [{self.state}]"
+
+    def copy(self) -> "CkanOrganizationInfo":
+        return copy.deepcopy(self)
+
+    def get_owner_org(self):
+        """
+        Returns the value used for the owner_org argument
+
+        :return:
+        """
+        return self.name
+
+    def to_dict(self, include_details:bool=True) -> dict:
+        d = dict()
+        if self.details is not None and include_details:
+            d.update(self.details)
+        d.update({"id": self.id, "name": self.name, "title": self.title, "state": str(self.state)})
+        return d
+
+    @staticmethod
+    def from_dict(d:dict) -> "CkanOrganizationInfo":
+        return CkanOrganizationInfo(d)
+
+
+class PackageShortDescriptor:
+    """
+    Class to define more stable names to describe a package
+    """
+    def __init__(self, package_name:str, owner_org:str, resource_names: Dict[str,str]):
+        self.name: str = package_name
+        self.owner_org: str = owner_org
+        self.resource_names: Dict[str,str] = resource_names
+
+