disqover-api 1.23.0__py3-none-any.whl

disqover/__init__.py

@@ -0,0 +1,156 @@
+from .application_info import ApplicationInfoAPI
+from .common import DisqoverAPIError
+from .credentials_provider import BearerTokenCredentialsProvider
+from .credentials_provider import CommandLineCredentialsProvider
+from .credentials_provider import InlineCredentialsProvider
+from .data_ingestion import Component
+from .data_ingestion import ComponentCounter
+from .data_ingestion import ComponentResult
+from .data_ingestion import ComponentStatus
+from .data_ingestion import ComponentType
+from .data_ingestion import ComponentTypeOption
+from .data_ingestion import ComponentTypes
+from .data_ingestion import ComponentTypeSubOptions
+from .data_ingestion import ComponentWarning
+from .data_ingestion import DataIngestionAPI
+from .data_ingestion import FacetDataType
+from .data_ingestion import FacetViewType
+from .data_ingestion import FileType
+from .data_ingestion import IncrementalInfo
+from .data_ingestion import Pipeline
+from .data_ingestion import PipelineRun
+from .data_ingestion import PipelineRunInfo
+from .data_ingestion import PipelineRuns
+from .data_ingestion import Pipelines
+from .data_ingestion import PipelineStatus
+from .data_ingestion import PropertyDataType
+from .data_ingestion import PropertyRenderer
+from .data_ingestion import RunOptions
+from .data_ingestion import RunPipelineJob
+from .data_ingestion import ScanResult
+from .data_ingestion import ScanResultBase
+from .data_ingestion import Segment
+from .data_ingestion import SourceContents
+from .data_ingestion import SourceData
+from .data_ingestion import SourceDirectories
+from .data_ingestion import SourceDirectory
+from .data_ingestion import SourceFile
+from .data_ingestion import SourceFiles
+from .data_ingestion import SourceItem
+from .data_ingestion import UnresolvedScanResult
+from .data_ingestion import UriTemplates
+from .data_ingestion import actions
+from .data_query import AtLocation
+from .data_query import BelongsToCanonicalType
+from .data_query import CanonicalType
+from .data_query import CategoricalFacetParameters
+from .data_query import CombinationQuery
+from .data_query import Concept
+from .data_query import DataConfiguration
+from .data_query import DataOntology
+from .data_query import DataQueryAPI
+from .data_query import DataSource
+from .data_query import FacetEqual
+from .data_query import FacetMissing
+from .data_query import FacetPresent
+from .data_query import FacetResult
+from .data_query import FacetType
+from .data_query import FederationLocation
+from .data_query import FromDataSource
+from .data_query import FromLocalIngestion
+from .data_query import FromRDSServer
+from .data_query import HierarchicalFacetParameters
+from .data_query import InCollection
+from .data_query import InProximity
+from .data_query import InRange
+from .data_query import Instance
+from .data_query import InstancesFacetValuesResult
+from .data_query import InstanceSlice
+from .data_query import InstanceURIs
+from .data_query import InTree
+from .data_query import LinkedTo
+from .data_query import LinkedToCanonicalType
+from .data_query import LinksResult
+from .data_query import LinksTo
+from .data_query import LinksToCanonicalType
+from .data_query import LocationEqual
+from .data_query import MatchPropertyText
+from .data_query import MatchText
+from .data_query import NestedFacetQuantifier
+from .data_query import NestedFacetsQuantification
+from .data_query import NestedFacetsResult
+from .data_query import NumericalFacetParameters
+from .data_query import NumericalFacetResult
+from .data_query import PluginSearch
+from .data_query import Query
+from .data_query import RawQuery
+from .data_query import RelatedTo
+from .data_query import RelatesTo
+from .data_query import Relation
+from .data_query import RelationType
+from .data_query import SimpleText
+from .data_query import SubInstancesOf
+from .data_query import SubType
+from .data_query import TabularColumn
+from .data_query import TabularOptions
+from .data_query import parse_query
+from .disqover_session import DisqoverSession
+from .instance_collections import CollectionInstance
+from .instance_collections import InstanceCollection
+from .instance_collections import InstanceCollections
+from .instance_collections import InstanceCollectionsAPI
+from .m2m_authenticator import M2MAuthenticator
+from .managed_files import ManagedFile
+from .managed_files import ManagedFiles
+from .managed_files import ManagedFilesAPI
+from .oidc_authenticator import DeviceFlowAuthenticator
+from .plugins import PluginsAPI
+from .rds import DataSet
+from .rds import PublishedDataSets
+from .rds import RemoteDataSubscriptionAPI
+from .rds import Schedule
+from .rds import ScheduleRun
+from .rds import ScheduleRunJob
+from .rds import Subscription
+from .rds import Subscriptions
+from .resources import Resource
+from .resources import Resources
+from .resources import ResourcesAPI
+from .stored_data import ResourcePermissionsRule
+from .stored_data import StoredDataAPI
+from .stored_data import StoredDataResource
+from .stored_data import StoredDataResourceData
+from .stored_data import StoredDataResources
+from .stored_data import StoredDataResourceType
+from .stored_data import StoredDataResourceTypes
+from .stored_data import SubResourceParameters
+from .stored_data import UserSpecification
+from .stored_data import UserSpecificationType
+from .stored_data import migrate_resources
+from .system import SystemAPI
+from .system import SystemSetting
+from .system import SystemSettings
+from .tasks import Task
+from .tasks import Tasks
+from .tasks import TasksAPI
+from .user_management import CriticalityLevel
+from .user_management import ExternalGroup
+from .user_management import ExternalGroups
+from .user_management import LicenseAgreement
+from .user_management import LicenseAgreements
+from .user_management import LicenseManagementAPI
+from .user_management import M2MUser
+from .user_management import M2MUsers
+from .user_management import Operation
+from .user_management import SecurityEvent
+from .user_management import SecurityEvents
+from .user_management import User
+from .user_management import UserAccess
+from .user_management import UserGroup
+from .user_management import UserGroups
+from .user_management import UserManagementAPI
+from .user_management import Users
+from .user_views import UserView
+from .user_views import UserViews
+from .user_views import UserViewsAPI
+from .user_views import UserViewsApi

disqover/_hooks.py

@@ -0,0 +1,118 @@
+import abc
+import json
+import logging
+import time
+from contextlib import contextmanager
+from typing import Callable
+from typing import Optional
+
+import requests
+
+from disqover.http_basics import UNAUTHENTICATED
+from disqover.http_basics import RequestOptions
+from disqover.http_basics import describe_error
+from disqover.http_basics import is_error
+
+ResponseHook = Callable[..., Optional[requests.Response]]  # parameters are (requests.Response, *args, **kwargs)
+
+
+class PredefinedResponseHook(abc.ABC):
+    def __init__(self, session: "DisqoverSession"):
+        self._session = session
+        self._current_request: Optional[RequestOptions] = None
+
+    @abc.abstractmethod
+    def __call__(self, response: requests.Response, *args, **kwargs): ...
+
+    def _log_debug(self, func, *args, **kwargs) -> None:
+        self._log(logging.DEBUG, func, *args, **kwargs)
+
+    def _log_info(self, func, *args, **kwargs) -> None:
+        self._log(logging.INFO, func, *args, **kwargs)
+
+    def _log_warning(self, func, *args, **kwargs) -> None:
+        self._log(logging.WARNING, func, *args, **kwargs)
+
+    def _log_error(self, func, *args, **kwargs) -> None:
+        self._log(logging.ERROR, func, *args, **kwargs)
+
+    def _log(self, level, func, *args, **kwargs) -> None:
+        """lazy logging"""
+        if self._session._logger.isEnabledFor(level):
+            try:
+                msg = func()
+            except TypeError:
+                msg = func
+            self._session._logger.log(level, msg, *args, **kwargs)
+
+    @contextmanager
+    def set_current_request(self, request: RequestOptions):
+        self._current_request = request
+        yield
+        self._current_request = None
+
+
+class ReAuthenticateAndReplay(PredefinedResponseHook):
+    def __call__(self, response: requests.Response, *args, **kwargs):
+        if response.status_code == UNAUTHENTICATED:
+            if self._session._authenticator.can_reauthenticate:
+                self._log_debug("Authenticated session has expired, re-authenticating ...")
+                self._session._reset_session()
+                self._session._authenticator.authenticate(self._session._http_session, self._session._logger)
+                return self._session._get_response(self._current_request)
+            else:
+                raise ValueError("Invalid user credentials")
+
+
+class ResponseLogger(PredefinedResponseHook):
+    def __init__(self, session: "DisqoverSession", max_content_length: int = 250):
+        super().__init__(session)
+        self._max_content_length = max_content_length
+        self._ignore_error_status: set[int] = set()
+
+    def __call__(self, response: requests.Response, *args, **kwargs):
+        self._handle_default(response)
+        if is_error(response.status_code) and response.status_code not in self._ignore_error_status:
+            self._handle_error(response)
+
+    def _handle_default(self, response: requests.Response) -> None:
+        def format_msg():
+            try:
+                content_str = response.text
+            except:
+                content_str = "<binary data>"
+            if len(content_str) > self._max_content_length:
+                content_str = f"{content_str[:self._max_content_length]} ... ({len(content_str)} characters)"
+            else:
+                content_str = response.text
+            return f"Response: {response.status_code} with HEADERS: {response.headers} and CONTENT: {content_str}"
+
+        self._log_debug(format_msg)
+
+    def _handle_error(self, response: requests.Response) -> None:
+        self._log_error(f"!! A {describe_error(response)} has occurred")
+        self._log_error(f"!! REQUEST:")
+        self._log_error(f"!!  - {self._current_request.verb.upper()} {self._current_request.path}")
+        if self._current_request.params:
+            self._log_error(f"!!  - parameters:")
+            self._log_json(self._current_request.params)
+        if self._current_request.headers:
+            self._log_error(f"!!  - headers:")
+            self._log_dict(self._current_request.headers)
+        if self._current_request.payload:
+            self._log_error(f"!!  - payload:")
+            self._log_json(self._current_request.payload)
+        self._log_error("!!")
+        self._log_error(f"!! RESPONSE: (status={response.status_code})")
+        self._log_error(f"!!  - headers:")
+        self._log_dict(response.headers)
+        self._log_error(f"!!  - data:")
+        self._log_error(f"!!     {response.content}")
+
+    def _log_dict(self, dct: dict):
+        for key, val in dct.items():
+            self._log_error(f"!!     {key.upper()} = {val}")
+
+    def _log_json(self, json_data):
+        for line in json.dumps(json_data, indent=2).splitlines():
+            self._log_error(f"!!      {line}")

disqover/application_info/__init__.py

@@ -0,0 +1 @@
+from .api import ApplicationInfoAPI

disqover/application_info/api.py

@@ -0,0 +1,143 @@
+import dataclasses
+import datetime
+import enum
+from typing import Optional
+
+from disqover.common import API
+from disqover.disqover_session import DisqoverSession
+from disqover.type_hints import JSONDict
+from disqover.utilities import parse_datetime
+
+
+class DataQueryVersion(enum.Enum):
+    v1_0 = "v1.0"
+    v1_1 = "v1.1"
+
+
+def get_query_api_prefix(version: Optional[DataQueryVersion]) -> str:
+    if version is None:
+        return "api"
+    else:
+        return f"api/query/{version.value}"
+
+
+@dataclasses.dataclass(frozen=True)
+class DisqoverInfo:
+
+    version: str
+    datasets_last_modified: datetime.datetime
+    federated: bool
+    federated_and_responds: bool
+    license_nr_actions: int
+    allow_sharing: bool
+    session_time_minutes: int
+    enable_full_purls: bool
+    maps_mapbox_access_token: str
+    maps_mapbox_style_url: str
+    default_query_api_version: DataQueryVersion
+    disqover7: bool = True  # For backwards compatibility, should be removed eventually
+
+    def _get_version_part(self, part: int) -> Optional[int]:
+        try:
+            return int(self.version.split(".")[part])
+        except:
+            return None  # e.g. 'dev'
+
+    def has_official_version(self) -> bool:
+        # 'dev' -> False
+        # '6.11.1' -> True
+        return (
+            self._get_version_part(0) is not None
+            and self._get_version_part(1) is not None
+            and self._get_version_part(2) is not None
+        )
+
+    @property
+    def major_version(self) -> Optional[int]:
+        # e.g. '5.20.1' -> 5
+        return self._get_version_part(0)
+
+    @property
+    def minor_version(self) -> Optional[int]:
+        # e.g. '5.20.1' -> 2
+        return self._get_version_part(1)
+
+    @property
+    def patch_version(self) -> Optional[int]:
+        # e.g. '5.20.1' -> 1
+        return self._get_version_part(2)
+
+
+@dataclasses.dataclass(frozen=True)
+class SanityInfo:
+
+    name: str
+    description: str
+    status: str
+
+    def is_service_up(self) -> bool:
+        return self.status == "OK"
+
+
+class SearchPluginInfo:
+    def __init__(self, json_data: JSONDict):
+        self.name = json_data.get("name")
+        self.description = json_data.get("description")
+        self.config = json_data.get("config")
+        self.type = json_data.get("type")
+        self.visible_for_groups = json_data.get("visible_for_groups")
+
+
+class ApplicationInfoAPI(API):
+    def __init__(self, session: DisqoverSession):
+        super().__init__(session)
+
+    def get_info(self) -> DisqoverInfo:
+        result = self.session.get("/api/info")
+        version = result.get("version")
+        datasets_last_modified = (
+            parse_datetime(result["datasets_last_modified"]) if result.get("datasets_last_modified") else None
+        )
+        federated = result.get("federated")
+        federated_and_responds = result.get("federated_and_responds")
+        license_nr_actions = result.get("license_nr_actions")
+        allow_sharing = result.get("allow_sharing")
+        session_time_minutes = result.get("session_time_minutes")
+        enable_full_purls = result.get("enable_full_purls")
+        maps_mapbox_access_token = result.get("maps_mapbox_access_token")
+        maps_mapbox_style_url = result.get("maps_mapbox_style_url")
+        query_api_version_str = result.get("query_api_version", "1.0")
+        query_api_version = DataQueryVersion(f"v{query_api_version_str}")
+        return DisqoverInfo(
+            version,
+            datasets_last_modified,
+            federated,
+            federated_and_responds,
+            license_nr_actions,
+            allow_sharing,
+            session_time_minutes,
+            enable_full_purls,
+            maps_mapbox_access_token,
+            maps_mapbox_style_url,
+            query_api_version,
+        )
+
+    def sanity_check(self) -> list[SanityInfo]:
+        response = self.session.get("/api/info/sanity_check")
+        services = []
+        for info_data in response:
+            name = info_data.get("name")
+            description = info_data.get("description")
+            status = info_data.get("status")
+            services.append(SanityInfo(name, description, status))
+        return services
+
+    @property
+    def auth_type(self):
+        response = self.session.get("/api/info/auth_type")
+        return response
+
+    @property
+    def search_plugin_list(self) -> list[SearchPluginInfo]:
+        response = self.session.get("/api/plugins")
+        return [SearchPluginInfo(info) for info in response]

disqover/authenticator.py

@@ -0,0 +1,34 @@
+import posixpath
+import urllib.parse
+
+from .type_hints import URL
+
+
+class Authenticator:
+    """Encapsulates the data needed to create a session on a DISQOVER server.
+
+    Abstract class.
+    """
+
+    can_reauthenticate = True
+
+    @property
+    def url(self) -> str:
+        """The URL of the server, including the protocol and without trailing path.
+        E.g. https://federation.disqover.com
+        """
+        raise NotImplementedError()
+
+    def authenticate(self, http_session, logger):
+        raise NotImplementedError()
+
+    def authenticate_request(self, path: str, headers, logger) -> URL:
+        raise NotImplementedError()
+
+    def reset_cookies(self):
+        pass
+
+    def _join_path(self, path) -> URL:
+        if urllib.parse.urlparse(path).scheme:
+            return path  # absolute URL path
+        return urllib.parse.urljoin(self.url, posixpath.normpath(path))

disqover/background_jobs/models.py

@@ -0,0 +1,50 @@
+from disqover.common import APIObjectBase
+from disqover.disqover_session import DisqoverSession
+
+DEFAULT_PAGE_SIZE = 50
+
+
+class JobError(Exception):
+    pass
+
+
+class JobExecutionError(JobError):
+    pass
+
+
+class AlreadyRunning(JobError):
+    pass
+
+
+class UnknownStatus(JobError):
+    pass
+
+
+class NotRunning(JobError):
+    pass
+
+
+class JobNotFound(JobError):
+    pass
+
+
+class Job(APIObjectBase):
+    def __init__(self, session: DisqoverSession):
+        super().__init__(session)
+        self.errors = []
+
+    def start(self) -> None:
+        raise NotImplementedError("This kind of job can not be started")  # can be implemented in derived class, though
+
+    def stop(self) -> None:
+        raise NotImplementedError("This kind of job can not be stopped")  # can be implemented in derived class, though
+
+    def get_status(self):
+        raise NotImplementedError("Must be implemented in a derived class")
+
+    def wait(self, interval: int = 50) -> None:
+        raise NotImplementedError("Must be implemented in a derived class")
+
+    def stop_and_wait(self) -> None:  # to make it clear that the regular 'stop' method is non-blocking!
+        self.stop()
+        self.wait()

disqover/common/__init__.py

@@ -0,0 +1,18 @@
+from .attribute import APIAttribute
+from .attribute import ReadOnlyAPIAttribute
+from .attribute import ReadWriteAPIAttribute
+from .exceptions import DisqoverAPIError
+from .exceptions import DisqoverError
+from .identifiable import Identifiable
+from .object import APIObject
+from .object import APIObjectBase
+from .object import APIObjectCollection
+from .object import APIObjectCollectionWithGetAll
+from .object import ReadOnlyAPIObject
+
+
+class API(APIObjectBase):
+
+    # A high-level DISQOVER API class that bundles related functionality
+
+    pass

disqover/common/attribute.py

@@ -0,0 +1,54 @@
+from typing import Any
+from typing import Optional
+
+AttributeKey = str
+AttributeValue = Any
+
+
+class APIAttribute:
+    """
+    An attribute is defined by key and used as descriptor on an 'APIObject' class. It attempts to
+    retrieve the object itself when the attribute value is requested.
+    """
+
+    # TODO: take 2 arguments when defining an `APIAttribute`: one for the type of the value (now often indicated
+    #  ad hoc with a comment) and one for a user-friendly description of the attribute
+
+    @property
+    def key(self) -> AttributeKey:
+        return self._key
+
+    def __set_name__(self, owner, key: AttributeKey) -> None:
+        self._key = key
+
+    def __get__(self, obj: Optional["APIObject"], objtype=None):
+        if obj is None:
+            return type("APIAttributeHelp", (), {"__doc__": f"Attribute '{self._key}'"})
+        obj._retrieve()
+        return obj._get(self._key)
+
+
+class ReadOnlyAPIAttribute(APIAttribute):
+    """
+    A read-only attribute can never be set.
+    """
+
+    def __set__(self, obj: "APIObject", value: AttributeValue):
+        raise AttributeError(f"Attribute {self._key} of {obj.__class__.__name__} is read-only.")
+
+
+class ReadWriteAPIAttribute(APIAttribute):
+    """
+    An attribute of this class can be 'get' and 'set'. Upon setting a new value, this value is compared to the original
+    one. The object is saved if the value has changed.
+    """
+
+    def __set__(self, obj: "APIObject", value: AttributeValue):
+        obj._retrieve()
+        old_value = obj._get(self._key)
+        if old_value != value:
+            obj._set(self._key, value)
+            self.save_attribute(obj)
+
+    def save_attribute(self, obj: "APIObject"):
+        obj._save(changed_attribute=self._key)

disqover/common/exceptions.py

@@ -0,0 +1,14 @@
+class DisqoverError(Exception):
+    pass
+
+
+class DisqoverAPIError(DisqoverError):
+    pass
+
+
+class APIObjectNotFoundError(DisqoverAPIError):
+    pass
+
+
+class ProtectedDataError(DisqoverAPIError):
+    pass

disqover/common/identifiable.py

@@ -0,0 +1,13 @@
+from typing import Optional
+
+from disqover.type_hints import URI
+
+
+class Identifiable:
+    def __init__(self, uri: URI, label: Optional[str] = None, description: Optional[str] = None):
+        self.uri: URI = uri
+        self.label: str = label
+        self.description: Optional[str] = description
+
+    def __repr__(self):
+        return f'{self.__class__.__name__}("{self.uri}")'

disqover/common/jsonapi.py

@@ -0,0 +1,73 @@
+import json
+from typing import Optional
+from typing import Union
+from urllib.parse import quote
+
+from ..paging import PageParameters
+from ..type_hints import JSONDict
+
+APIResourceID = Union[str, int]
+
+
+def create_jsonapi_object(
+        type_id: str, resource_id: Optional[APIResourceID] = None, attributes: Optional[JSONDict] = None
+) -> JSONDict:
+    data = {"type": type_id}
+    if resource_id:
+        data["id"] = resource_id
+    if attributes:
+        data["attributes"] = attributes
+    return {"data": data}
+
+
+def create_jsonapi_objects(type_id: str, resources: list[JSONDict]) -> JSONDict:
+    data = []
+    for resource in resources:
+        id = resource.pop("id", None)
+        resource_json = {"type": type_id, "attributes": resource}
+        if id is not None:
+            resource_json["id"] = id
+        data.append(resource_json)
+    return {"data": data}
+
+
+def parse_jsonapi_object(data: JSONDict, check_type: Optional[str] = None) -> tuple[APIResourceID, JSONDict]:
+    if check_type:
+        assert data["data"]["type"] == check_type
+    return data["data"].get("id"), data["data"].get("attributes", {})
+
+
+def parse_jsonapi_objects(data: JSONDict) -> dict[APIResourceID, JSONDict]:
+    result = {}
+    resources = data["data"]
+    for resource in resources:
+        result[resource["id"]] = resource["attributes"]
+    return result
+
+
+def get_slicing_url_part(
+        resource: str,
+        paging: PageParameters,
+        order_by: Optional[str],
+        direction: Optional[str],
+        filters: Optional[JSONDict] = None,
+):
+    url = ""
+
+    # Pagination
+    if paging:
+        url += f"&page[page]={paging.index+1}&page[size]={paging.size}"
+
+    # Sorting
+    if order_by:
+        is_ascending = direction is None or direction == "asc"
+        direction_prefix = "" if is_ascending else "-"
+        url += f"&sort={direction_prefix}{order_by}"
+
+    # Filtering
+    if filters:
+        for key, value in filters.items():
+            value = json.dumps(value) if isinstance(value, dict) else value
+            url += f"&filter[{resource}.{key}]={quote(str(value))}"
+
+    return f"?{url[1:]}"