okdata-cli 3.2.0__tar.gz → 4.1.0__tar.gz

{okdata-cli-3.2.0 → okdata-cli-4.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: okdata-cli
-Version: 3.2.0
+Version: 4.1.0
 Summary: CLI for services provided by Oslo Origo
 Home-page: https://github.com/oslokommune/okdata-cli/
 Author: Oslo Origo
@@ -24,7 +24,7 @@ Okdata CLI provides a unified interface to data services for Oslo Origo teams.
 
 * [Install](doc/install.md)
 * [Configuration](doc/configuration.md)
-* [Public registers](doc/pubreg.md)
+* [Public services](doc/pubs.md)
 * [Teams](doc/teams.md)
 * [Datasets](doc/datasets.md)
 * [Permissions](doc/permissions.md)

{okdata-cli-3.2.0 → okdata-cli-4.1.0}/README.md

@@ -4,7 +4,7 @@ Okdata CLI provides a unified interface to data services for Oslo Origo teams.
 
 * [Install](doc/install.md)
 * [Configuration](doc/configuration.md)
-* [Public registers](doc/pubreg.md)
+* [Public services](doc/pubs.md)
 * [Teams](doc/teams.md)
 * [Datasets](doc/datasets.md)
 * [Permissions](doc/permissions.md)

{okdata-cli-3.2.0 → okdata-cli-4.1.0}/okdata/cli/__main__.py

@@ -1,7 +1,7 @@
 import json
 import sys
 
-from keycloak.exceptions import KeycloakGetError
+from keycloak.exceptions import KeycloakGetError, KeycloakPostError
 from okdata.sdk.exceptions import ApiAuthenticateError
 from requests.exceptions import RequestException
 
@@ -9,7 +9,7 @@ from okdata.cli import MAINTAINER
 from okdata.cli.command import BaseCommand
 from okdata.cli.commands.datasets import DatasetsCommand
 from okdata.cli.commands.permissions import PermissionsCommand
-from okdata.cli.commands.pubreg import PubregCommand
+from okdata.cli.commands.pubs import PubsCommand
 from okdata.cli.commands.status import StatusCommand
 from okdata.cli.commands.teams.teams import TeamsCommand
 
@@ -40,11 +40,12 @@ def main():
                 "An error occurred (ApiAuthenticateError): Invalid credentials",
                 {"error": 1, "message": "Invalid credentials"},
             )
-        except KeycloakGetError as e:
+        except (KeycloakGetError, KeycloakPostError) as e:
             error = json.loads(e.error_message)
+            error_type = e.__class__.__name__
             instance.log.info(f"Keycloak reported: {e}")
             instance.print(
-                f"An error occurred (KeycloakGetError): {error['error_description']}"
+                f"An error occurred ({error_type}): {error['error_description']}"
             )
         except (EOFError, KeyboardInterrupt):
             instance.print("\nAbort.")
@@ -69,7 +70,7 @@ def get_command_class(argv):
     commands = {
         "datasets": DatasetsCommand,
         "permissions": PermissionsCommand,
-        "pubreg": PubregCommand,
+        "pubs": PubsCommand,
         "status": StatusCommand,
         "teams": TeamsCommand,
     }

{okdata-cli-3.2.0 → okdata-cli-4.1.0}/okdata/cli/command.py

@@ -21,7 +21,7 @@ class BaseCommand:
 usage:
   okdata datasets [options]
   okdata permissions [options]
-  okdata pubreg [options]
+  okdata pubs [options]
   okdata status [options]
   okdata teams [options]
   okdata -h | --help
@@ -29,7 +29,7 @@ usage:
 Commands available:
   datasets
   permissions
-  pubreg
+  pubs
   status
   teams
 

{okdata-cli-3.2.0 → okdata-cli-4.1.0}/okdata/cli/commands/datasets/datasets.py

@@ -292,6 +292,10 @@ Options:{BASE_COMMAND_OPTIONS}
         parts = dataset_uri.split("/")
         dataset_id, version, edition = parts + [None] * (3 - len(parts))
 
+        # First verify that the dataset exists; `get_dataset` raises an error
+        # if not.
+        self.sdk.get_dataset(dataset_id)
+
         if auto_resolve:
             if not version:
                 version = self._get_latest_version(dataset_id)["version"]

{okdata-cli-3.2.0 → okdata-cli-4.1.0}/okdata/cli/commands/datasets/questions.py

@@ -1,21 +1,17 @@
-import csv
-
 from questionary import Choice
 
-from okdata.cli.commands.datasets.validators import (
+from okdata.cli.commands.validators import (
     KeywordValidator,
-    PhoneValidator,
     SimpleEmailValidator,
-    SpatialResolutionValidator,
-    SpatialValidator,
-    StandardsValidator,
     TitleValidator,
 )
-from okdata.cli.commands.wizard import required_style
+from okdata.cli.commands.wizard import filter_comma_separated, required_style
 
 pipeline_choices = {
     "file": [
         Choice("Lagre dataen slik den er", "data-copy"),
+        Choice("Konverter fra CSV til Delta", "csv-to-delta"),
+        Choice("Konverter fra JSON til Delta", "json-to-delta"),
         Choice("Konverter fra CSV til Parquet", "csv-to-parquet"),
         Choice("Konverter fra Excel til CSV", "pipeline-excel-to-csv"),
         Choice("Ingen prosessering (krever manuell konfigurasjon av pipeline)", False),
@@ -30,14 +26,7 @@ available_pipelines = [
 ]
 
 
-def filter_comma_separated(value):
-    values = next(
-        csv.reader([value], delimiter=",", escapechar="\\", skipinitialspace=True)
-    )
-    return [x.strip() for x in values if x]
-
-
-def qs_create(include_extra_metadata=True):
+def qs_create():
     return [
         {
             **required_style,
@@ -67,38 +56,6 @@ def qs_create(include_extra_metadata=True):
             "validate": KeywordValidator,
             "filter": filter_comma_separated,
         },
-        {
-            **required_style,
-            "type": "confirm",
-            "name": "contains_geodata",
-            "message": "Inneholder datasettet geodata?",
-            "default": False,
-            "when": lambda x: include_extra_metadata,
-        },
-        {
-            "type": "text",
-            "name": "spatial",
-            "message": "Romlig avgrensning (komma-separert)",
-            "validate": SpatialValidator,
-            "filter": filter_comma_separated,
-            "when": lambda x: include_extra_metadata and x["contains_geodata"],
-        },
-        {
-            "type": "text",
-            "name": "spatialResolutionInMeters",
-            "message": "Romlig oppløsning (i meter)",
-            "validate": SpatialResolutionValidator,
-            "filter": lambda v: float(v.replace(",", ".")) if v else None,
-            "when": lambda x: include_extra_metadata and x["contains_geodata"],
-        },
-        {
-            "type": "text",
-            "name": "conformsTo",
-            "message": "I samsvar med standarder (komma-separert)",
-            "validate": StandardsValidator,
-            "filter": filter_comma_separated,
-            "when": lambda x: include_extra_metadata,
-        },
         {
             **required_style,
             "type": "select",
@@ -138,7 +95,6 @@ def qs_create(include_extra_metadata=True):
                     "http://creativecommons.org/publicdomain/zero/1.0/",
                 ),
             ],
-            "when": lambda x: include_extra_metadata,
         },
         {"type": "text", "name": "name", "message": "Kontaktperson – navn"},
         {
@@ -148,13 +104,6 @@ def qs_create(include_extra_metadata=True):
             "message": "Kontaktperson – epost",
             "validate": SimpleEmailValidator,
         },
-        {
-            **required_style,
-            "type": "text",
-            "name": "phone",
-            "message": "Kontaktperson – telefon",
-            "validate": PhoneValidator,
-        },
         {"type": "text", "name": "publisher", "message": "Utgiver"},
         {
             **required_style,

{okdata-cli-3.2.0 → okdata-cli-4.1.0}/okdata/cli/commands/datasets/wizards.py

@@ -29,17 +29,10 @@ class DatasetCreateWizard:
             "contactPoint": {
                 "name": choices["name"],
                 "email": choices["email"],
-                "phone": choices["phone"],
             },
             "publisher": choices["publisher"],
         }
 
-        if choices.get("spatial"):
-            config["spatial"] = choices["spatial"]
-        if choices.get("spatialResolutionInMeters"):
-            config["spatialResolutionInMeters"] = choices["spatialResolutionInMeters"]
-        if choices.get("conformsTo"):
-            config["conformsTo"] = choices["conformsTo"]
         if choices.get("license"):
             config["license"] = choices["license"]
 

{okdata-cli-3.2.0 → okdata-cli-4.1.0}/okdata/cli/commands/permissions.py

@@ -12,12 +12,13 @@ class PermissionsCommand(BaseCommand):
 
 Usage:
   okdata permissions ls [<resource_name>] [options]
-  okdata permissions add <resource_name> <user> <scope> [--team | --client] [options]
+  okdata permissions add <resource_name> <user> [<scope>] [--team | --client] [options]
   okdata permissions rm <resource_name> <user> [<scope>] [--team | --client] [options]
 
 Examples:
   okdata permissions ls
   okdata permissions ls okdata:dataset:my-dataset
+  okdata permissions add okdata:dataset:my-dataset janedoe
   okdata permissions add okdata:dataset:my-dataset janedoe okdata:dataset:read
   okdata permissions rm okdata:dataset:my-dataset janedoe
   okdata permissions rm okdata:dataset:my-dataset janedoe okdata:dataset:write
@@ -57,10 +58,10 @@ Options:{BASE_COMMAND_OPTIONS}
             ]
             if self.cmd("add"):
                 self.add_user(resource_name, user_class(user), scope)
-                self.print("Granted {} on '{}' for {} '{}'".format(*fmt_args))
+                self.print("Granted {} on '{}' to {} '{}'".format(*fmt_args))
             else:
                 self.remove_user(resource_name, user_class(user), scope)
-                self.print("Revoked {} on '{}' for {} '{}'".format(*fmt_args))
+                self.print("Revoked {} on '{}' from {} '{}'".format(*fmt_args))
 
     def list_my_permissions(self):
         """Print all permissions for the current user."""
@@ -99,9 +100,14 @@ Options:{BASE_COMMAND_OPTIONS}
         out.add_rows(sorted(results, key=itemgetter("user_type", "user_name")))
         self.print(f"Permissions for '{resource_name}'", out)
 
-    def add_user(self, resource_name, user, scope):
-        """Grant `scope` on `resource_name` to `user`."""
-        self.client.update_permission(resource_name, scope, add_users=[user])
+    def add_user(self, resource_name, user, scope=None):
+        """Grant `scope` on `resource_name` to `user`.
+
+        When `scope` is unspecified, grant every permission on `resource_name`.
+        """
+        return self.client.update_permission(
+            resource_name, scope or "__all__", add_users=[user]
+        )
 
     def remove_user(self, resource_name, user, scope=None):
         """Revoke `scope` on `resource_name` for `user`.
@@ -109,11 +115,6 @@ Options:{BASE_COMMAND_OPTIONS}
         When `scope` is unspecified, revoke every permission on `resource_name`
         for `user`.
         """
-        if scope:
-            self.client.update_permission(resource_name, scope, remove_users=[user])
-        else:
-            for permission in self.client.get_permissions(resource_name):
-                if user.user_id in permission[f"{user.user_type}s"]:
-                    self.client.update_permission(
-                        resource_name, permission["scope"], remove_users=[user]
-                    )
+        return self.client.update_permission(
+            resource_name, scope or "__all__", remove_users=[user]
+        )

okdata-cli-4.1.0/okdata/cli/commands/pubs/__init__.py

@@ -0,0 +1,3 @@
+from okdata.cli.commands.pubs.pubs import PubsCommand
+
+__all__ = ["PubsCommand"]

{okdata-cli-3.2.0/okdata/cli/commands/pubreg → okdata-cli-4.1.0/okdata/cli/commands/pubs}/client.py

@@ -5,23 +5,48 @@ from okdata.sdk import SDK
 log = logging.getLogger()
 
 
-class PubregClient(SDK):
+class PubsClient(SDK):
     def __init__(self, config=None, auth=None, env=None):
-        self.__name__ = "pubreg"
+        self.__name__ = "pubs"
         super().__init__(config, auth, env)
         self.api_url = "https://api.data{}.oslo.systems/maskinporten".format(
             "-dev" if self.config.config["env"] == "dev" else ""
         )
 
-    def create_client(self, team_id, provider, integration, scopes, env):
+    def create_maskinporten_client(self, team_id, provider, integration, scopes, env):
         data = {
+            "client_type": "maskinporten",
             "team_id": team_id,
             "provider": provider,
             "integration": integration,
             "scopes": scopes,
             "env": env,
         }
-        log.info(f"Creating client with payload: {data}")
+        log.info(f"Creating Maskinporten client with payload: {data}")
+        return self.post(f"{self.api_url}/clients", data=data).json()
+
+    def create_idporten_client(
+        self,
+        team_id,
+        integration,
+        client_uri,
+        frontchannel_logout_uri,
+        redirect_uris,
+        post_logout_redirect_uris,
+        env,
+    ):
+        data = {
+            "client_type": "idporten",
+            "team_id": team_id,
+            "provider": "idporten",
+            "integration": integration,
+            "client_uri": client_uri,
+            "frontchannel_logout_uri": frontchannel_logout_uri,
+            "redirect_uris": redirect_uris,
+            "post_logout_redirect_uris": post_logout_redirect_uris,
+            "env": env,
+        }
+        log.info(f"Creating ID-porten client with payload: {data}")
         return self.post(f"{self.api_url}/clients", data=data).json()
 
     def get_clients(self, env):

okdata-cli-3.2.0/okdata/cli/commands/pubreg/pubreg.py → okdata-cli-4.1.0/okdata/cli/commands/pubs/pubs.py

@@ -6,9 +6,14 @@ from okdata.sdk.team.client import TeamClient
 
 from okdata.cli import MAINTAINER
 from okdata.cli.command import BASE_COMMAND_OPTIONS, BaseCommand
-from okdata.cli.commands.pubreg.client import PubregClient
-from okdata.cli.commands.pubreg.questions import NoClientsError, NoKeysError
-from okdata.cli.commands.pubreg.wizards import (
+from okdata.cli.commands.pubs.client import PubsClient
+from okdata.cli.commands.pubs.questions import (
+    NoClientsError,
+    NoKeysError,
+    client_types,
+    providers,
+)
+from okdata.cli.commands.pubs.wizards import (
     audit_log_wizard,
     create_client_wizard,
     create_key_wizard,
@@ -21,33 +26,33 @@ from okdata.cli.commands.teams.questions import NoTeamError
 from okdata.cli.output import create_output
 
 
-class PubregCommand(BaseCommand):
-    __doc__ = f"""Oslo :: Public registers
+class PubsCommand(BaseCommand):
+    __doc__ = f"""Oslo :: Public services
 
 Usage:
-  okdata pubreg create-client [options]
-  okdata pubreg list-clients [options]
-  okdata pubreg delete-client [options]
-  okdata pubreg create-key [options]
-  okdata pubreg list-keys [options]
-  okdata pubreg delete-key [options]
-  okdata pubreg audit-log [options]
+  okdata pubs create-client [options]
+  okdata pubs list-clients [options]
+  okdata pubs delete-client [options]
+  okdata pubs create-key [options]
+  okdata pubs list-keys [options]
+  okdata pubs delete-key [options]
+  okdata pubs audit-log [options]
 
 Examples:
-  okdata pubreg create-client
-  okdata pubreg list-clients
-  okdata pubreg delete-client
-  okdata pubreg create-key
-  okdata pubreg list-keys
-  okdata pubreg delete-key
-  okdata pubreg audit-log
+  okdata pubs create-client
+  okdata pubs list-clients
+  okdata pubs delete-client
+  okdata pubs create-key
+  okdata pubs list-keys
+  okdata pubs delete-key
+  okdata pubs audit-log
 
 Options:{BASE_COMMAND_OPTIONS}
     """
 
     def __init__(self):
         super().__init__()
-        self.pubreg_client = PubregClient(env=self.opt("env"))
+        self.pubs_client = PubsClient(env=self.opt("env"))
         self.team_client = TeamClient(env=self.opt("env"))
 
     def handler(self):
@@ -77,40 +82,55 @@ Options:{BASE_COMMAND_OPTIONS}
             return
 
         team_id = config["team_id"]
+        client_type_id = config["client_type_id"]
         provider_id = config["provider_id"]
-        provider_name = config["provider_name"]
         integration = config["integration"]
         scopes = config["scopes"]
         env = config["env"]
 
         self.confirm_to_continue(
-            f"Will create a new client for {provider_name} in {env} with "
-            f"scopes {scopes}."
+            "Will create a new {} client{} in {}{}.".format(
+                client_types[client_type_id],
+                f" for {providers[provider_id]}" if provider_id else "",
+                env,
+                f" with scopes {scopes}" if scopes else "",
+            )
         )
 
         self.print("Creating client...")
-        response = self.pubreg_client.create_client(
-            team_id, provider_id, integration, scopes, env
-        )
+        if client_type_id == "maskinporten":
+            response = self.pubs_client.create_maskinporten_client(
+                team_id, provider_id, integration, scopes, env
+            )
+        else:
+            response = self.pubs_client.create_idporten_client(
+                team_id,
+                integration,
+                config["client_uri"],
+                config["frontchannel_logout_uri"],
+                config["redirect_uris"],
+                config["post_logout_redirect_uris"],
+                env,
+            )
         client_name = response["client_name"]
         self.print(
             f"""
 Done! Created a new client '{client_name}'.
 You may now go ahead and create a key for it by running:
 
-  okdata pubreg create-key"""
+  okdata pubs create-key"""
         )
 
     def list_clients(self):
         config = list_clients_wizard()
         env = config["env"]
-        clients = self.pubreg_client.get_clients(env)
-        out = create_output(self.opt("format"), "pubreg_clients_config.json")
+        clients = self.pubs_client.get_clients(env)
+        out = create_output(self.opt("format"), "pubs_clients_config.json")
         out.add_rows(sorted(clients, key=itemgetter("client_name")))
         self.print(f"Clients in {env}:", out)
 
     def delete_client(self):
-        choices = delete_client_wizard(self.pubreg_client)
+        choices = delete_client_wizard(self.pubs_client)
         env = choices["env"]
         client_id = choices["client_id"]
         client_name = choices["client_name"]
@@ -123,16 +143,18 @@ You may now go ahead and create a key for it by running:
                 client_name,
                 env,
                 (
-                    f" along with the key stored in AWS account {aws_account} "
-                    f"({aws_region})"
-                )
-                if delete_from_aws
-                else "",
+                    (
+                        f" along with the key stored in AWS account {aws_account} "
+                        f"({aws_region})"
+                    )
+                    if delete_from_aws
+                    else ""
+                ),
             ),
         )
 
         self.print(f"Deleting client '{client_name}' [{env}]...")
-        res = self.pubreg_client.delete_client(env, client_id, aws_account, aws_region)
+        res = self.pubs_client.delete_client(env, client_id, aws_account, aws_region)
         deleted_ssm_params = res["deleted_ssm_params"]
 
         self.print("\nDone! The client is deleted and will no longer work.")
@@ -203,11 +225,11 @@ You may now go ahead and create a key for it by running:
 
     def create_client_key(self):
         try:
-            config = create_key_wizard(self.pubreg_client)
+            config = create_key_wizard(self.pubs_client)
         except NoClientsError:
             self.print(
                 "No clients in the given environment yet!\n\n"
-                "  Create one with `okdata pubreg create-client`.\n"
+                "  Create one with `okdata pubs create-client`.\n"
             )
             return
 
@@ -226,22 +248,26 @@ You may now go ahead and create a key for it by running:
                         client_name,
                         env,
                         (
-                            f"send it to AWS account {aws_account} ({aws_region}), "
-                            "REPLACING any existing key for this client"
-                        )
-                        if key_destination == "aws"
-                        else "save it locally",
+                            (
+                                f"send it to AWS account {aws_account} ({aws_region}), "
+                                "REPLACING any existing key for this client"
+                            )
+                            if key_destination == "aws"
+                            else "save it locally"
+                        ),
+                    ),
+                    (
+                        "The key will be rotated automatically every night on weekdays.\n"
+                        if enable_auto_rotate
+                        else "The key will NOT be rotated automatically.\n"
                     ),
-                    "The key will be rotated automatically every night on weekdays.\n"
-                    if enable_auto_rotate
-                    else "The key will NOT be rotated automatically.\n",
                 ]
             )
         )
 
         self.print("Creating key for '{}' [{}]...".format(client_name, env))
 
-        key = self.pubreg_client.create_key(
+        key = self.pubs_client.create_key(
             env, client_id, aws_account, aws_region, enable_auto_rotate
         )
 
@@ -256,10 +282,10 @@ You may now go ahead and create a key for it by running:
             )
 
     def _list_client_keys_single(self, client_id, client_name, env):
-        keys = self.pubreg_client.get_keys(env, client_id)
+        keys = self.pubs_client.get_keys(env, client_id)
         out = create_output(
             self.opt("format"),
-            "pubreg_single_client_keys_config.json",
+            "pubs_single_client_keys_config.json",
         )
         out.add_rows(sorted(keys, key=itemgetter("expires")))
         self.print(f"Keys for client {client_name} [{env}]:", out)
@@ -269,17 +295,17 @@ You may now go ahead and create a key for it by running:
         keys = [
             {**key, "client_name": client["name"]}
             for client in clients
-            for key in self.pubreg_client.get_keys(env, client["id"])
+            for key in self.pubs_client.get_keys(env, client["id"])
         ]
         out = create_output(
             self.opt("format"),
-            "pubreg_multiple_client_keys_config.json",
+            "pubs_multiple_client_keys_config.json",
         )
         out.add_rows(sorted(keys, key=itemgetter("expires")))
         self.print(f"All client keys [{env}]:", out)
 
     def list_client_keys(self):
-        choices = list_keys_wizard(self.pubreg_client)
+        choices = list_keys_wizard(self.pubs_client)
         env = choices["env"]
 
         if "clients" in choices:
@@ -291,7 +317,7 @@ You may now go ahead and create a key for it by running:
 
     def delete_client_key(self):
         try:
-            choices = delete_key_wizard(self.pubreg_client)
+            choices = delete_key_wizard(self.pubs_client)
         except NoKeysError:
             self.print("The selected client doesn't have any keys.")
             return
@@ -306,17 +332,17 @@ You may now go ahead and create a key for it by running:
         )
 
         self.print(f"Deleting key '{key_id}' from '{client_name}' [{env}]...")
-        self.pubreg_client.delete_key(env, client_id, key_id)
+        self.pubs_client.delete_key(env, client_id, key_id)
         self.print("Done! The key is deleted and will no longer work.")
 
     def audit_log(self):
-        choices = audit_log_wizard(self.pubreg_client)
+        choices = audit_log_wizard(self.pubs_client)
         env = choices["env"]
         client_id = choices["client_id"]
         client_name = choices["client_name"]
 
-        audit_log = self.pubreg_client.get_audit_log(env, client_id)
+        audit_log = self.pubs_client.get_audit_log(env, client_id)
 
-        out = create_output(self.opt("format"), "pubreg_audit_log_config.json")
+        out = create_output(self.opt("format"), "pubs_audit_log_config.json")
         out.add_rows(sorted(audit_log, key=itemgetter("timestamp")))
         self.print(f"Audit log for client {client_name} [{env}]:", out)