okdata-cli 4.4.0__tar.gz → 5.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/PKG-INFO +6 -5
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/__main__.py +9 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/command.py +3 -2
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/datasets/datasets.py +97 -37
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/status.py +11 -1
- okdata_cli-5.0.1/okdata/cli/data/output-format/datasets_config.json +18 -0
- okdata_cli-5.0.1/okdata/cli/data/output-format/datasets_config_verbose.json +50 -0
- okdata_cli-5.0.1/okdata/cli/data/output-format/datasets_dataset_config.json +30 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/datasets_dataset_version_config.json +2 -2
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/datasets_dataset_version_edition_config.json +2 -2
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/datasets_dataset_version_edition_distributions_config.json +2 -2
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/datasets_dataset_versions_config.json +2 -2
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/output.py +51 -20
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata_cli.egg-info/PKG-INFO +6 -5
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata_cli.egg-info/SOURCES.txt +1 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata_cli.egg-info/requires.txt +3 -2
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/setup.py +10 -5
- okdata_cli-4.4.0/okdata/cli/data/output-format/datasets_config.json +0 -14
- okdata_cli-4.4.0/okdata/cli/data/output-format/datasets_dataset_config.json +0 -22
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/LICENSE +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/README.md +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/__init__.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/__init__.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/__init__.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/datasets/__init__.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/datasets/questions.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/datasets/wizards.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/permissions.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/pubs/__init__.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/pubs/clients.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/pubs/pubs.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/pubs/questions.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/pubs/wizards.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/teams/questions.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/teams/teams.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/teams/util.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/teams/wizards.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/validators.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/commands/wizard.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/datasets_copy_file_config.json +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/datasets_copy_file_config_2.json +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/my_permissions_config.json +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/permissions_config.json +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/pubs_audit_log_config.json +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/pubs_clients_config.json +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/pubs_multiple_client_keys_config.json +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/pubs_single_client_keys_config.json +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/status_config.json +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/status_history_config.json +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/team_members_config.json +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/teams_config.json +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/io.py +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata_cli.egg-info/dependency_links.txt +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata_cli.egg-info/entry_points.txt +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata_cli.egg-info/namespace_packages.txt +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata_cli.egg-info/top_level.txt +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/pyproject.toml +0 -0
- {okdata_cli-4.4.0 → okdata_cli-5.0.1}/setup.cfg +0 -0
|
@@ -1,26 +1,27 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: okdata-cli
|
|
3
|
-
Version:
|
|
3
|
+
Version: 5.0.1
|
|
4
4
|
Summary: CLI for services provided by Oslo Origo
|
|
5
5
|
Home-page: https://github.com/oslokommune/okdata-cli/
|
|
6
6
|
Author: Oslo Origo
|
|
7
7
|
Author-email: dataspeilet@oslo.kommune.no
|
|
8
8
|
License: MIT
|
|
9
9
|
Classifier: Development Status :: 5 - Production/Stable
|
|
10
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
11
10
|
Classifier: Programming Language :: Python :: 3.9
|
|
12
11
|
Classifier: Programming Language :: Python :: 3.10
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
14
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
15
|
Classifier: License :: OSI Approved :: MIT License
|
|
16
16
|
Classifier: Operating System :: OS Independent
|
|
17
|
-
Requires-Python: >=3.
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
18
|
Description-Content-Type: text/markdown
|
|
19
19
|
License-File: LICENSE
|
|
20
|
-
Requires-Dist: PrettyTable
|
|
21
20
|
Requires-Dist: docopt
|
|
22
|
-
Requires-Dist: okdata-sdk<4,>=3.
|
|
21
|
+
Requires-Dist: okdata-sdk<4,>=3.4
|
|
22
|
+
Requires-Dist: PrettyTable
|
|
23
23
|
Requires-Dist: questionary<2.0.0,>=1.10.0
|
|
24
|
+
Requires-Dist: prompt-toolkit<3.0.52
|
|
24
25
|
Requires-Dist: requests
|
|
25
26
|
Dynamic: author
|
|
26
27
|
Dynamic: author-email
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import json
|
|
2
|
+
import os
|
|
2
3
|
import sys
|
|
3
4
|
|
|
4
5
|
from keycloak.exceptions import KeycloakGetError, KeycloakPostError
|
|
@@ -33,6 +34,9 @@ def main():
|
|
|
33
34
|
try:
|
|
34
35
|
instance.login()
|
|
35
36
|
instance.handle()
|
|
37
|
+
# Flush output here to force SIGPIPE to be triggered while inside
|
|
38
|
+
# this try block.
|
|
39
|
+
sys.stdout.flush()
|
|
36
40
|
except RequestException as e:
|
|
37
41
|
if hasattr(e.response, "json"):
|
|
38
42
|
instance.print_error_response(e.response.json())
|
|
@@ -55,6 +59,11 @@ def main():
|
|
|
55
59
|
)
|
|
56
60
|
except (EOFError, KeyboardInterrupt):
|
|
57
61
|
instance.print("\nAbort.")
|
|
62
|
+
except BrokenPipeError:
|
|
63
|
+
# https://docs.python.org/3/library/signal.html#note-on-sigpipe
|
|
64
|
+
devnull = os.open(os.devnull, os.O_WRONLY)
|
|
65
|
+
os.dup2(devnull, sys.stdout.fileno())
|
|
66
|
+
sys.exit(1)
|
|
58
67
|
except Exception as e:
|
|
59
68
|
instance.print(
|
|
60
69
|
"An exception occurred",
|
|
@@ -79,8 +79,9 @@ Options:{BASE_COMMAND_OPTIONS}
|
|
|
79
79
|
return content
|
|
80
80
|
|
|
81
81
|
def print(self, str, payload=None):
|
|
82
|
-
|
|
83
|
-
|
|
82
|
+
is_structured_data = self.opt("format") in ("csv", "json")
|
|
83
|
+
|
|
84
|
+
if not is_structured_data:
|
|
84
85
|
print(str)
|
|
85
86
|
# Normally a return json value from the API
|
|
86
87
|
if payload:
|
|
@@ -5,7 +5,7 @@ from okdata.sdk.data.download import Download
|
|
|
5
5
|
from okdata.sdk.data.upload import Upload
|
|
6
6
|
from requests.exceptions import HTTPError
|
|
7
7
|
|
|
8
|
-
from okdata.cli.command import BaseCommand, BASE_COMMAND_OPTIONS
|
|
8
|
+
from okdata.cli.command import BaseCommand, BASE_COMMAND_OPTIONS, confirm_to_continue
|
|
9
9
|
from okdata.cli.commands.datasets.wizards import (
|
|
10
10
|
DatasetCreateWizard,
|
|
11
11
|
PipelineCreateWizard,
|
|
@@ -18,21 +18,27 @@ class DatasetsCommand(BaseCommand):
|
|
|
18
18
|
__doc__ = f"""Oslo :: Datasets
|
|
19
19
|
|
|
20
20
|
Usage:
|
|
21
|
-
okdata datasets ls [--filter=<filter> options]
|
|
22
|
-
okdata datasets ls <uri> [options]
|
|
21
|
+
okdata datasets ls [--filter=<filter> --verbose options]
|
|
22
|
+
okdata datasets ls <uri> [--verbose options]
|
|
23
23
|
okdata datasets cp <source> <target> [options]
|
|
24
24
|
okdata datasets create [options]
|
|
25
|
-
okdata datasets create-version <
|
|
26
|
-
okdata datasets create-edition <
|
|
27
|
-
okdata datasets create-distribution <
|
|
28
|
-
okdata datasets create-pipeline <
|
|
25
|
+
okdata datasets create-version <dataset_id> [options]
|
|
26
|
+
okdata datasets create-edition <dataset_id> [<version>] [options]
|
|
27
|
+
okdata datasets create-distribution <dataset_id> [<version> <edition>] [options]
|
|
28
|
+
okdata datasets create-pipeline <dataset_id> [options]
|
|
29
|
+
okdata datasets delete-version <version_id> [options]
|
|
30
|
+
okdata datasets delete-edition <edition_id> [options]
|
|
31
|
+
okdata datasets delete-distribution <distribution_id> [options]
|
|
29
32
|
|
|
30
33
|
Examples:
|
|
31
34
|
okdata datasets ls
|
|
32
35
|
okdata datasets ls --filter=bydelsfakta
|
|
36
|
+
okdata datasets ls --verbose
|
|
33
37
|
okdata datasets ls my-dataset
|
|
38
|
+
okdata datasets ls my-dataset --verbose
|
|
34
39
|
okdata datasets ls my-dataset/1
|
|
35
40
|
okdata datasets ls my-dataset/1/20240101T102030
|
|
41
|
+
okdata datasets ls my-dataset/1/20240101T102030 --format=csv
|
|
36
42
|
okdata datasets ls my-dataset/1/20240101T102030 --format=json
|
|
37
43
|
okdata datasets create --file=dataset.json
|
|
38
44
|
okdata datasets cp /tmp/file.csv ds:my-dataset-id
|
|
@@ -46,7 +52,6 @@ Options:{BASE_COMMAND_OPTIONS}
|
|
|
46
52
|
|
|
47
53
|
def __init__(self):
|
|
48
54
|
super().__init__(Dataset)
|
|
49
|
-
self.download = Download(env=self.opt("env"))
|
|
50
55
|
|
|
51
56
|
def handler(self):
|
|
52
57
|
self.log.info("DatasetsCommand.handle()")
|
|
@@ -65,8 +70,14 @@ Options:{BASE_COMMAND_OPTIONS}
|
|
|
65
70
|
self.create_edition()
|
|
66
71
|
elif self.cmd("create-distribution"):
|
|
67
72
|
self.create_distribution()
|
|
73
|
+
elif self.cmd("delete-version"):
|
|
74
|
+
self.delete_version()
|
|
75
|
+
elif self.cmd("delete-edition"):
|
|
76
|
+
self.delete_edition()
|
|
77
|
+
elif self.cmd("delete-distribution"):
|
|
78
|
+
self.delete_distribution()
|
|
68
79
|
elif self.cmd("create-pipeline"):
|
|
69
|
-
PipelineCreateWizard(self, self.arg("
|
|
80
|
+
PipelineCreateWizard(self, self.arg("dataset_id")).start()
|
|
70
81
|
else:
|
|
71
82
|
self.help()
|
|
72
83
|
|
|
@@ -90,7 +101,10 @@ Options:{BASE_COMMAND_OPTIONS}
|
|
|
90
101
|
def datasets(self):
|
|
91
102
|
self.log.info("Listing datasets")
|
|
92
103
|
dataset_list = self.sdk.get_datasets(filter=self.opt("filter"))
|
|
93
|
-
out = create_output(
|
|
104
|
+
out = create_output(
|
|
105
|
+
self.opt("format"),
|
|
106
|
+
f"datasets_config{'_verbose' if self.opt('verbose') else ''}.json",
|
|
107
|
+
)
|
|
94
108
|
out.add_rows(dataset_list)
|
|
95
109
|
self.print("Available datasets", out)
|
|
96
110
|
|
|
@@ -108,7 +122,12 @@ Options:{BASE_COMMAND_OPTIONS}
|
|
|
108
122
|
)
|
|
109
123
|
return
|
|
110
124
|
|
|
111
|
-
|
|
125
|
+
config = (
|
|
126
|
+
"datasets_config_verbose"
|
|
127
|
+
if self.opt("verbose")
|
|
128
|
+
else "datasets_dataset_config.json"
|
|
129
|
+
)
|
|
130
|
+
out = create_output(self.opt("format"), config)
|
|
112
131
|
out.add_rows([dataset])
|
|
113
132
|
self.print(f"Dataset: {dataset_id}", out)
|
|
114
133
|
|
|
@@ -141,7 +160,7 @@ Options:{BASE_COMMAND_OPTIONS}
|
|
|
141
160
|
self.print(f"Editions available for: {dataset_id}, version: {version}", out)
|
|
142
161
|
|
|
143
162
|
def create_version(self):
|
|
144
|
-
dataset_id = self.arg("
|
|
163
|
+
dataset_id = self.arg("dataset_id")
|
|
145
164
|
payload = read_json(self.opt("file"))
|
|
146
165
|
self.log.info(
|
|
147
166
|
f"Creating version for dataset: {dataset_id} with payload: {payload}"
|
|
@@ -149,7 +168,7 @@ Options:{BASE_COMMAND_OPTIONS}
|
|
|
149
168
|
|
|
150
169
|
version = self.sdk.create_version(dataset_id, payload)
|
|
151
170
|
version_id = version["Id"]
|
|
152
|
-
self.log.info(f"Created version: {version_id}
|
|
171
|
+
self.log.info(f"Created version: {version_id} on dataset: {dataset_id}")
|
|
153
172
|
self.print(f"Created version: {version_id}", version)
|
|
154
173
|
|
|
155
174
|
def _get_latest_version(self, dataset_id, exit_on_error=True):
|
|
@@ -170,18 +189,32 @@ Options:{BASE_COMMAND_OPTIONS}
|
|
|
170
189
|
return None
|
|
171
190
|
raise
|
|
172
191
|
|
|
173
|
-
def
|
|
174
|
-
self.log.info(f"Trying to resolve
|
|
175
|
-
|
|
176
|
-
if
|
|
177
|
-
self.log.info(f"Found version in arguments: {
|
|
178
|
-
return
|
|
192
|
+
def resolve_or_load_version(self, dataset_id):
|
|
193
|
+
self.log.info(f"Trying to resolve version for {dataset_id}")
|
|
194
|
+
version = self.arg("version")
|
|
195
|
+
if version is not None:
|
|
196
|
+
self.log.info(f"Found version in arguments: {version}")
|
|
197
|
+
return version
|
|
179
198
|
latest_version = self._get_latest_version(dataset_id)
|
|
180
199
|
self.log.info(
|
|
181
200
|
f"Found version in latest dataset version: {latest_version['version']}"
|
|
182
201
|
)
|
|
183
202
|
return latest_version["version"]
|
|
184
203
|
|
|
204
|
+
def delete_version(self):
|
|
205
|
+
version_id = self.arg("version_id")
|
|
206
|
+
self.log.info(f"Deleting version {version_id}")
|
|
207
|
+
try:
|
|
208
|
+
dataset_id, version = version_id.split("/")
|
|
209
|
+
except ValueError:
|
|
210
|
+
sys.exit("Version ID must be on the format 'dataset_id/version'.")
|
|
211
|
+
confirm_to_continue(
|
|
212
|
+
f"Will delete version {version} together with every child edition "
|
|
213
|
+
"and distribution."
|
|
214
|
+
)
|
|
215
|
+
self.sdk.delete_version(dataset_id, version, cascade=True)
|
|
216
|
+
self.print(f"Deleted version {version}.")
|
|
217
|
+
|
|
185
218
|
# #################################### #
|
|
186
219
|
# Edition
|
|
187
220
|
# #################################### #
|
|
@@ -204,24 +237,24 @@ Options:{BASE_COMMAND_OPTIONS}
|
|
|
204
237
|
|
|
205
238
|
def create_edition(self):
|
|
206
239
|
payload = read_json(self.opt("file"))
|
|
207
|
-
dataset_id = self.arg("
|
|
208
|
-
|
|
240
|
+
dataset_id = self.arg("dataset_id")
|
|
241
|
+
version = self.resolve_or_load_version(dataset_id)
|
|
209
242
|
self.log.info(
|
|
210
|
-
f"Creating edition for {
|
|
243
|
+
f"Creating edition for {version} on {dataset_id} with payload: {payload}"
|
|
211
244
|
)
|
|
212
245
|
|
|
213
|
-
edition = self.sdk.create_edition(dataset_id,
|
|
214
|
-
self.print(f"Created edition for {
|
|
246
|
+
edition = self.sdk.create_edition(dataset_id, version, payload)
|
|
247
|
+
self.print(f"Created edition for {version} on {dataset_id}", edition)
|
|
215
248
|
return edition
|
|
216
249
|
|
|
217
|
-
def resolve_or_create_edition(self, dataset_id,
|
|
218
|
-
self.log.info(f"Trying to resolve edition for {
|
|
219
|
-
|
|
220
|
-
if
|
|
221
|
-
self.log.info(f"Found edition in arguments: {
|
|
222
|
-
return
|
|
250
|
+
def resolve_or_create_edition(self, dataset_id, version):
|
|
251
|
+
self.log.info(f"Trying to resolve edition for {version} on {dataset_id}")
|
|
252
|
+
edition = self.arg("edition")
|
|
253
|
+
if edition is not None:
|
|
254
|
+
self.log.info(f"Found edition in arguments: {edition}")
|
|
255
|
+
return edition
|
|
223
256
|
|
|
224
|
-
return self.sdk.auto_create_edition(dataset_id,
|
|
257
|
+
return self.sdk.auto_create_edition(dataset_id, version)
|
|
225
258
|
|
|
226
259
|
def get_latest_or_create_edition(self, dataset_id, version):
|
|
227
260
|
self.log.info(f"Resolving edition for dataset-uri: {dataset_id}/{version}")
|
|
@@ -232,23 +265,49 @@ Options:{BASE_COMMAND_OPTIONS}
|
|
|
232
265
|
return self.sdk.auto_create_edition(dataset_id, version)
|
|
233
266
|
raise
|
|
234
267
|
|
|
268
|
+
def delete_edition(self):
|
|
269
|
+
edition_id = self.arg("edition_id")
|
|
270
|
+
self.log.info(f"Deleting edition {edition_id}")
|
|
271
|
+
try:
|
|
272
|
+
dataset_id, version, edition = edition_id.split("/")
|
|
273
|
+
except ValueError:
|
|
274
|
+
sys.exit("Edition ID must be on the format 'dataset_id/version/edition'.")
|
|
275
|
+
confirm_to_continue(
|
|
276
|
+
f"Will delete edition {edition} together with every child distribution."
|
|
277
|
+
)
|
|
278
|
+
self.sdk.delete_edition(dataset_id, version, edition, cascade=True)
|
|
279
|
+
self.print(f"Deleted edition {edition}.")
|
|
280
|
+
|
|
235
281
|
# #################################### #
|
|
236
282
|
# Distribution
|
|
237
283
|
# #################################### #
|
|
238
284
|
def create_distribution(self):
|
|
239
285
|
payload = read_json(self.opt("file"))
|
|
240
|
-
dataset_id = self.arg("
|
|
241
|
-
|
|
242
|
-
edition_id = self.resolve_or_create_edition(dataset_id,
|
|
286
|
+
dataset_id = self.arg("dataset_id")
|
|
287
|
+
version = self.resolve_or_load_version(dataset_id)
|
|
288
|
+
edition_id = self.resolve_or_create_edition(dataset_id, version)["Id"]
|
|
243
289
|
edition = edition_id.split("/")[-1]
|
|
244
290
|
self.log.info(f"Creating distribution for {edition_id} with payload: {payload}")
|
|
245
291
|
|
|
246
292
|
distribution = self.sdk.create_distribution(
|
|
247
|
-
dataset_id,
|
|
293
|
+
dataset_id, version, edition, payload
|
|
248
294
|
)
|
|
249
295
|
self.print(f"Created distribution for {edition_id}", distribution)
|
|
250
296
|
return distribution
|
|
251
297
|
|
|
298
|
+
def delete_distribution(self):
|
|
299
|
+
dist_id = self.arg("distribution_id")
|
|
300
|
+
self.log.info(f"Deleting distribution {dist_id}")
|
|
301
|
+
try:
|
|
302
|
+
dataset_id, version, edition, dist = dist_id.split("/")
|
|
303
|
+
except ValueError:
|
|
304
|
+
sys.exit(
|
|
305
|
+
"Distribution ID must be on the format 'dataset_id/version/edition/distribution'."
|
|
306
|
+
)
|
|
307
|
+
confirm_to_continue(f"Will delete distribution {dist_id}.")
|
|
308
|
+
self.sdk.delete_distribution(dataset_id, version, edition, dist)
|
|
309
|
+
self.print(f"Deleted distribution {dist_id}.")
|
|
310
|
+
|
|
252
311
|
# #################################### #
|
|
253
312
|
# File handling
|
|
254
313
|
# #################################### #
|
|
@@ -329,7 +388,7 @@ Options:{BASE_COMMAND_OPTIONS}
|
|
|
329
388
|
return dataset_id, version, edition
|
|
330
389
|
|
|
331
390
|
def upload_file(self, source, target):
|
|
332
|
-
upload = Upload()
|
|
391
|
+
upload = Upload(env=self.opt("env"))
|
|
333
392
|
dataset_id, version, edition = self._dataset_components_from_uri(target, True)
|
|
334
393
|
|
|
335
394
|
self.log.info(f"Will upload file to: {dataset_id}/{version}/{edition})")
|
|
@@ -360,8 +419,9 @@ Options:{BASE_COMMAND_OPTIONS}
|
|
|
360
419
|
self.print("\n".join(summary))
|
|
361
420
|
|
|
362
421
|
def download_files(self, source, target):
|
|
422
|
+
download = Download(env=self.opt("env"))
|
|
363
423
|
dataset_id, version, edition = self._dataset_components_from_uri(source)
|
|
364
|
-
downloaded_files =
|
|
424
|
+
downloaded_files = download.download(
|
|
365
425
|
dataset_id, version, edition, resolve_output_filepath(target)
|
|
366
426
|
)
|
|
367
427
|
self.log.info(f"Download returned: {downloaded_files}")
|
|
@@ -101,13 +101,23 @@ Options:{BASE_COMMAND_OPTIONS}
|
|
|
101
101
|
if trace_status != "FINISHED":
|
|
102
102
|
print("Waiting for processing to finish", end="", flush=True)
|
|
103
103
|
|
|
104
|
-
|
|
104
|
+
attempts = 0
|
|
105
|
+
max_attempts = 100
|
|
106
|
+
while trace_status != "FINISHED" and attempts < max_attempts:
|
|
105
107
|
print(".", end="", flush=True)
|
|
106
108
|
sleep(2)
|
|
109
|
+
attempts += 1
|
|
107
110
|
trace_events = self.get_trace_events(trace_id)
|
|
108
111
|
trace_status = StatusCommand.find_latest_event(trace_events)["trace_status"]
|
|
109
112
|
print()
|
|
110
113
|
|
|
114
|
+
if attempts == max_attempts:
|
|
115
|
+
print(
|
|
116
|
+
"\nWe've waited for a good while now, and processing hasn't "
|
|
117
|
+
"yet finished.\nSomething else is most likely wrong, maybe a "
|
|
118
|
+
"pipeline hasn't been configured for the dataset?\n"
|
|
119
|
+
)
|
|
120
|
+
|
|
111
121
|
return trace_events
|
|
112
122
|
|
|
113
123
|
def status_for_id(self, trace_id):
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
{
|
|
2
|
+
"ID": {
|
|
3
|
+
"name": "ID",
|
|
4
|
+
"key": "Id"
|
|
5
|
+
},
|
|
6
|
+
"Publisher": {
|
|
7
|
+
"name": "Publisher",
|
|
8
|
+
"key": "publisher"
|
|
9
|
+
},
|
|
10
|
+
"Parent ID": {
|
|
11
|
+
"name": "Parent ID",
|
|
12
|
+
"key": "parent_id"
|
|
13
|
+
},
|
|
14
|
+
"Access rights": {
|
|
15
|
+
"name": "Access rights",
|
|
16
|
+
"key": "accessRights"
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
{
|
|
2
|
+
"ID": {
|
|
3
|
+
"name": "ID",
|
|
4
|
+
"key": "Id"
|
|
5
|
+
},
|
|
6
|
+
"Title": {
|
|
7
|
+
"name": "Title",
|
|
8
|
+
"key": "title"
|
|
9
|
+
},
|
|
10
|
+
"Description": {
|
|
11
|
+
"name": "Description",
|
|
12
|
+
"key": "description"
|
|
13
|
+
},
|
|
14
|
+
"Publisher": {
|
|
15
|
+
"name": "Publisher",
|
|
16
|
+
"key": "publisher"
|
|
17
|
+
},
|
|
18
|
+
"Objective": {
|
|
19
|
+
"name": "Objective",
|
|
20
|
+
"key": "objective"
|
|
21
|
+
},
|
|
22
|
+
"Parent ID": {
|
|
23
|
+
"name": "Parent ID",
|
|
24
|
+
"key": "parent_id"
|
|
25
|
+
},
|
|
26
|
+
"Access rights": {
|
|
27
|
+
"name": "Access rights",
|
|
28
|
+
"key": "accessRights"
|
|
29
|
+
},
|
|
30
|
+
"Theme": {
|
|
31
|
+
"name": "Theme",
|
|
32
|
+
"key": "theme"
|
|
33
|
+
},
|
|
34
|
+
"Keywords": {
|
|
35
|
+
"name": "Keywords",
|
|
36
|
+
"key": "keywords"
|
|
37
|
+
},
|
|
38
|
+
"Frequency": {
|
|
39
|
+
"name": "Frequency",
|
|
40
|
+
"key": "frequency"
|
|
41
|
+
},
|
|
42
|
+
"Contact point": {
|
|
43
|
+
"name": "Contact Point",
|
|
44
|
+
"key": "contactPoint",
|
|
45
|
+
"fields": [
|
|
46
|
+
"name",
|
|
47
|
+
"email"
|
|
48
|
+
]
|
|
49
|
+
}
|
|
50
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"Title": {
|
|
3
|
+
"name": "Title",
|
|
4
|
+
"key": "title"
|
|
5
|
+
},
|
|
6
|
+
"Description": {
|
|
7
|
+
"name": "Description",
|
|
8
|
+
"key": "description"
|
|
9
|
+
},
|
|
10
|
+
"Publisher": {
|
|
11
|
+
"name": "Publisher",
|
|
12
|
+
"key": "publisher"
|
|
13
|
+
},
|
|
14
|
+
"Parent ID": {
|
|
15
|
+
"name": "Parent ID",
|
|
16
|
+
"key": "parent_id"
|
|
17
|
+
},
|
|
18
|
+
"Access rights": {
|
|
19
|
+
"name": "Access rights",
|
|
20
|
+
"key": "accessRights"
|
|
21
|
+
},
|
|
22
|
+
"Contact point": {
|
|
23
|
+
"name": "Contact point",
|
|
24
|
+
"key": "contactPoint",
|
|
25
|
+
"fields": [
|
|
26
|
+
"name",
|
|
27
|
+
"email"
|
|
28
|
+
]
|
|
29
|
+
}
|
|
30
|
+
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import io
|
|
1
3
|
import json
|
|
2
4
|
import logging
|
|
3
5
|
import os
|
|
@@ -9,25 +11,28 @@ from prettytable import PrettyTable
|
|
|
9
11
|
log = logging.getLogger()
|
|
10
12
|
|
|
11
13
|
|
|
12
|
-
def
|
|
14
|
+
def _get_script_path():
|
|
13
15
|
return os.path.dirname(__file__)
|
|
14
16
|
|
|
15
17
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
# Formatting of output is defined in okdata/cli/data/ouput-format.
|
|
19
|
-
# Each command defines what (from the API response) should be printed, and
|
|
20
|
-
# what the human-readable name should be
|
|
21
|
-
def create_output(format, configfile):
|
|
22
|
-
datadir = f"{get_script_path()}/data/output-format"
|
|
23
|
-
filename = f"{datadir}/{configfile}"
|
|
24
|
-
log.info(f"Creating output format: {format}, from: {filename}")
|
|
25
|
-
with open(filename) as config_file:
|
|
26
|
-
config = json.load(config_file)
|
|
18
|
+
def create_output(fmt, config_file):
|
|
19
|
+
"""Create and return an output printer.
|
|
27
20
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
21
|
+
Formatting of output is defined in the `okdata/cli/data/ouput-format`
|
|
22
|
+
directory. Each command defines which fields from the API response should
|
|
23
|
+
be printed, and what the human-readable name should be.
|
|
24
|
+
"""
|
|
25
|
+
datadir = f"{_get_script_path()}/data/output-format"
|
|
26
|
+
filename = f"{datadir}/{config_file}"
|
|
27
|
+
log.info(f"Creating output format: {fmt}, from: {filename}")
|
|
28
|
+
|
|
29
|
+
with open(filename) as f:
|
|
30
|
+
config = json.load(f)
|
|
31
|
+
|
|
32
|
+
return {
|
|
33
|
+
"json": JsonOutput(config),
|
|
34
|
+
"csv": CSVOutput(config),
|
|
35
|
+
}.get(fmt, TableOutput(config))
|
|
31
36
|
|
|
32
37
|
|
|
33
38
|
class TableOutput(PrettyTable):
|
|
@@ -104,13 +109,9 @@ class TableOutput(PrettyTable):
|
|
|
104
109
|
return value
|
|
105
110
|
|
|
106
111
|
|
|
107
|
-
|
|
108
|
-
class JsonOutput:
|
|
112
|
+
class StructuredDataOutput:
|
|
109
113
|
def __init__(self, config):
|
|
110
114
|
self.config = config
|
|
111
|
-
# set to True it will return a single object from self.out when printed. This will make it
|
|
112
|
-
# easier to pass output to jq without having to access out[0] from the CLI
|
|
113
|
-
self.output_singular_object = False
|
|
114
115
|
self.out = []
|
|
115
116
|
|
|
116
117
|
def field_values(self, row, row_key, key):
|
|
@@ -144,6 +145,36 @@ class JsonOutput:
|
|
|
144
145
|
for row in rows:
|
|
145
146
|
self.add_row(row)
|
|
146
147
|
|
|
148
|
+
def __str__(self):
|
|
149
|
+
raise NotImplementedError
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class CSVOutput(StructuredDataOutput):
|
|
153
|
+
def __str__(self):
|
|
154
|
+
out = io.StringIO()
|
|
155
|
+
writer = csv.DictWriter(out, self.out[0].keys())
|
|
156
|
+
writer.writeheader()
|
|
157
|
+
writer.writerows(self.out)
|
|
158
|
+
return out.getvalue()
|
|
159
|
+
|
|
160
|
+
def get_row_value(self, row, key):
|
|
161
|
+
val = super().get_row_value(row, key)
|
|
162
|
+
|
|
163
|
+
if isinstance(val, str):
|
|
164
|
+
return val.replace("\n", "")
|
|
165
|
+
elif isinstance(val, list):
|
|
166
|
+
return ";".join(val)
|
|
167
|
+
return val
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# TODO: merge many outputs to one array of elements when listing a version
|
|
171
|
+
class JsonOutput(StructuredDataOutput):
|
|
172
|
+
def __init__(self, config):
|
|
173
|
+
super().__init__(config)
|
|
174
|
+
# set to True it will return a single object from self.out when printed. This will make it
|
|
175
|
+
# easier to pass output to jq without having to access out[0] from the CLI
|
|
176
|
+
self.output_singular_object = False
|
|
177
|
+
|
|
147
178
|
def __repr__(self):
|
|
148
179
|
return json.dumps(self.out)
|
|
149
180
|
|
|
@@ -1,26 +1,27 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: okdata-cli
|
|
3
|
-
Version:
|
|
3
|
+
Version: 5.0.1
|
|
4
4
|
Summary: CLI for services provided by Oslo Origo
|
|
5
5
|
Home-page: https://github.com/oslokommune/okdata-cli/
|
|
6
6
|
Author: Oslo Origo
|
|
7
7
|
Author-email: dataspeilet@oslo.kommune.no
|
|
8
8
|
License: MIT
|
|
9
9
|
Classifier: Development Status :: 5 - Production/Stable
|
|
10
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
11
10
|
Classifier: Programming Language :: Python :: 3.9
|
|
12
11
|
Classifier: Programming Language :: Python :: 3.10
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
14
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
15
|
Classifier: License :: OSI Approved :: MIT License
|
|
16
16
|
Classifier: Operating System :: OS Independent
|
|
17
|
-
Requires-Python: >=3.
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
18
|
Description-Content-Type: text/markdown
|
|
19
19
|
License-File: LICENSE
|
|
20
|
-
Requires-Dist: PrettyTable
|
|
21
20
|
Requires-Dist: docopt
|
|
22
|
-
Requires-Dist: okdata-sdk<4,>=3.
|
|
21
|
+
Requires-Dist: okdata-sdk<4,>=3.4
|
|
22
|
+
Requires-Dist: PrettyTable
|
|
23
23
|
Requires-Dist: questionary<2.0.0,>=1.10.0
|
|
24
|
+
Requires-Dist: prompt-toolkit<3.0.52
|
|
24
25
|
Requires-Dist: requests
|
|
25
26
|
Dynamic: author
|
|
26
27
|
Dynamic: author-email
|
|
@@ -27,6 +27,7 @@ okdata/cli/commands/teams/teams.py
|
|
|
27
27
|
okdata/cli/commands/teams/util.py
|
|
28
28
|
okdata/cli/commands/teams/wizards.py
|
|
29
29
|
okdata/cli/data/output-format/datasets_config.json
|
|
30
|
+
okdata/cli/data/output-format/datasets_config_verbose.json
|
|
30
31
|
okdata/cli/data/output-format/datasets_copy_file_config.json
|
|
31
32
|
okdata/cli/data/output-format/datasets_copy_file_config_2.json
|
|
32
33
|
okdata/cli/data/output-format/datasets_dataset_config.json
|
|
@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
|
|
|
5
5
|
|
|
6
6
|
setuptools.setup(
|
|
7
7
|
name="okdata-cli",
|
|
8
|
-
version="
|
|
8
|
+
version="5.0.1",
|
|
9
9
|
author="Oslo Origo",
|
|
10
10
|
author_email="dataspeilet@oslo.kommune.no",
|
|
11
11
|
description="CLI for services provided by Oslo Origo",
|
|
@@ -23,22 +23,27 @@ setuptools.setup(
|
|
|
23
23
|
],
|
|
24
24
|
},
|
|
25
25
|
install_requires=[
|
|
26
|
-
"PrettyTable",
|
|
27
26
|
"docopt",
|
|
28
|
-
"okdata-sdk>=3.
|
|
27
|
+
"okdata-sdk>=3.4,<4",
|
|
28
|
+
"PrettyTable",
|
|
29
|
+
# TODO: Upgrade to questionary 2.x.
|
|
29
30
|
"questionary>=1.10.0,<2.0.0",
|
|
31
|
+
# Not a direct dependency, but a dependency for questionary. However
|
|
32
|
+
# questionary breaks with prompt-toolkit 3.0.52 (and possibly later
|
|
33
|
+
# versions as well), so let's pin it sub 3.0.52 for now.
|
|
34
|
+
"prompt-toolkit<3.0.52",
|
|
30
35
|
"requests",
|
|
31
36
|
],
|
|
32
37
|
entry_points={"console_scripts": ["okdata=okdata.cli.__main__:main"]},
|
|
33
38
|
classifiers=[
|
|
34
39
|
"Development Status :: 5 - Production/Stable",
|
|
35
|
-
"Programming Language :: Python :: 3.8",
|
|
36
40
|
"Programming Language :: Python :: 3.9",
|
|
37
41
|
"Programming Language :: Python :: 3.10",
|
|
38
42
|
"Programming Language :: Python :: 3.11",
|
|
39
43
|
"Programming Language :: Python :: 3.12",
|
|
44
|
+
"Programming Language :: Python :: 3.13",
|
|
40
45
|
"License :: OSI Approved :: MIT License",
|
|
41
46
|
"Operating System :: OS Independent",
|
|
42
47
|
],
|
|
43
|
-
python_requires=">=3.
|
|
48
|
+
python_requires=">=3.9",
|
|
44
49
|
)
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"Publisher": {
|
|
3
|
-
"name": "Publisher",
|
|
4
|
-
"key": "publisher"
|
|
5
|
-
},
|
|
6
|
-
"Frequency": {
|
|
7
|
-
"name": "Frequency",
|
|
8
|
-
"key": "frequency"
|
|
9
|
-
},
|
|
10
|
-
"Objective": {
|
|
11
|
-
"name": "Objective",
|
|
12
|
-
"key": "objective"
|
|
13
|
-
},
|
|
14
|
-
"ContactPoint": {
|
|
15
|
-
"name": "Contact Point",
|
|
16
|
-
"key": "contactPoint",
|
|
17
|
-
"fields": [
|
|
18
|
-
"name",
|
|
19
|
-
"email"
|
|
20
|
-
]
|
|
21
|
-
}
|
|
22
|
-
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/datasets_copy_file_config.json
RENAMED
|
File without changes
|
{okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/datasets_copy_file_config_2.json
RENAMED
|
File without changes
|
{okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/my_permissions_config.json
RENAMED
|
File without changes
|
|
File without changes
|
{okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/pubs_audit_log_config.json
RENAMED
|
File without changes
|
{okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/pubs_clients_config.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/status_history_config.json
RENAMED
|
File without changes
|
{okdata_cli-4.4.0 → okdata_cli-5.0.1}/okdata/cli/data/output-format/team_members_config.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|