cmem-cmemc 24.2.0rc1__py3-none-any.whl → 24.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. cmem_cmemc/__init__.py +7 -12
  2. cmem_cmemc/command.py +20 -0
  3. cmem_cmemc/command_group.py +70 -0
  4. cmem_cmemc/commands/__init__.py +0 -81
  5. cmem_cmemc/commands/acl.py +118 -62
  6. cmem_cmemc/commands/admin.py +46 -35
  7. cmem_cmemc/commands/client.py +2 -1
  8. cmem_cmemc/commands/config.py +3 -1
  9. cmem_cmemc/commands/dataset.py +27 -24
  10. cmem_cmemc/commands/graph.py +160 -19
  11. cmem_cmemc/commands/metrics.py +195 -79
  12. cmem_cmemc/commands/migration.py +267 -0
  13. cmem_cmemc/commands/project.py +62 -17
  14. cmem_cmemc/commands/python.py +56 -25
  15. cmem_cmemc/commands/query.py +23 -14
  16. cmem_cmemc/commands/resource.py +10 -2
  17. cmem_cmemc/commands/scheduler.py +10 -2
  18. cmem_cmemc/commands/store.py +118 -14
  19. cmem_cmemc/commands/user.py +8 -2
  20. cmem_cmemc/commands/validation.py +304 -113
  21. cmem_cmemc/commands/variable.py +10 -2
  22. cmem_cmemc/commands/vocabulary.py +48 -29
  23. cmem_cmemc/commands/workflow.py +86 -59
  24. cmem_cmemc/commands/workspace.py +27 -8
  25. cmem_cmemc/completion.py +190 -140
  26. cmem_cmemc/constants.py +2 -0
  27. cmem_cmemc/context.py +88 -42
  28. cmem_cmemc/manual_helper/graph.py +1 -0
  29. cmem_cmemc/manual_helper/multi_page.py +3 -1
  30. cmem_cmemc/migrations/__init__.py +1 -0
  31. cmem_cmemc/migrations/abc.py +84 -0
  32. cmem_cmemc/migrations/access_conditions_243.py +122 -0
  33. cmem_cmemc/migrations/bootstrap_data.py +28 -0
  34. cmem_cmemc/migrations/shapes_widget_integrations_243.py +274 -0
  35. cmem_cmemc/migrations/workspace_configurations.py +28 -0
  36. cmem_cmemc/object_list.py +53 -22
  37. cmem_cmemc/parameter_types/__init__.py +1 -0
  38. cmem_cmemc/parameter_types/path.py +69 -0
  39. cmem_cmemc/smart_path/__init__.py +94 -0
  40. cmem_cmemc/smart_path/clients/__init__.py +63 -0
  41. cmem_cmemc/smart_path/clients/http.py +65 -0
  42. cmem_cmemc/string_processor.py +83 -0
  43. cmem_cmemc/title_helper.py +41 -0
  44. cmem_cmemc/utils.py +100 -45
  45. {cmem_cmemc-24.2.0rc1.dist-info → cmem_cmemc-24.3.0.dist-info}/LICENSE +1 -1
  46. cmem_cmemc-24.3.0.dist-info/METADATA +89 -0
  47. cmem_cmemc-24.3.0.dist-info/RECORD +53 -0
  48. {cmem_cmemc-24.2.0rc1.dist-info → cmem_cmemc-24.3.0.dist-info}/WHEEL +1 -1
  49. cmem_cmemc-24.2.0rc1.dist-info/METADATA +0 -69
  50. cmem_cmemc-24.2.0rc1.dist-info/RECORD +0 -37
  51. {cmem_cmemc-24.2.0rc1.dist-info → cmem_cmemc-24.3.0.dist-info}/entry_points.txt +0 -0
@@ -1,5 +1,6 @@
1
1
  """admin commands for cmem command line interface."""
2
- from datetime import UTC, datetime
2
+
3
+ from datetime import datetime, timezone
3
4
 
4
5
  import click
5
6
  import jwt
@@ -9,23 +10,34 @@ from cmem.cmempy.health import get_complete_status_info
9
10
  from dateutil.relativedelta import relativedelta
10
11
 
11
12
  from cmem_cmemc import completion
12
- from cmem_cmemc.commands import CmemcCommand, CmemcGroup
13
+ from cmem_cmemc.command import CmemcCommand
14
+ from cmem_cmemc.command_group import CmemcGroup
13
15
  from cmem_cmemc.commands.acl import acl
14
16
  from cmem_cmemc.commands.client import client
15
17
  from cmem_cmemc.commands.metrics import metrics
18
+ from cmem_cmemc.commands.migration import migration
16
19
  from cmem_cmemc.commands.store import store
17
20
  from cmem_cmemc.commands.user import user
18
21
  from cmem_cmemc.commands.workspace import workspace
19
22
  from cmem_cmemc.context import ApplicationContext
20
23
  from cmem_cmemc.utils import struct_to_table
21
24
 
25
+ WARNING_MIGRATION = (
26
+ "Your workspace configuration version does not match your DataPlatform version. "
27
+ "Please consider migrating your workspace configuration (admin store migrate command)."
28
+ )
29
+ WARNING_SHAPES = (
30
+ "Your ShapeCatalog version does not match your DataPlatform version. "
31
+ "Please consider updating your bootstrap data (admin store boostrap command)."
32
+ )
33
+
22
34
 
23
35
  def _check_cmem_license(app: ApplicationContext, data: dict, exit_1: str) -> None:
24
36
  """Check grace period of CMEM license."""
25
- if "license" not in data["dp"]["info"]:
37
+ if "license" not in data["explore"]["info"]:
26
38
  # DP < 24.1 has no cmem license information here
27
39
  return
28
- license_ = data["dp"]["info"]["license"]
40
+ license_ = data["explore"]["info"]["license"]
29
41
  in_grace_period: bool = license_.get("inGracePeriod", False)
30
42
  if in_grace_period:
31
43
  cmem_license_end = license_["validDate"]
@@ -37,14 +49,14 @@ def _check_cmem_license(app: ApplicationContext, data: dict, exit_1: str) -> Non
37
49
 
38
50
  def _check_graphdb_license(app: ApplicationContext, data: dict, months: int, exit_1: str) -> None:
39
51
  """Check grace period of graphdb license."""
40
- if "licenseExpiration" not in data["dp"]["info"]["store"]:
52
+ if "licenseExpiration" not in data["explore"]["info"]["store"]:
41
53
  # DP < 24.1 has no graph license information here
42
54
  return
43
- expiration_date_str = data["dp"]["info"]["store"]["licenseExpiration"]
44
- expiration_date = datetime.strptime(expiration_date_str, "%Y-%m-%d").astimezone(tz=UTC)
55
+ expiration_date_str = data["explore"]["info"]["store"]["licenseExpiration"]
56
+ expiration_date = datetime.strptime(expiration_date_str, "%Y-%m-%d").astimezone(tz=timezone.utc)
45
57
  grace_starts = expiration_date - relativedelta(months=months)
46
- if grace_starts < datetime.now(tz=UTC):
47
- graphdb_license_end = data["dp"]["info"]["store"]["licenseExpiration"]
58
+ if grace_starts < datetime.now(tz=timezone.utc):
59
+ graphdb_license_end = data["explore"]["info"]["store"]["licenseExpiration"]
48
60
  output = f"Your GraphDB license expires on {graphdb_license_end}."
49
61
  if exit_1 == "always":
50
62
  raise ValueError(output)
@@ -83,7 +95,7 @@ def _check_graphdb_license(app: ApplicationContext, data: dict, months: int, exi
83
95
  "--raw", is_flag=True, help="Outputs combined raw JSON output of the health/info endpoints."
84
96
  )
85
97
  @click.pass_obj
86
- def status_command( # noqa: C901
98
+ def status_command( # noqa: C901, PLR0912
87
99
  app: ApplicationContext, key: str, exit_1: str, enforce_table: bool, raw: bool
88
100
  ) -> None:
89
101
  """Output health and version information.
@@ -105,20 +117,12 @@ def status_command( # noqa: C901
105
117
  Example: cmemc config list | parallel --ctag cmemc -c {} admin status
106
118
  """
107
119
  _ = get_complete_status_info()
108
- if "error" in _["di"]:
109
- app.echo_debug(_["di"]["error"])
110
- if "error" in _["dp"]:
111
- app.echo_debug(_["dp"]["error"])
112
- if "error" in _["dm"]:
113
- app.echo_debug(_["dm"]["error"])
114
- basic_status = (
115
- _["dp"]["healthy"],
116
- _["di"]["healthy"],
117
- _["dm"]["healthy"],
118
- _["shapes"]["healthy"],
119
- _["store"]["healthy"],
120
- )
121
- if exit_1 in ("always", "error") and ("DOWN" in basic_status or "UNKNOWN" in basic_status):
120
+ if "error" in _["build"]:
121
+ app.echo_debug(_["build"]["error"])
122
+ if "error" in _["explore"]:
123
+ app.echo_debug(_["explore"]["error"])
124
+
125
+ if exit_1 in ("always", "error") and (_["overall"]["healthy"] != "UP"):
122
126
  raise ValueError(
123
127
  f"One or more major status flags are DOWN or UNKNOWN: {_!r}",
124
128
  )
@@ -135,22 +139,28 @@ def status_command( # noqa: C901
135
139
  app.echo_info_table(table, headers=["Key", "Value"], sort_column=0)
136
140
  return
137
141
  app.check_versions()
138
- if _["shapes"]["version"] not in (_["dp"]["version"], "UNKNOWN"):
139
- output = (
140
- "Your ShapeCatalog version does not match your DataPlatform "
141
- "version. Please consider updating your bootstrap data."
142
- )
142
+ _workspace_config = _["explore"]["info"].get("workspaceConfiguration", {})
143
+ if _workspace_config.get("workspacesToMigrate"):
143
144
  if exit_1 == "always":
144
- raise ValueError(output)
145
- app.echo_warning(output)
145
+ raise ValueError(WARNING_MIGRATION)
146
+ app.echo_warning(WARNING_MIGRATION)
147
+
148
+ if _["shapes"]["version"] not in (_["explore"]["version"], "UNKNOWN"):
149
+ if exit_1 == "always":
150
+ raise ValueError(WARNING_SHAPES)
151
+ app.echo_warning(WARNING_SHAPES)
152
+
146
153
  _check_cmem_license(app=app, data=_, exit_1=exit_1)
147
154
  _check_graphdb_license(app=app, data=_, months=1, exit_1=exit_1)
155
+ if _["store"]["type"] != "GRAPHDB":
156
+ store_version = _["store"]["type"] + "/" + _["store"]["version"]
157
+ else:
158
+ store_version = _["store"]["version"]
148
159
  table = [
149
- ("DP", _["dp"]["version"], _["dp"]["healthy"]),
150
- ("DI", _["di"]["version"], _["di"]["healthy"]),
151
- ("DM", _["dm"]["version"], _["dm"]["healthy"]),
160
+ ("EXPLORE", _["explore"]["version"], _["explore"]["healthy"]),
161
+ ("BUILD", _["build"]["version"], _["build"]["healthy"]),
152
162
  ("SHAPES", _["shapes"]["version"], _["shapes"]["healthy"]),
153
- (_["store"]["type"], _["store"]["version"], _["store"]["healthy"]),
163
+ ("STORE", store_version, _["store"]["healthy"]),
154
164
  ]
155
165
  app.echo_info_table(
156
166
  table,
@@ -224,3 +234,4 @@ admin.add_command(store)
224
234
  admin.add_command(user)
225
235
  admin.add_command(acl)
226
236
  admin.add_command(client)
237
+ admin.add_command(migration)
@@ -10,7 +10,8 @@ from cmem.cmempy.keycloak.client import (
10
10
  )
11
11
 
12
12
  from cmem_cmemc import completion
13
- from cmem_cmemc.commands import CmemcCommand, CmemcGroup
13
+ from cmem_cmemc.command import CmemcCommand
14
+ from cmem_cmemc.command_group import CmemcGroup
14
15
  from cmem_cmemc.context import ApplicationContext
15
16
 
16
17
  NO_CLIENT_ERROR = (
@@ -1,7 +1,9 @@
1
1
  """configuration commands for cmem command line interface."""
2
+
2
3
  import click
3
4
 
4
- from cmem_cmemc.commands import CmemcCommand, CmemcGroup
5
+ from cmem_cmemc.command import CmemcCommand
6
+ from cmem_cmemc.command_group import CmemcGroup
5
7
  from cmem_cmemc.context import KNOWN_CONFIG_KEYS, ApplicationContext
6
8
 
7
9
 
@@ -1,7 +1,7 @@
1
1
  """dataset commands for cmem command line interface."""
2
+
2
3
  import json
3
4
  import re
4
- from pathlib import Path
5
5
 
6
6
  import click
7
7
  import requests.exceptions
@@ -23,9 +23,13 @@ from cmem.cmempy.workspace.projects.resources.resource import (
23
23
  from cmem.cmempy.workspace.search import list_items
24
24
 
25
25
  from cmem_cmemc import completion
26
- from cmem_cmemc.commands import CmemcCommand, CmemcGroup
26
+ from cmem_cmemc.command import CmemcCommand
27
+ from cmem_cmemc.command_group import CmemcGroup
27
28
  from cmem_cmemc.commands.resource import resource
29
+ from cmem_cmemc.completion import get_dataset_file_mapping
28
30
  from cmem_cmemc.context import ApplicationContext
31
+ from cmem_cmemc.parameter_types.path import ClickSmartPath
32
+ from cmem_cmemc.smart_path import SmartPath as Path
29
33
  from cmem_cmemc.utils import check_or_select_project, struct_to_table
30
34
 
31
35
  DATASET_FILTER_TYPES = sorted(["project", "regex", "tag", "type"])
@@ -137,7 +141,7 @@ def _post_file_resource(
137
141
  post_resource(
138
142
  project_id=project_id,
139
143
  dataset_id=dataset_id,
140
- file_resource=click.open_file(local_file_name, "rb"),
144
+ file_resource=ClickSmartPath.open(local_file_name),
141
145
  )
142
146
  app.echo_success("done")
143
147
 
@@ -187,7 +191,7 @@ def _upload_file_resource(
187
191
  create_resource(
188
192
  project_name=project_id,
189
193
  resource_name=remote_file_name,
190
- file_resource=click.open_file(local_file_name, "rb"),
194
+ file_resource=ClickSmartPath.open(local_file_name),
191
195
  replace=replace,
192
196
  )
193
197
  app.echo_success("done")
@@ -284,23 +288,15 @@ def _check_or_set_dataset_type(
284
288
 
285
289
  """
286
290
  source = Path(dataset_file).name if dataset_file else ""
287
- target = parameter_dict["file"] if "file" in parameter_dict else ""
288
- suggestions = (
289
- (".ttl", "file"),
290
- (".csv", "csv"),
291
- (".xlsx", "excel"),
292
- (".xml", "xml"),
293
- (".json", "json"),
294
- (".jsonl", "json"),
295
- (".orc", "orc"),
296
- (".zip", "multiCsv"),
297
- (".yaml", "text"),
298
- (".yml", "text"),
299
- )
291
+ target = parameter_dict.get("file", "")
292
+ suggestions = [
293
+ (extension, info["type"]) for extension, info in get_dataset_file_mapping().items()
294
+ ]
300
295
  if not dataset_type:
301
296
  for check, type_ in suggestions:
302
297
  if source.endswith(check) or target.endswith(check):
303
298
  dataset_type = type_
299
+ break
304
300
  if not dataset_type:
305
301
  raise ValueError("Missing parameter. Please specify a dataset " "type with '--type'.")
306
302
  app.echo_warning(
@@ -440,7 +436,13 @@ def list_command(
440
436
  _["label"],
441
437
  ]
442
438
  table.append(row)
443
- app.echo_info_table(table, headers=["Dataset ID", "Type", "Label"], sort_column=2)
439
+ app.echo_info_table(
440
+ table,
441
+ headers=["Dataset ID", "Type", "Label"],
442
+ sort_column=2,
443
+ empty_table_message="No datasets found. "
444
+ "Use the `dataset create` command to create a new dataset.",
445
+ )
444
446
 
445
447
 
446
448
  @click.command(cls=CmemcCommand, name="delete")
@@ -525,7 +527,9 @@ def delete_command(
525
527
  @click.command(cls=CmemcCommand, name="download")
526
528
  @click.argument("dataset_id", type=click.STRING, shell_complete=completion.dataset_ids)
527
529
  @click.argument(
528
- "output_path", required=True, type=click.Path(allow_dash=True, dir_okay=False, writable=True)
530
+ "output_path",
531
+ required=True,
532
+ type=ClickSmartPath(allow_dash=True, dir_okay=False, writable=True),
529
533
  )
530
534
  @click.option(
531
535
  "--replace",
@@ -588,7 +592,7 @@ def download_command(
588
592
  "input_path",
589
593
  required=True,
590
594
  shell_complete=completion.dataset_files,
591
- type=click.Path(allow_dash=True, dir_okay=False, writable=True),
595
+ type=ClickSmartPath(allow_dash=True, dir_okay=False, writable=True, remote_okay=True),
592
596
  )
593
597
  @click.pass_obj
594
598
  def upload_command(app: ApplicationContext, dataset_id: str, input_path: str) -> None:
@@ -642,7 +646,7 @@ def inspect_command(app: ApplicationContext, dataset_id: str, raw: bool) -> None
642
646
  "DATASET_FILE",
643
647
  required=False,
644
648
  shell_complete=completion.dataset_files,
645
- type=click.Path(allow_dash=False, readable=True, exists=True),
649
+ type=ClickSmartPath(allow_dash=False, readable=True, exists=True, remote_okay=True),
646
650
  )
647
651
  @click.option(
648
652
  "--type",
@@ -725,9 +729,7 @@ def create_command( # noqa: PLR0913
725
729
  return
726
730
 
727
731
  # transform the parameter list of tuple to a dictionary
728
- parameter_dict = {}
729
- for key, value in parameter:
730
- parameter_dict[key] = value
732
+ parameter_dict = dict(parameter)
731
733
 
732
734
  dataset_type = _check_or_set_dataset_type(
733
735
  app=app,
@@ -762,6 +764,7 @@ def create_command( # noqa: PLR0913
762
764
  # add file parameter for the project if needed
763
765
  if "file" not in parameter_dict:
764
766
  parameter_dict["file"] = Path(dataset_file).name
767
+
765
768
  _upload_file_resource(
766
769
  app=app,
767
770
  project_id=project_id,
@@ -1,14 +1,19 @@
1
1
  """graph commands for cmem command line interface."""
2
+
3
+ import gzip
2
4
  import hashlib
5
+ import io
3
6
  import json
7
+ import mimetypes
4
8
  import os
5
- from pathlib import Path
9
+ from json import JSONDecodeError
6
10
  from xml.dom import minidom # nosec
7
11
  from xml.etree.ElementTree import ( # nosec
8
12
  Element,
9
13
  SubElement,
10
14
  tostring,
11
15
  )
16
+ from xml.sax import SAXParseException
12
17
 
13
18
  import click
14
19
  from click import Argument
@@ -18,13 +23,19 @@ from cmem.cmempy.dp.proxy import graph as graph_api
18
23
  from cmem.cmempy.dp.proxy.graph import get_graph_import_tree, get_graph_imports
19
24
  from cmem.cmempy.dp.proxy.sparql import get as sparql_api
20
25
  from jinja2 import Template
26
+ from rdflib import Graph
27
+ from rdflib.exceptions import ParserError
28
+ from rdflib.plugins.parsers.notation3 import BadSyntax
21
29
  from six.moves.urllib.parse import quote
22
30
  from treelib import Tree
23
31
 
24
32
  from cmem_cmemc import completion
25
- from cmem_cmemc.commands import CmemcCommand, CmemcGroup
33
+ from cmem_cmemc.command import CmemcCommand
34
+ from cmem_cmemc.command_group import CmemcGroup
26
35
  from cmem_cmemc.commands.validation import validation_group
27
36
  from cmem_cmemc.context import ApplicationContext
37
+ from cmem_cmemc.parameter_types.path import ClickSmartPath
38
+ from cmem_cmemc.smart_path import SmartPath as Path
28
39
  from cmem_cmemc.utils import (
29
40
  convert_uri_to_filename,
30
41
  get_graphs,
@@ -77,11 +88,14 @@ def _get_graph_to_file( # noqa: PLR0913
77
88
  nl=False,
78
89
  )
79
90
  # create and write the .ttl content file
80
- if overwrite is True:
81
- triple_file = click.open_file(file_path, "wb")
82
- else:
83
- triple_file = click.open_file(file_path, "ab")
84
- with graph_api.get_streamed(graph_iri, accept=mime_type) as response:
91
+ mode = "wb" if overwrite is True else "ab"
92
+
93
+ with (
94
+ gzip.open(file_path, mode)
95
+ if file_path.endswith(".gz")
96
+ else click.open_file(file_path, mode) as triple_file,
97
+ graph_api.get_streamed(graph_iri, accept=mime_type) as response,
98
+ ):
85
99
  response.raise_for_status()
86
100
  for chunk in response.iter_content(chunk_size=None):
87
101
  if chunk:
@@ -94,7 +108,9 @@ def _get_graph_to_file( # noqa: PLR0913
94
108
  app.echo_success("done")
95
109
 
96
110
 
97
- def _get_export_names(app: ApplicationContext, iris: list[str], template: str) -> dict:
111
+ def _get_export_names(
112
+ app: ApplicationContext, iris: list[str], template: str, file_extension: str = ".ttl"
113
+ ) -> dict:
98
114
  """Get a dictionary of generated file names based on a template.
99
115
 
100
116
  Args:
@@ -102,6 +118,7 @@ def _get_export_names(app: ApplicationContext, iris: list[str], template: str) -
102
118
  app: the context click application
103
119
  iris: list of graph iris
104
120
  template (str): the template string to use
121
+ file_extension(str): the file extension to use
105
122
 
106
123
  Returns:
107
124
  -------
@@ -120,7 +137,7 @@ def _get_export_names(app: ApplicationContext, iris: list[str], template: str) -
120
137
  hash=hashlib.sha256(iri.encode("utf-8")).hexdigest(),
121
138
  iriname=convert_uri_to_filename(iri),
122
139
  )
123
- _name_created = Template(template).render(template_data) + ".ttl"
140
+ _name_created = f"{Template(template).render(template_data)}{file_extension}"
124
141
  _names[iri] = _name_created
125
142
  if len(_names.values()) != len(set(_names.values())):
126
143
  raise ValueError(
@@ -459,7 +476,27 @@ def list_command(
459
476
  _["label"]["title"],
460
477
  ]
461
478
  table.append(row)
462
- app.echo_info_table(table, headers=["Graph IRI", "Type", "Label"], sort_column=2)
479
+ app.echo_info_table(
480
+ table,
481
+ headers=["Graph IRI", "Type", "Label"],
482
+ sort_column=2,
483
+ empty_table_message="No graphs found. "
484
+ "Use the `graph import` command to import a graph from a file, or "
485
+ "use the `admin store bootstrap` command to import the default graphs.",
486
+ )
487
+
488
+
489
+ def _validate_export_command_input_parameters(
490
+ output_dir: str, output_file: str, compress: str, create_catalog: bool
491
+ ) -> None:
492
+ """Validate export command input parameters combinations"""
493
+ if output_dir and create_catalog and compress:
494
+ raise click.UsageError(
495
+ "Cannot create a catalog file when using a compressed graph file."
496
+ " Please remove either the --create-catalog or --compress option."
497
+ )
498
+ if output_file == "- " and compress:
499
+ raise click.UsageError("Cannot output a binary file to terminal. Use --output-file option.")
463
500
 
464
501
 
465
502
  # pylint: disable=too-many-arguments,too-many-locals
@@ -480,12 +517,12 @@ def list_command(
480
517
  )
481
518
  @click.option(
482
519
  "--output-dir",
483
- type=click.Path(writable=True, file_okay=False),
520
+ type=ClickSmartPath(writable=True, file_okay=False),
484
521
  help="Export to this directory.",
485
522
  )
486
523
  @click.option(
487
524
  "--output-file",
488
- type=click.Path(writable=True, allow_dash=True, dir_okay=False),
525
+ type=ClickSmartPath(writable=True, allow_dash=True, dir_okay=False),
489
526
  default="-",
490
527
  show_default=True,
491
528
  shell_complete=completion.triple_files,
@@ -511,11 +548,16 @@ def list_command(
511
548
  )
512
549
  @click.option(
513
550
  "--mime-type",
514
- default="application/n-triples",
551
+ default="text/turtle",
515
552
  show_default=True,
516
- type=click.Choice(["application/n-triples", "text/turtle"]),
553
+ type=click.Choice(["application/n-triples", "text/turtle", "application/rdf+xml"]),
517
554
  help="Define the requested mime type",
518
555
  )
556
+ @click.option(
557
+ "--compress",
558
+ type=click.Choice(["gzip"]),
559
+ help="Compress the exported graph files.",
560
+ )
519
561
  @click.argument(
520
562
  "iris",
521
563
  nargs=-1,
@@ -534,6 +576,7 @@ def export_command( # noqa: PLR0913
534
576
  template: str,
535
577
  mime_type: str,
536
578
  iris: list[str],
579
+ compress: str,
537
580
  ) -> None:
538
581
  """Export graph(s) as NTriples to stdout (-), file or directory.
539
582
 
@@ -542,6 +585,7 @@ def export_command( # noqa: PLR0913
542
585
  In case of directory export, .graph and .ttl files will be created
543
586
  for each graph.
544
587
  """
588
+ _validate_export_command_input_parameters(output_dir, output_file, compress, create_catalog)
545
589
  iris = _check_and_extend_exported_graphs(iris, all_, include_imports, get_graphs_as_dict())
546
590
 
547
591
  count: int = len(iris)
@@ -551,7 +595,11 @@ def export_command( # noqa: PLR0913
551
595
  app.echo_debug("output is directory")
552
596
  # pre-calculate all filenames with the template,
553
597
  # in order to output errors on naming clashes as early as possible
554
- _names = _get_export_names(app, iris, template)
598
+ extension = mimetypes.guess_extension(mime_type)
599
+ _names = _get_export_names(
600
+ app, iris, template, f"{extension}.gz" if compress else f"{extension}"
601
+ )
602
+ _graph_file_names = _get_export_names(app, iris, template, f"{extension}.graph")
555
603
  # create directory
556
604
  if not Path(output_dir).exists():
557
605
  app.echo_warning("Output directory does not exist: " + "will create it.")
@@ -560,7 +608,7 @@ def export_command( # noqa: PLR0913
560
608
  for current, iri in enumerate(iris, start=1):
561
609
  # join with given output directory and normalize full path
562
610
  triple_file_name = os.path.normpath(Path(output_dir) / _names[iri])
563
- graph_file_name = triple_file_name + ".graph"
611
+ graph_file_name = os.path.normpath(Path(output_dir) / _graph_file_names[iri])
564
612
  # output directory is created lazy
565
613
  Path(triple_file_name).parent.mkdir(parents=True, exist_ok=True)
566
614
  # create and write the .ttl.graph metadata file
@@ -574,6 +622,10 @@ def export_command( # noqa: PLR0913
574
622
  return
575
623
  # no output directory set -> file export
576
624
  if output_file == "-":
625
+ if compress:
626
+ raise click.UsageError(
627
+ "Cannot output a binary file to terminal. Use --output-file option."
628
+ )
577
629
  # in case a file is stdout,
578
630
  # all triples from all graphs go in and other output is suppressed
579
631
  app.echo_debug("output is stdout")
@@ -581,6 +633,9 @@ def export_command( # noqa: PLR0913
581
633
  _get_graph_to_file(iri, output_file, app, mime_type=mime_type)
582
634
  else:
583
635
  # in case a file is given, all triples from all graphs go in
636
+ if compress and not output_file.endswith(".gz"):
637
+ output_file = output_file + ".gz"
638
+
584
639
  app.echo_debug("output is file")
585
640
  for current, iri in enumerate(iris, start=1):
586
641
  _get_graph_to_file(
@@ -593,6 +648,79 @@ def export_command( # noqa: PLR0913
593
648
  )
594
649
 
595
650
 
651
+ def validate_input_path(input_path: str) -> None:
652
+ """Validate input path
653
+
654
+ This function checks the provided folder for any .ttl or .nt files
655
+ that have corresponding .gz files. If such files are found, it raises a ValueError.
656
+ """
657
+ files = os.listdir(input_path)
658
+
659
+ # Check for files with the given extensions (.ttl and .nt)
660
+ rdf_files = [f for f in files if f.endswith((".ttl", ".nt"))]
661
+
662
+ # Check for corresponding .gz files
663
+ gz_files = [f"{f}.gz" for f in rdf_files]
664
+ conflicting_files = [f for f in gz_files if f in files]
665
+
666
+ if conflicting_files:
667
+ raise ValueError(
668
+ f"The following RDF files (.ttl/.nt) have corresponding '.gz' files,"
669
+ f" which is not allowed: {', '.join(conflicting_files)}"
670
+ )
671
+
672
+
673
+ def _get_graph_supported_formats() -> dict[str, str]:
674
+ return {
675
+ "application/rdf+xml": "xml",
676
+ "application/ld+json": "json-ld",
677
+ "text/turtle": "turtle",
678
+ "application/n-triples": "nt",
679
+ }
680
+
681
+
682
+ def _guess_rdf_mime_type(content: str) -> str:
683
+ formats = _get_graph_supported_formats()
684
+ for mime_type, rdf_format in formats.items():
685
+ try:
686
+ g = Graph()
687
+ g.parse(data=content, format=rdf_format)
688
+ except (SAXParseException, JSONDecodeError, BadSyntax, ParserError):
689
+ continue
690
+ else:
691
+ return mime_type
692
+ raise ValueError("Unknown format")
693
+
694
+
695
+ def _parse_triple_file(triple_file: str) -> tuple[io.BytesIO, str]:
696
+ """Parse the content of the triple file."""
697
+ buffer = io.BytesIO()
698
+ transport_params = {}
699
+
700
+ if Path(str(triple_file)).schema in ["http", "https"]:
701
+ transport_params["headers"] = {
702
+ "Accept": "text/turtle; q=1.0, application/x-turtle; q=0.9, text/n3;"
703
+ " q=0.8, application/rdf+xml; q=0.5, text/plain; q=0.1"
704
+ }
705
+
706
+ with ClickSmartPath.open(triple_file, transport_params=transport_params) as file_obj:
707
+ buffer.write(file_obj.read())
708
+
709
+ buffer.seek(0)
710
+ is_gzip = buffer.read(2) == b"\x1f\x8b"
711
+ buffer.seek(0)
712
+
713
+ if is_gzip:
714
+ with gzip.GzipFile(fileobj=buffer, mode="rb") as gzip_file:
715
+ graph_content = gzip_file.read().decode("utf-8")
716
+ else:
717
+ graph_content = buffer.read().decode("utf-8")
718
+
719
+ content_type = _guess_rdf_mime_type(graph_content)
720
+ buffer.seek(0)
721
+ return buffer, content_type
722
+
723
+
596
724
  @click.command(cls=CmemcCommand, name="import")
597
725
  @click.option(
598
726
  "--replace",
@@ -611,7 +739,7 @@ def export_command( # noqa: PLR0913
611
739
  "input_path",
612
740
  required=True,
613
741
  shell_complete=completion.triple_files,
614
- type=click.Path(allow_dash=False, readable=True),
742
+ type=ClickSmartPath(allow_dash=False, readable=True, remote_okay=True),
615
743
  )
616
744
  @click.argument("iri", type=click.STRING, required=False, shell_complete=completion.graph_uris)
617
745
  @click.pass_obj
@@ -647,13 +775,19 @@ def import_command(
647
775
  )
648
776
  graphs: list
649
777
  if Path(input_path).is_dir():
778
+ validate_input_path(input_path)
650
779
  if iri is None:
651
780
  # in case a directory is the source (and no IRI is given),
652
781
  # the graph/nt file structure is crawled
653
782
  graphs = read_rdf_graph_files(input_path)
654
783
  else:
655
784
  # in case a directory is the source AND IRI is given
656
- graphs = [(file, iri) for file in Path(input_path).glob("*.ttl")]
785
+ graphs = []
786
+ for _ in _get_graph_supported_formats():
787
+ extension = mimetypes.guess_extension(_)
788
+ graphs += [(file, iri) for file in Path(input_path).glob(f"*{extension}")]
789
+ graphs += [(file, iri) for file in Path(input_path).glob(f"*{extension}.gz")]
790
+
657
791
  elif Path(input_path).is_file():
658
792
  if iri is None:
659
793
  raise ValueError(
@@ -678,7 +812,14 @@ def import_command(
678
812
  continue
679
813
  # prevents re-replacing of graphs in a single run
680
814
  _replace = False if graph_iri in processed_graphs else replace
681
- graph_api.post_streamed(graph_iri, triple_file, replace=_replace)
815
+ _buffer, content_type = _parse_triple_file(triple_file)
816
+ response = graph_api.post_streamed(
817
+ graph_iri, _buffer, replace=_replace, content_type=content_type
818
+ )
819
+ request_headers = response.request.headers
820
+ request_headers.pop("Authorization")
821
+ app.echo_debug(f"cmemc request headers: {request_headers}")
822
+ app.echo_debug(f"server response headers: {response.headers}")
682
823
  app.echo_success("replaced" if _replace else "added")
683
824
  # refresh access conditions in case of dropped AC graph
684
825
  if graph_iri == refresh.AUTHORIZATION_GRAPH_URI: