cmem-cmemc 24.1.5__py3-none-any.whl → 24.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,12 +4,11 @@ import re
4
4
  import sys
5
5
  import time
6
6
  from datetime import datetime, timezone
7
- from pathlib import Path
8
7
 
9
8
  import click
10
9
  import timeago
11
10
  from click import UsageError
12
- from cmem.cmempy.workflow import get_resource_based_dataset_types, get_workflows
11
+ from cmem.cmempy.workflow import get_workflows
13
12
  from cmem.cmempy.workflow.workflow import (
14
13
  execute_workflow_io,
15
14
  get_workflow_editor_uri,
@@ -30,6 +29,8 @@ from cmem_cmemc import completion
30
29
  from cmem_cmemc.commands import CmemcCommand, CmemcGroup
31
30
  from cmem_cmemc.commands.scheduler import scheduler
32
31
  from cmem_cmemc.context import ApplicationContext
32
+ from cmem_cmemc.parameter_types.path import ClickSmartPath
33
+ from cmem_cmemc.smart_path import SmartPath as Path
33
34
 
34
35
  WORKFLOW_FILTER_TYPES = sorted(["project", "regex", "tag", "io"])
35
36
  WORKFLOW_LIST_FILTER_HELP_TEXT = (
@@ -41,40 +42,41 @@ WORKFLOW_LIST_FILTER_HELP_TEXT = (
41
42
  IO_WARNING_NO_RESULT = "The workflow was executed but produced no result."
42
43
  IO_WARNING_NO_OUTPUT_DEFINED = "The workflow was executed, a result was " "received but dropped."
43
44
 
44
- MIME_CSV = "application/x-plugin-csv"
45
- MIME_XLS = "application/x-plugin-excel"
46
- MIME_NT = "application/n-triples"
47
- MIME_JSON = "application/x-plugin-json"
48
- MIME_XML = "application/xml"
49
- MIME_FILE = "application/octet-stream"
50
- MIME_ZIP = "application/x-plugin-multiCsv"
51
- MIME_ALIGNMENT = "text/alignment"
52
- MIME_TEXT = "text/plain"
53
-
54
- VALID_OUTPUT_EXTENSIONS = {
55
- ".csv": MIME_CSV,
56
- ".xlsx": MIME_XLS,
57
- ".nt": MIME_NT,
58
- ".ttl": MIME_NT,
59
- ".json": MIME_JSON,
60
- ".xml": MIME_XML,
61
- }
62
45
 
63
- VALID_INPUT_EXTENSIONS = {
64
- ".csv": MIME_CSV,
65
- ".json": MIME_JSON,
66
- ".xml": MIME_XML,
67
- ".xlsx": MIME_XLS,
68
- ".file": MIME_FILE,
69
- ".zip": MIME_ZIP,
70
- ".txt": MIME_TEXT,
46
+ FILE_EXTENSIONS_TO_PLUGIN_ID = {
47
+ ".nt": "file",
48
+ ".ttl": "file",
49
+ ".csv": "csv",
50
+ ".json": "json",
51
+ ".xml": "xml",
52
+ ".txt": "text",
53
+ ".xlsx": "excel",
54
+ ".zip": "multiCsv",
71
55
  }
72
56
 
73
-
74
- def is_supported_mime_type(mime_type: str) -> bool:
75
- """Boolean to determine if a request is multipart or not"""
76
- supported_mime_types = get_resource_based_dataset_types()
77
- return any(supported_mime_type in mime_type for supported_mime_type in supported_mime_types)
57
+ # Derive valid extensions from FILE_EXTENSIONS_TO_PLUGIN_ID keys
58
+ VALID_EXTENSIONS = list(FILE_EXTENSIONS_TO_PLUGIN_ID.keys())
59
+ PLUGIN_MIME_TYPES = [f"application/x-plugin-{_}" for _ in FILE_EXTENSIONS_TO_PLUGIN_ID.values()]
60
+ # Define additional mime types for input and output
61
+ EXTRA_INPUT_MIME_TYPES = [
62
+ "application/json",
63
+ "application/xml",
64
+ "text/csv",
65
+ ]
66
+
67
+ EXTRA_OUTPUT_MIME_TYPES = [
68
+ "application/json",
69
+ "application/xml",
70
+ "application/n-triples",
71
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
72
+ "text/csv",
73
+ ]
74
+
75
+ STDOUT_UNSUPPORTED_MIME_TYPES = {
76
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "excel",
77
+ "application/x-plugin-excel": "excel",
78
+ "application/x-plugin-multiCsv": "ZIP",
79
+ }
78
80
 
79
81
 
80
82
  def _get_workflow_tag_labels(workflow_: dict) -> list:
@@ -186,9 +188,10 @@ def _io_check_request(info: dict, input_file: str, output_file: str, output_mime
186
188
  "This workflow has a defined output so you need to use the '-o' "
187
189
  "parameter to retrieve data from it."
188
190
  )
189
- if output_mimetype == MIME_XLS and output_file == "-":
191
+ if output_mimetype in STDOUT_UNSUPPORTED_MIME_TYPES and output_file == "-":
190
192
  raise ValueError(
191
- "Trying to output an excel document to stdout will fail.\n"
193
+ f"Trying to output an {STDOUT_UNSUPPORTED_MIME_TYPES[output_mimetype]} "
194
+ "file to stdout will fail.\n"
192
195
  "Please output to a regular file instead "
193
196
  "(workflow was not executed)."
194
197
  )
@@ -227,11 +230,9 @@ def _io_guess_output(output_file: str) -> str:
227
230
  if output_file == "-":
228
231
  raise ValueError("Output mime-type not guessable, please use the --output-mimetype option.")
229
232
  file_extension = Path(output_file).suffix
230
- if file_extension in VALID_OUTPUT_EXTENSIONS and is_supported_mime_type(
231
- VALID_OUTPUT_EXTENSIONS[file_extension]
232
- ):
233
- return VALID_OUTPUT_EXTENSIONS[file_extension]
234
- valid_extensions = ", ".join(VALID_OUTPUT_EXTENSIONS.keys())
233
+ if file_extension in VALID_EXTENSIONS:
234
+ return f"application/x-plugin-{FILE_EXTENSIONS_TO_PLUGIN_ID[file_extension]}"
235
+ valid_extensions = ", ".join(VALID_EXTENSIONS)
235
236
  raise ValueError(
236
237
  f"Files with the extension {file_extension} can not be generated. "
237
238
  f"Try one of {valid_extensions}"
@@ -243,11 +244,9 @@ def _io_guess_input(input_file: str) -> str:
243
244
  if input_file == "-":
244
245
  raise ValueError("Input mime-type not guessable, please use the --output-mimetype option.")
245
246
  file_extension = Path(input_file).suffix
246
- if file_extension in VALID_INPUT_EXTENSIONS and is_supported_mime_type(
247
- VALID_INPUT_EXTENSIONS[file_extension]
248
- ):
249
- return VALID_INPUT_EXTENSIONS[file_extension]
250
- valid_extensions = ", ".join(VALID_INPUT_EXTENSIONS.keys())
247
+ if file_extension in VALID_EXTENSIONS:
248
+ return f"application/x-plugin-{FILE_EXTENSIONS_TO_PLUGIN_ID[file_extension]}"
249
+ valid_extensions = ", ".join(VALID_EXTENSIONS)
251
250
  raise ValueError(
252
251
  f"Files with the extension {file_extension} can not be processed. "
253
252
  f"Try one of {valid_extensions}"
@@ -441,7 +440,7 @@ def execute_command( # noqa: PLR0913
441
440
  "--input",
442
441
  "-i",
443
442
  "input_file",
444
- type=click.Path(allow_dash=False, dir_okay=False, readable=True),
443
+ type=ClickSmartPath(allow_dash=False, dir_okay=False, readable=True),
445
444
  shell_complete=completion.workflow_io_input_files,
446
445
  help="From which file the input is taken. If the workflow "
447
446
  "has no defined variable input dataset, this option is not allowed.",
@@ -450,7 +449,7 @@ def execute_command( # noqa: PLR0913
450
449
  "--output",
451
450
  "-o",
452
451
  "output_file",
453
- type=click.Path(
452
+ type=ClickSmartPath(
454
453
  allow_dash=False,
455
454
  dir_okay=False,
456
455
  writable=True,
@@ -466,7 +465,13 @@ def execute_command( # noqa: PLR0913
466
465
  help="Which input format should be processed: If not given, cmemc will "
467
466
  "try to guess the mime type based on the file extension or will "
468
467
  "fail.",
469
- type=click.Choice([*list(VALID_INPUT_EXTENSIONS.values()), "guess"]),
468
+ type=click.Choice(
469
+ [
470
+ *PLUGIN_MIME_TYPES,
471
+ *EXTRA_INPUT_MIME_TYPES,
472
+ "guess",
473
+ ]
474
+ ),
470
475
  default="guess",
471
476
  )
472
477
  @click.option(
@@ -475,7 +480,13 @@ def execute_command( # noqa: PLR0913
475
480
  "try to guess the mime type based on the file extension or will "
476
481
  "fail. In case of an output to stdout, a default mime type "
477
482
  "will be used (JSON).",
478
- type=click.Choice([*list(VALID_OUTPUT_EXTENSIONS.values()), "guess"]),
483
+ type=click.Choice(
484
+ [
485
+ *PLUGIN_MIME_TYPES,
486
+ *EXTRA_OUTPUT_MIME_TYPES,
487
+ "guess",
488
+ ]
489
+ ),
479
490
  default="guess",
480
491
  )
481
492
  @click.option(
@@ -499,10 +510,19 @@ def io_command( # noqa: PLR0913
499
510
  ) -> None:
500
511
  """Execute a workflow with file input/output.
501
512
 
502
- With this command, you can execute a workflow that uses variable datasets
513
+ With this command, you can execute a workflow that uses replaceable datasets
503
514
  as input, output or for configuration. Use the input parameter to feed
504
515
  data into the workflow. Likewise, use output for retrieval of the workflow
505
- result. Workflows without a variable dataset will throw an error.
516
+ result. Workflows without a replaceable dataset will throw an error.
517
+
518
+ Note: Regarding the input dataset configuration - the following rules apply:
519
+ If autoconfig is enabled ('--autoconfig', the default), the dataset
520
+ configuration is guessed.
521
+ If autoconfig is disabled ('--no-autoconfig') and the type of the dataset
522
+ file is the same as the replaceable dataset in the workflow, the configuration
523
+ from this dataset is copied.
524
+ If autoconfig is disabled and the type of the dataset file is different from the
525
+ replaceable dataset in the workflow, the default config is used.
506
526
  """
507
527
  project_id, task_id = workflow_id.split(":")
508
528
  if output_file and output_mimetype == "guess":
@@ -526,7 +546,6 @@ def io_command( # noqa: PLR0913
526
546
  f"output_mime_type={output_mimetype}, "
527
547
  f"auto_config={autoconfig}"
528
548
  )
529
-
530
549
  response = execute_workflow_io(
531
550
  project_name=project_id,
532
551
  task_name=task_id,
@@ -1,7 +1,6 @@
1
1
  """workspace commands for cmem command line interface."""
2
2
 
3
3
  import os
4
- from pathlib import Path
5
4
 
6
5
  import click
7
6
  from cmem.cmempy.workspace import reload_workspace
@@ -13,6 +12,8 @@ from cmem_cmemc import completion
13
12
  from cmem_cmemc.commands import CmemcCommand, CmemcGroup
14
13
  from cmem_cmemc.commands.python import python
15
14
  from cmem_cmemc.context import ApplicationContext
15
+ from cmem_cmemc.parameter_types.path import ClickSmartPath
16
+ from cmem_cmemc.smart_path import SmartPath as Path
16
17
 
17
18
 
18
19
  @click.command(cls=CmemcCommand, name="export")
@@ -50,7 +51,7 @@ from cmem_cmemc.context import ApplicationContext
50
51
  "file",
51
52
  shell_complete=completion.workspace_files,
52
53
  required=False,
53
- type=click.Path(writable=True, allow_dash=False, dir_okay=False),
54
+ type=ClickSmartPath(writable=True, allow_dash=False, dir_okay=False),
54
55
  )
55
56
  @click.pass_obj
56
57
  def export_command(
@@ -96,7 +97,7 @@ def export_command(
96
97
  @click.argument(
97
98
  "file",
98
99
  shell_complete=completion.workspace_files,
99
- type=click.Path(readable=True, allow_dash=False, dir_okay=False),
100
+ type=ClickSmartPath(readable=True, allow_dash=False, dir_okay=False),
100
101
  )
101
102
  @click.pass_obj
102
103
  def import_command(app: ApplicationContext, file: str, marshalling_plugin: str) -> None:
cmem_cmemc/completion.py CHANGED
@@ -2,8 +2,8 @@
2
2
 
3
3
  # ruff: noqa: ARG001
4
4
  import os
5
+ import pathlib
5
6
  from contextlib import suppress
6
- from pathlib import Path
7
7
  from typing import Any
8
8
 
9
9
  import requests.exceptions
@@ -17,12 +17,13 @@ from cmem.cmempy.dp.authorization.conditions import (
17
17
  get_groups,
18
18
  get_users,
19
19
  )
20
+ from cmem.cmempy.dp.proxy.graph import get_graph_import_tree
20
21
  from cmem.cmempy.health import get_complete_status_info
21
22
  from cmem.cmempy.keycloak.client import list_open_id_clients
22
23
  from cmem.cmempy.keycloak.group import list_groups
23
24
  from cmem.cmempy.keycloak.user import get_user_by_username, list_users, user_groups
24
25
  from cmem.cmempy.plugins.marshalling import get_marshalling_plugins
25
- from cmem.cmempy.queries import QUERY_CATALOG
26
+ from cmem.cmempy.queries import QueryCatalog
26
27
  from cmem.cmempy.vocabularies import get_vocabularies
27
28
  from cmem.cmempy.workflow.workflows import get_workflows_io
28
29
  from cmem.cmempy.workspace import (
@@ -38,8 +39,9 @@ from cmem.cmempy.workspace.search import list_items
38
39
  from natsort import natsorted, ns
39
40
  from prometheus_client.parser import text_string_to_metric_families
40
41
 
41
- from cmem_cmemc.constants import NS_ACL, NS_USER
42
+ from cmem_cmemc.constants import NS_ACL, NS_GROUP, NS_USER
42
43
  from cmem_cmemc.context import CONTEXT
44
+ from cmem_cmemc.smart_path import SmartPath as Path
43
45
  from cmem_cmemc.utils import (
44
46
  convert_iri_to_qname,
45
47
  get_graphs,
@@ -144,9 +146,7 @@ def _check_option_in_params(option: str, params: Any) -> bool: # noqa: ANN401
144
146
  """Check if the given 'option' is present in the 'params' dictionary or any of its values."""
145
147
  if hasattr(params, "__iter__") and option in params:
146
148
  return True
147
- if option == params:
148
- return True
149
- return False
149
+ return bool(option == params)
150
150
 
151
151
 
152
152
  def add_metadata_parameter(list_: list | None = None) -> list:
@@ -218,6 +218,7 @@ def acl_groups(ctx: Context, param: Argument, incomplete: str) -> list[Completio
218
218
  pass
219
219
  results = get_groups().json()
220
220
  for _ in results:
221
+ _ = _.replace(NS_GROUP, "") if _.startswith(NS_GROUP) else _
221
222
  if _check_option_in_params(_, ctx.params.get(param.name)) or _ in options: # type: ignore[attr-defined]
222
223
  continue
223
224
  options.append(_)
@@ -446,7 +447,7 @@ def file_list(
446
447
  incomplete: str = "", suffix: str = "", description: str = "", prefix: str = ""
447
448
  ) -> list[CompletionItem]:
448
449
  """Prepare a list of files with specific parameter."""
449
- directory = str(Path().cwd())
450
+ directory = str(pathlib.Path().cwd())
450
451
  options = [
451
452
  (file_name, description)
452
453
  for file_name in os.listdir(directory)
@@ -619,7 +620,7 @@ def placeholder(ctx: Context, param: Argument, incomplete: str) -> list[Completi
619
620
  # extract placeholder from given queries in the command line
620
621
  options = []
621
622
  for _, arg in enumerate(args):
622
- query = QUERY_CATALOG.get_query(arg)
623
+ query = QueryCatalog().get_query(arg)
623
624
  if query is not None:
624
625
  options.extend(list(query.get_placeholder_keys()))
625
626
  # look for already given parameter in the arguments and remove them from
@@ -634,7 +635,7 @@ def remote_queries(ctx: Context, param: Argument, incomplete: str) -> list[Compl
634
635
  """Prepare a list of query URIs."""
635
636
  CONTEXT.set_connection_from_params(ctx.find_root().params)
636
637
  options = []
637
- for query in QUERY_CATALOG.get_queries().values():
638
+ for query in QueryCatalog().get_queries().values():
638
639
  url = query.short_url
639
640
  label = query.label
640
641
  options.append((url, label))
@@ -694,20 +695,38 @@ def project_ids(ctx: Context, param: Argument, incomplete: str) -> list[Completi
694
695
  return _finalize_completion(candidates=options, incomplete=incomplete, sort_by=SORT_BY_DESC)
695
696
 
696
697
 
697
- def graph_uris(
698
+ def _prepare_graph_options(
698
699
  ctx: Context, param: Argument, incomplete: str, writeable: bool = True, readonly: bool = True
699
- ) -> list[CompletionItem]:
700
- """Prepare a list of graphs for auto-completion."""
700
+ ) -> list[tuple[str, str]]:
701
+ """Prepare a list of graphs with iri and label"""
701
702
  CONTEXT.set_connection_from_params(ctx.find_root().params)
702
- graphs = get_graphs()
703
+ graphs = get_graphs(writeable=writeable, readonly=readonly)
703
704
  options = []
704
- for _ in graphs:
705
- iri = _["iri"]
706
- label = _["label"]["title"]
705
+ for graph in graphs:
706
+ iri = graph["iri"]
707
+ label = graph["label"]["title"]
707
708
  # do not add graph if already in the command line
708
709
  if _check_option_in_params(iri, ctx.params.get(param.name)): # type: ignore[attr-defined]
709
710
  continue
710
711
  options.append((iri, label))
712
+ return options
713
+
714
+
715
+ def graph_uris(
716
+ ctx: Context, param: Argument, incomplete: str, writeable: bool = True, readonly: bool = True
717
+ ) -> list[CompletionItem]:
718
+ """Prepare a list of graphs for auto-completion."""
719
+ options = _prepare_graph_options(ctx, param, incomplete, writeable=writeable, readonly=readonly)
720
+ return _finalize_completion(candidates=options, incomplete=incomplete, sort_by=SORT_BY_DESC)
721
+
722
+
723
+ def ignore_graph_uris(ctx: Context, param: Argument, incomplete: str) -> list[CompletionItem]:
724
+ """Prepare a list of import graphs for auto-completion."""
725
+ data_graph = ctx.args[0]
726
+ import_tree = get_graph_import_tree(data_graph)
727
+ imported_graphs = {iri for values in import_tree["tree"].values() for iri in values}
728
+ options = _prepare_graph_options(ctx, param, incomplete, writeable=True, readonly=True)
729
+ options = [_ for _ in options if _[0] in imported_graphs]
711
730
  return _finalize_completion(candidates=options, incomplete=incomplete, sort_by=SORT_BY_DESC)
712
731
 
713
732
 
cmem_cmemc/context.py CHANGED
@@ -26,9 +26,9 @@ from urllib3.exceptions import InsecureRequestWarning
26
26
 
27
27
  from cmem_cmemc.exceptions import InvalidConfigurationError
28
28
 
29
- DI_TARGET_VERSION = "v24.1.0"
29
+ DI_TARGET_VERSION = "v24.2.0"
30
30
 
31
- DP_TARGET_VERSION = "v24.1.0"
31
+ DP_TARGET_VERSION = "v24.2.0"
32
32
 
33
33
  KNOWN_CONFIG_KEYS = {
34
34
  "CMEM_BASE_URI": cmempy_config.get_cmem_base_uri,
@@ -247,9 +247,7 @@ class ApplicationContext:
247
247
  cmemc_complete = os.getenv("_CMEMC_COMPLETE", default=None)
248
248
  if comp_words is not None:
249
249
  return True
250
- if cmemc_complete is not None:
251
- return True
252
- return False
250
+ return cmemc_complete is not None
253
251
 
254
252
  @staticmethod
255
253
  def echo_warning(message: str, nl: bool = True) -> None:
@@ -259,17 +257,31 @@ class ApplicationContext:
259
257
  click.secho(message, fg="yellow", err=True, nl=nl)
260
258
 
261
259
  @staticmethod
262
- def echo_error(message: str, nl: bool = True, err: bool = True) -> None:
263
- """Output an error message."""
260
+ def echo_error(
261
+ message: str | list[str], nl: bool = True, err: bool = True, prepend_line: bool = False
262
+ ) -> None:
263
+ """Output an error message.
264
+
265
+ 2024-05-17: also allows list of strings now
266
+ 2024-05-17: new prepend_line parameter
267
+ """
264
268
  # pylint: disable=invalid-name
265
- click.secho(message, fg="red", err=err, nl=nl)
269
+ click.echo("") if prepend_line is True else None
270
+ messages: list[str] = [message] if isinstance(message, str) else message
271
+ for _ in messages:
272
+ click.secho(_, fg="red", err=err, nl=nl)
273
+
274
+ def echo_debug(self, message: str | list[str]) -> None:
275
+ """Output a debug message if --debug is enabled.
266
276
 
267
- def echo_debug(self, message: str) -> None:
268
- """Output a debug message if --debug is enabled."""
277
+ 2024-05-17: also allows list of strings now
278
+ """
269
279
  # pylint: disable=invalid-name
280
+ messages: list[str] = [message] if isinstance(message, str) else message
270
281
  if self.debug:
271
282
  now = datetime.now(tz=timezone.utc)
272
- click.secho(f"[{now!s}] {message}", err=True, dim=True)
283
+ for _ in messages:
284
+ click.secho(f"[{now!s}] {_}", err=True, dim=True)
273
285
 
274
286
  def echo_info(self, message: str | list[str] | set[str], nl: bool = True, fg: str = "") -> None:
275
287
  """Output one or more info messages, if not suppressed by --quiet."""
@@ -293,6 +305,16 @@ class ApplicationContext:
293
305
  )
294
306
  self.echo_info(message)
295
307
 
308
+ def echo_info_xml(self, document: str) -> None:
309
+ """Output a formatted and highlighted XML as info message."""
310
+ # pylint: disable=invalid-name
311
+ message = highlight(
312
+ document,
313
+ get_lexer_by_name("xml"),
314
+ get_formatter_by_name("terminal"),
315
+ )
316
+ self.echo_info(message)
317
+
296
318
  def echo_info_table(
297
319
  self,
298
320
  rows: list,
@@ -425,8 +447,8 @@ class ApplicationContext:
425
447
  )
426
448
  self.echo_debug(f"External credential process started {checked_command}")
427
449
  split_output = (
428
- subprocess.run( # nosec
429
- checked_command, # noqa: S603
450
+ subprocess.run( # nosec # noqa: S603
451
+ checked_command,
430
452
  capture_output=True,
431
453
  check=True,
432
454
  )
cmem_cmemc/object_list.py CHANGED
@@ -68,9 +68,7 @@ def compare_regex(ctx: Filter, object_value: str, filter_value: str) -> bool:
68
68
  f"Invalid filter value '{filter_value}' - "
69
69
  f"need a valid regular expression for filter '{ctx.name}'."
70
70
  ) from error
71
- if re.search(pattern, object_value):
72
- return True
73
- return False
71
+ return bool(re.search(pattern, object_value))
74
72
 
75
73
 
76
74
  def transform_none(ctx: Filter, value: str) -> str: # noqa: ARG001
@@ -164,13 +162,9 @@ class DirectValuePropertyFilter(Filter):
164
162
  if self.property_key not in object_ or object_[self.property_key] is None:
165
163
  if self.default_value is None:
166
164
  return False
167
- if self.compare(self, self.default_value, filter_value):
168
- return True
169
- return False
165
+ return bool(self.compare(self, self.default_value, filter_value))
170
166
  object_value = self.transform(self, str(object_[self.property_key]))
171
- if self.compare(self, object_value, filter_value):
172
- return True
173
- return False
167
+ return bool(self.compare(self, object_value, filter_value))
174
168
 
175
169
  def complete_values(self, objects: list[dict], incomplete: str) -> list[CompletionItem]:
176
170
  """Provide completion items for filter values"""
@@ -219,9 +213,7 @@ class DirectListPropertyFilter(Filter):
219
213
  return False # key value is None
220
214
  if not isinstance(object_[self.property_key], list):
221
215
  return False # key value is not a list
222
- if value in [str(_) for _ in object_[self.property_key]]:
223
- return True
224
- return False
216
+ return value in [str(_) for _ in object_[self.property_key]]
225
217
 
226
218
  def complete_values(self, objects: list[dict], incomplete: str) -> list[CompletionItem]:
227
219
  """Provide completion items for filter values"""
@@ -0,0 +1 @@
1
+ """cmemc custom parameter types."""
@@ -0,0 +1,63 @@
1
+ """Custom Click smart_path ParamType"""
2
+
3
+ import os
4
+ from typing import IO, Any
5
+
6
+ import click
7
+ import smart_open
8
+ from click.core import Context, Parameter
9
+
10
+
11
+ class ClickSmartPath(click.Path):
12
+ """Custom Click smart_path ParamType"""
13
+
14
+ name = "click-smart-path"
15
+
16
+ def __init__( # noqa: PLR0913
17
+ self,
18
+ exists: bool = False,
19
+ file_okay: bool = True,
20
+ dir_okay: bool = True,
21
+ writable: bool = False,
22
+ readable: bool = True,
23
+ resolve_path: bool = False,
24
+ allow_dash: bool = False,
25
+ remote_okay: bool = False,
26
+ ):
27
+ super().__init__(
28
+ exists=exists,
29
+ file_okay=file_okay,
30
+ dir_okay=dir_okay,
31
+ writable=writable,
32
+ readable=readable,
33
+ resolve_path=resolve_path,
34
+ allow_dash=allow_dash,
35
+ )
36
+ self.remote_okay = remote_okay
37
+
38
+ def convert(
39
+ self,
40
+ value: str | os.PathLike[str],
41
+ param: Parameter | None,
42
+ ctx: Context | None,
43
+ ) -> str | bytes | os.PathLike[str]:
44
+ """Convert the given value"""
45
+ try:
46
+ parsed_path = smart_open.parse_uri(value)
47
+ except NotImplementedError as exe:
48
+ self.fail(f"{exe}", param, ctx)
49
+ if parsed_path.scheme == "file":
50
+ return super().convert(parsed_path.uri_path, param, ctx)
51
+ if not self.remote_okay:
52
+ self.fail("Remote path not supported", param, ctx)
53
+
54
+ return value
55
+
56
+ @staticmethod
57
+ def open(
58
+ file_path: str, mode: str = "rb", transport_params: dict[str, Any] | None = None
59
+ ) -> IO:
60
+ """Open the file and return the file handle."""
61
+ if file_path == "-":
62
+ return click.open_file(file_path, mode=mode)
63
+ return smart_open.open(file_path, mode, transport_params=transport_params) # type: ignore[no-any-return]
@@ -0,0 +1,94 @@
1
+ """Provides client classes for interacting with different storage systems."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import urllib.parse
7
+ from pathlib import Path
8
+ from typing import IO, TYPE_CHECKING, ClassVar
9
+
10
+ import smart_open
11
+
12
+ from cmem_cmemc.smart_path.clients.http import HttpPath
13
+
14
+ if TYPE_CHECKING:
15
+ from collections.abc import Generator
16
+
17
+ from cmem_cmemc.smart_path.clients import StoragePath
18
+
19
+
20
+ class SmartPath:
21
+ """Smart path"""
22
+
23
+ SUPPORTED_SCHEMAS: ClassVar = {
24
+ "file": Path,
25
+ "http": HttpPath,
26
+ "https": HttpPath,
27
+ }
28
+
29
+ def __init__(self, path: str):
30
+ self.path = path
31
+ self.schema = self._sniff_schema(self.path)
32
+ if self.schema not in self.SUPPORTED_SCHEMAS:
33
+ raise NotImplementedError(f"Schema '{self.schema}' not supported")
34
+ self._client: StoragePath = self.SUPPORTED_SCHEMAS.get(self.schema)(self.path)
35
+
36
+ @staticmethod
37
+ def _sniff_schema(path: str) -> str:
38
+ """Return the scheme of the URL only, as a string."""
39
+ #
40
+ # urlsplit doesn't work on Windows -- it parses the drive as the scheme...
41
+ # no protocol given => assume a local file
42
+ #
43
+ if os.name == "nt" and "://" not in path:
44
+ path = "file://" + path
45
+ schema = urllib.parse.urlsplit(path).scheme
46
+ return schema if schema else "file"
47
+
48
+ def is_dir(self) -> bool:
49
+ """Determine if path is a directory or not."""
50
+ return self._client.is_dir()
51
+
52
+ def is_file(self) -> bool:
53
+ """Return the suffix of the path."""
54
+ return self._client.is_file()
55
+
56
+ def exists(self) -> bool:
57
+ """Determine if path exists or not."""
58
+ return self._client.exists()
59
+
60
+ @property
61
+ def suffix(self) -> str:
62
+ """Return the suffix of the path."""
63
+ return self._client.suffix
64
+
65
+ @property
66
+ def parent(self) -> StoragePath:
67
+ """The logical parent of the path."""
68
+ return self._client.parent
69
+
70
+ @property
71
+ def name(self) -> str:
72
+ """Determine the name of the path."""
73
+ return self._client.name
74
+
75
+ def open(self, mode: str = "r", encoding: str | None = None) -> IO:
76
+ """Open the file pointed by this path."""
77
+ file: IO = smart_open.open(self.path, mode=mode, encoding=encoding)
78
+ return file
79
+
80
+ def mkdir(self, parents: bool = False, exist_ok: bool = False) -> None:
81
+ """Return the suffix of the path."""
82
+ self._client.mkdir(parents=parents, exist_ok=exist_ok)
83
+
84
+ def glob(self, pattern: str) -> Generator[StoragePath, StoragePath, StoragePath]:
85
+ """Iterate over this subtree and yield all existing files"""
86
+ return self._client.glob(pattern=pattern)
87
+
88
+ def resolve(self) -> StoragePath:
89
+ """Iterate over this subtree and yield all existing files"""
90
+ return self._client.resolve()
91
+
92
+ def __truediv__(self, key: str) -> StoragePath:
93
+ """Return StoragePath with appending the key to the exising path"""
94
+ return self._client.__truediv__(key)