dump-things-pyclient 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl

dump_things_pyclient/commands/dtc_plugins/auto_curate.py

@@ -19,6 +19,7 @@ from rich.progress import track
 from ...communicate import (
     HTTPError,
     curated_write_record,
+    get_session,
     incoming_delete_record,
     incoming_read_labels,
     incoming_read_records,
@@ -126,10 +127,8 @@ def cli(
             dry_run,
         )
     except HTTPError as e:
-        rprint(
+        console.print(
             f'[red]Error[/red]: {e}: {e.response.text}',
-            file=sys.stderr,
-            flush=True,
         )
     return 1
 
@@ -154,6 +153,9 @@ def auto_curate(
         console.print(f'[red]Error[/red]: no token was provided (use --token or DTC_TOKEN environment variable)')
         return 1
 
+    if destination_collection is None:
+        destination_collection = collection
+
     if destination_service_url is None:
         destination_service_url = service_url
 
@@ -173,10 +175,12 @@ def auto_curate(
     if list_labels:
         output = []
 
+    session = get_session()
     all_labels = incoming_read_labels(
         service_url=service_url,
         collection=collection,
         token=obj,
+        session=session,
     )
     for label in all_labels:
         if include and label not in include:
@@ -200,6 +204,7 @@ def auto_curate(
             collection=collection,
             label=label,
             token=obj,
+            session=session,
         )
 
         # Get the first entry to find the total number of records
@@ -241,19 +246,25 @@ def auto_curate(
                 class_name = re.search('([_A-Za-z0-9]*$)', record['schema_type']).group(0)
             except (IndexError, KeyError):
                 global stl_info
-                console.print(f'[yellow]Warning[/yellow]: ignoring record with pid {record["pid"]} because `schema_type` attribute is missing.')
+                console.print(f'[yellow]Warning[/yellow]: ignoring record with pid [yellow]{record["pid"]}[/yellow] because `schema_type` attribute is missing.')
                 if not stl_info:
                     console.print(
-                        '         Please ensure that `schema_type` is stored in the records '
-                        'or that the associated incoming area store has a backend with a '
-                        '"Schema Type Layer", i.e., "record_dir+stl" or "sqlite+stl"."',
+                        '         [yellow]Please ensure that `schema_type` is stored in the records. Note: '
+                        'if the incoming area store has a backend with a "Schema Type Layer", i.e., '
+                        '"record_dir+stl" or "sqlite+stl", `schema_type` will not be stored on persistent '
+                        'storage and will not be returned when retrieving records from the incoming area. '
+                        'dump-things-service <= 5.4.0 circumvented the "Schema Type Layer", therefore they '
+                        'will return records without `schema_type` attributes on curator access to '
+                        'incoming areas or curated areas. Therefore it might be a good idea to NOT use a '
+                        '"Schema Type Layer" in collections that shall be auto-curated, when using '
+                        'dump-things-service <= 5.4.0.[/yellow]',
                     )
                     stl_info = True
                 continue
 
             if dry_run:
-                console.print(f'WRITE record [green]"{record["pid"]}"[/green] of class "{class_name}" to "{destination_collection}@{destination_service_url}"')
-                console.print(f'DELETE record [green]"{record["pid"]}"[/green] from inbox "{label}" of "{collection}@{service_url}"')
+                console.print(f'WRITE record [green]"{record["pid"]}"[/green] of class "{class_name}" to collection "{destination_collection}" on "{destination_service_url}"')
+                console.print(f'DELETE record [green]"{record["pid"]}"[/green] from inbox "{label}" of collection "{collection}" on "{service_url}"')
                 continue
 
             # Store record in destination collection
@@ -264,12 +275,13 @@ def auto_curate(
                     class_name=class_name,
                     record=record,
                     token=destination_token,
+                    session=session,
                 )
             except HTTPError as e:
                 console.print(
                     f'[red]Error[/red]: writing record with pid {record["pid"]} failed: {e}: {e.response.text}',
                 )
-                raise
+                return 1
 
             # Delete record from incoming area
             try:
@@ -279,12 +291,13 @@ def auto_curate(
                     label=label,
                     pid=record['pid'],
                     token=curator_token,
+                    session=session,
                 )
             except HTTPError as e:
                 console.print(
                     f'[red]ERROR[/red]: deleting record with pid {record["pid"]} failed: {e}: {e.response.text}',
                 )
-                raise
+                return 1
 
     if output is not None:
         rprint(json.dumps(output, ensure_ascii=False))

dump_things_pyclient/commands/dtc_plugins/clean_incoming.py

@@ -4,6 +4,7 @@ import rich_click as click
 
 from ...communicate import (
     HTTPError,
+    get_session,
     incoming_delete_record,
     incoming_read_records,
 )
@@ -71,11 +72,13 @@ def clean_incoming(
         click.echo('ERROR: token not provided', err=True)
         return 1
 
+    session = get_session()
     for record, _, _, _, _ in incoming_read_records(
             service_url=service_url,
             collection=collection,
             label=inbox_label,
             token=token,
+            session=session,
     ):
         if list_only:
             click.echo(json.dumps(record, ensure_ascii=False))
@@ -88,6 +91,6 @@ def clean_incoming(
             label=inbox_label,
             pid=record['pid'],
             token=token,
-
+            session=session,
         )
     return 0

dump_things_pyclient/commands/dtc_plugins/{delete_record.py → delete_records.py}

@@ -4,9 +4,12 @@ import sys
 from functools import partial
 
 import rich_click as click
+from rich.progress import track
+from rich.console import Console
 
 from ...communicate import (
     HTTPError,
+    get_session,
     collection_delete_record,
     curated_delete_record,
     incoming_delete_record,
@@ -18,6 +21,8 @@ subcommand_name = 'delete-records'
 
 logger = logging.getLogger('delete-records')
 
+console = Console(file=sys.stderr)
+
 
 @click.command(short_help='Delete records from a dump-things collection')
 @click.pass_obj
@@ -83,7 +88,7 @@ def cli(
             ignore_errors,
         )
     except HTTPError as e:
-        click.echo(f'ERROR: {e}: {e.response.text}', err=True)
+        console.print(f'[red]Error[/red]: {e}: {e.response.text}')
     return 1
 
 
@@ -102,16 +107,15 @@ def delete_records(
         click.echo(f'WARNING: no token provided', err=True)
 
     if incoming and curated:
-        click.echo(
-            'ERROR: -i/--incoming and -c/--curated are mutually exclusive',
-            err=True,
-        )
+        console.print('[red]Error[/red]: -i/--incoming and -c/--curated are mutually exclusive')
         return 1
 
+    session = get_session()
     kwargs = dict(
         service_url=service_url,
         collection=collection,
         token=token,
+        session=session,
     )
 
     if incoming == '-':
@@ -134,17 +138,18 @@ def delete_records(
     if not pids:
         pids = sys.stdin
 
-    for pid in pids:
+    for pid in track(pids, console=console):
         try:
             operation(
                 service_url=service_url,
                 collection=collection,
                 pid=pid.strip(),
                 token=token,
+                session=session,
             )
         except HTTPError as e:
+            console.print(f'[red]Error[/red]: while deleting pid {pid}: {e}, {e.response.text}')
             if ignore_errors:
-                click.echo(f'ERROR: while deleting pid {pid}: {e}', err=True)
                 continue
-            raise
+            return 1
     return 0

dump_things_pyclient/commands/dtc_plugins/export.py

@@ -1,17 +1,26 @@
+import hashlib
 import json
+import sys
 from collections import defaultdict
-from itertools import count
+from itertools import (
+    chain,
+    count,
+)
 from pathlib import Path
 from typing import (
     Any,
-    Iterable,
+    Generator,
 )
 
 import rich_click as click
+import yaml
+from rich.console import Console
+from rich.progress import track
 
 from ...communicate import (
     HTTPError,
     curated_read_records,
+    get_session,
     incoming_read_labels,
     incoming_read_records,
     server,
@@ -20,6 +29,8 @@ from ...communicate import (
 
 subcommand_name = 'export'
 
+console = Console(file=sys.stderr)
+
 
 @click.command(short_help='Export a collection to the file system')
 @click.pass_obj
@@ -43,18 +54,34 @@ subcommand_name = 'export'
     ),
     metavar='DESTINATION_DIR',
 )
+@click.option(
+    '--format', '-f', 'output_format',
+    type=click.Choice(('json', 'yaml'), case_sensitive=True),
+    default='json',
+    help='select output format for the exported records (default: json)',
+)
 @click.option(
     '--ignore-errors',
     default=False,
     is_flag=True,
     help='ignore records with missing `schema_type` instead of raising an error',
 )
+@click.option(
+    '--keep-schema-type', '-k',
+    default=False,
+    is_flag=True,
+    help='keep `schema_type`-attribute in records on file-system. By default the '
+         'schema_type-attribute is removed because the class is encoded in the '
+         'storage path of the records.'
+)
 def cli(
         obj: Any,
         service_url: str,
         collection: str,
         destination: Path,
-        ignore_errors,
+        output_format: str,
+        ignore_errors: bool,
+        keep_schema_type: bool,
 ):
     """Export a collection to disk
 
@@ -73,12 +100,14 @@ def cli(
             service_url,
             collection,
             destination,
+            output_format,
             ignore_errors,
+            keep_schema_type,
         )
     except HTTPError as e:
-        click.echo(f'ERROR: {e}: {e.response.text}', err=True)
+        console.print(f'[red]Error[/red]: {e}: {e.response.text}')
     except ValueError as e:
-        click.echo(f'ERROR: {e}', err=True)
+        console.print(f'[red]Error[/red]: {e}')
     return 1
 
 
@@ -87,19 +116,22 @@ def export(
         service_url: str,
         collection: str,
         destination: Path,
+        output_format: str,
         ignore_errors: bool,
+        keep_schema_type: bool,
 ):
     token = obj
 
     if token is None:
-        click.echo(f'ERROR: no token provided', err=True)
+        console.print(f'[red]Error[/red]: no token provided')
         return 1
 
-    server_info = server(service_url)
+    session = get_session()
+    server_info = server(service_url, session=session)
     collection_info = ([c for c in server_info['collections'] if c['name'] == collection] or None)[0]
 
     if not collection_info:
-        click.echo(f'ERROR: no collection {collection} on service', err=True)
+        console.print(f'[red]Error[/red]: no collection {collection} on service')
         return 1
 
     description = {
@@ -116,17 +148,19 @@ def export(
     curated_destination = destination / 'curated'
     curated_destination.mkdir()
 
+    console.print('Exporting records from curated area')
     _store_records(
-        map(
-            lambda x: x[0],
-            curated_read_records(
-                service_url=service_url,
-                collection=collection,
-                token=token,
-            )
+        curated_read_records(
+            service_url=service_url,
+            collection=collection,
+            token=token,
+            session=session,
         ),
         curated_destination,
+        output_format,
         ignore_errors,
+        keep_schema_type,
+        source_name='curated area',
     )
 
     # Store the incoming records
@@ -134,61 +168,128 @@ def export(
             service_url=service_url,
             collection=collection,
             token=token,
+            session=session,
     ):
+        console.print(f'Exporting records from incoming area: {label}')
         incoming_destination = destination / 'incoming' / label
         incoming_destination.mkdir(parents=True, exist_ok=False)
         _store_records(
-            map(
-                lambda x: x[0],
-                incoming_read_records(
-                    service_url=service_url,
-                    collection=collection,
-                    label=label,
-                    token=token,
-                )
+            incoming_read_records(
+                service_url=service_url,
+                collection=collection,
+                label=label,
+                token=token,
+                session=session,
             ),
             incoming_destination,
+            output_format,
             ignore_errors,
+            keep_schema_type,
+            source_name=f'incoming area: {label}'
         )
 
     return 0
 
 
 def _store_records(
-        source: Iterable,
+        source: Generator,
         destination: Path,
-        ignore_errors: bool = False,
+        output_format: str,
+        ignore_errors: bool,
+        keep_schema_type: bool,
+        source_name: str,
 ):
     created_dirs = set()
     class_counters = defaultdict(count)
 
-    for record in source:
-        class_name = _de_prefix(record.get('schema_type', None))
-        if class_name is None:
+    # Get the first result from the source to determine the total number
+    # of records.
+    try:
+        first_tuple = next(source)
+    except StopIteration:
+        console.print(f'no records in incoming [green]{source_name}[/green], skipping it')
+        return
+
+    total = first_tuple[4]
+    for record, _, _, _, _ in track(chain([first_tuple], source), total=total, console=console):
+        schema_type = record.get('schema_type', None)
+        if schema_type is None:
             if ignore_errors:
-                click.echo(
-                    f'WARNING: no `schema_type` in record `{record["pid"]}`',
-                    err=True
-                )
+                console.print(f'[red]Error[/red]: no `schema type` in record [red]{record["pid"]}[/red] in {source_name}')
                 continue
-            msg = f'no `schema_type` in record `{record["pid"]}`'
+            msg = f'no `schema_type` in record {record["pid"]}'
             raise ValueError(msg)
 
-        next_name_for_class = f'{next(class_counters[class_name]):09d}.json'
+        class_name = _de_prefix(schema_type)
+        if not keep_schema_type:
+            del record['schema_type']
+
+        hash_dir, hash_name = _hash_p3(record['pid'])
         file_dir, file_name = (
-            destination / class_name / next_name_for_class[:3],
-            next_name_for_class[3:]
+            destination / class_name / hash_dir,
+            hash_name,
         )
         if file_dir not in created_dirs:
             file_dir.mkdir(parents=True, exist_ok=False)
             created_dirs.add(file_dir)
 
-        (file_dir / file_name).write_text(
-            json.dumps(record, indent=2, ensure_ascii=False),
-        )
+        try:
+            writer[output_format](
+                file_dir=file_dir,
+                file_name=file_name,
+                record=record,
+            )
+        except KeyError as e:
+            msg = f'unsupported output format: {output_format}'
+            raise ValueError(msg)
 
 
 def _de_prefix(
         name: str,
 ):
     return name.split(':', 1)[-1]
+
+
+def _get_hex_digest(
+        data: str,
+) -> str:
+    hash_context = hashlib.md5(data.encode())
+    return hash_context.hexdigest()
+
+
+def _hash_p3(
+        pid: str,
+) -> tuple[str, str]:
+    hex_digest = _get_hex_digest(pid)
+    return hex_digest[:3], hex_digest[3:]
+
+
+def write_json(
+        file_dir: Path,
+        file_name: str,
+        record: dict,
+):
+    (file_dir / (file_name + '.json')).write_text(
+        json.dumps(record, indent=2, ensure_ascii=False) + '\n',
+    )
+
+
+def write_yaml(
+        file_dir: Path,
+        file_name: str,
+        record: dict,
+):
+    (file_dir / (file_name + '.yaml')).write_text(
+        yaml.dump(
+            data=record,
+            sort_keys=False,
+            allow_unicode=True,
+            default_flow_style=False,
+        ),
+    )
+
+
+writer = {
+    'json': write_json,
+    'yaml': write_yaml,
+}

dump_things_pyclient/commands/dtc_plugins/get_records.py

@@ -1,7 +1,9 @@
 import json
+import sys
 from functools import partial
 
 import rich_click as click
+from rich.console import Console
 
 from ...communicate import (
     HTTPError,
@@ -11,6 +13,7 @@ from ...communicate import (
     curated_read_records,
     curated_read_records_of_class,
     curated_read_record_with_pid,
+    get_session,
     incoming_read_labels,
     incoming_read_records,
     incoming_read_records_of_class,
@@ -20,6 +23,8 @@ from ...communicate import (
 
 subcommand_name = 'get-records'
 
+console = Console(file=sys.stderr)
+
 
 @click.command(short_help='Get records from a dump-things collection')
 @click.pass_obj
@@ -48,7 +53,7 @@ subcommand_name = 'get-records'
 @click.option(
     '--incoming', '-i',
     metavar='LABEL',
-    help='read from the collection\'s inbox with label LABEL, if LABEL is "-", return labels of all collection inboxes and exit',
+    help='read from the collection\'s inbox with label LABEL, if LABEL is "-", print labels of all collection inboxes and exit',
 )
 @click.option(
     '--curated', '-c',
@@ -144,7 +149,7 @@ def cli(
             pagination,
         )
     except HTTPError as e:
-        click.echo(f'ERROR: {e}: {e.response.text}', err=True)
+        console.print(f'[red]Error[/red]: {e}: {e.response.text}')
     return 1
 
 
@@ -167,19 +172,18 @@ def get_records(
     token = obj
 
     if token is None:
-        click.echo(f'WARNING: no token provided', err=True)
+        console.print(f'[yellow]Warning[/yellow]: no token provided')
 
     if incoming and curated:
-        click.echo(
-            'ERROR: -i/--incoming and -c/--curated are mutually exclusive',
-            err=True,
-        )
+        console.print('[red]Error[/red]: -i/--incoming and -c/--curated are mutually exclusive')
         return 1
 
+    session = get_session()
     kwargs = dict(
         service_url=service_url,
         collection=collection,
         token=token,
+        session=session,
     )
 
     if incoming == '-':
@@ -187,7 +191,9 @@ def get_records(
         click.echo('\n'.join(
             map(
                 partial(json.dumps, ensure_ascii=False),
-                result)))
+                result
+            )
+        ))
         return 0
 
     elif pid:
@@ -251,8 +257,8 @@ def get_records(
 
     if pagination:
         for record in result:
-            print(json.dumps(record, ensure_ascii=False))
+            click.echo(json.dumps(record, ensure_ascii=False))
     else:
         for record in result:
-            print(json.dumps(record[0], ensure_ascii=False))
+            click.echo(json.dumps(record[0], ensure_ascii=False))
     return 0

dump_things_pyclient/commands/dtc_plugins/maintenance.py

@@ -1,15 +1,20 @@
 import logging
+import sys
 
 import rich_click as click
+from rich.console import Console
 
 from ...communicate import (
     HTTPError,
+    get_session,
     maintenance as communicate_maintenance,
 )
 
 
 logger = logging.getLogger('maintenance')
 
+console = Console(file=sys.stderr)
+
 subcommand_name = 'maintenance'
 
 
@@ -61,13 +66,15 @@ def maintenance(
 ):
     token = obj
     if token is None:
-        click.echo('ERROR: no token provided', err=True)
+        console.print('[red]Error[/red]: no token provided')
         return 1
 
+    session = get_session()
     communicate_maintenance(
         service_url=service_url,
         collection=collection,
         active=active,
         token=token,
+        session=session,
     )
     return 0