dump-things-pyclient 0.2.3__py3-none-any.whl → 0.2.6__py3-none-any.whl

dump_things_pyclient/commands/dtc_plugins/auto_curate.py

@@ -2,12 +2,24 @@ import json
 import logging
 import re
 import sys
+from itertools import (
+    count,
+    chain,
+)
+from typing import (
+    cast,
+    Iterable,
+)
 
 import rich_click as click
+from rich import print as rprint
+from rich.console import Console
+from rich.progress import track
 
 from ...communicate import (
     HTTPError,
     curated_write_record,
+    get_session,
     incoming_delete_record,
     incoming_read_labels,
     incoming_read_records,
@@ -16,6 +28,7 @@ from ...communicate import (
 
 logger = logging.getLogger('auto-curate')
 
+console = Console(file=sys.stderr)
 stl_info = False
 
 
@@ -32,17 +45,17 @@ stl_info = False
 @click.option(
     '--destination-service-url',
     metavar='DEST_SERVICE_URL',
-    help='select a different dump-thing-service, i.e. not SERVICE_URL, as destination for auto-curated records',
+    help='select a different dump-thing-service, i.e. not SERVICE_URL, as destination for auto-curated records (the default is SERVICE_URL)',
 )
 @click.option(
     '--destination-collection',
     metavar='DEST_COLLECTION',
-    help='select a different collection, i.e. not the COLLECTION of SERVICE_URL, as destination for auto-curated records',
+    help='select a different collection, i.e. not COLLECTION, as destination for auto-curated records',
 )
 @click.option(
     '--destination-token',
     metavar='DEST_TOKEN',
-    help='if provided, this token will be used the authenticate again DEST_SERVICE_URL, otherwise the token for SERVICE_URL will be used',
+    help='if provided, this token will be used the authenticate against DEST_SERVICE_URL, which defaults to SERVICE_URL (the default is the token provided via --token)',
 )
 @click.option(
     '--pid', '-p',
@@ -114,7 +127,9 @@ def cli(
             dry_run,
         )
     except HTTPError as e:
-        print(f'ERROR: {e}: {e.response.text}', file=sys.stderr, flush=True)
+        console.print(
+            f'[red]Error[/red]: {e}: {e.response.text}',
+        )
     return 1
 
 
@@ -135,14 +150,17 @@ def auto_curate(
     curator_token = obj
 
     if curator_token is None:
-        print(
-            f'ERROR: no token was provided (use --token or DTC_TOKEN environment variable)',
-            file=sys.stderr,
-            flush=True,
-        )
+        console.print(f'[red]Error[/red]: no token was provided (use --token or DTC_TOKEN environment variable)')
         return 1
 
-    click.echo(f'auto curate: {obj}')
+    if destination_collection is None:
+        destination_collection = collection
+
+    if destination_service_url is None:
+        destination_service_url = service_url
+
+    if destination_token is None:
+        destination_token = curator_token
 
     output = None
 
@@ -157,12 +175,14 @@ def auto_curate(
     if list_labels:
         output = []
 
-    for label in incoming_read_labels(
-            service_url=service_url,
-            collection=collection,
-            token=obj,
-    ):
-
+    session = get_session()
+    all_labels = incoming_read_labels(
+        service_url=service_url,
+        collection=collection,
+        token=obj,
+        session=session,
+    )
+    for label in all_labels:
         if include and label not in include:
             logger.debug('ignoring non-included incoming label: %s', label)
             continue
@@ -178,13 +198,35 @@ def auto_curate(
         if list_records:
             output[label] = []
 
-        for record, _, _, _, _ in incoming_read_records(
-                service_url=service_url,
-                collection=collection,
-                label=label,
-                token=obj,
-        ):
+        # Get the total number of entries for the
+        record_source = incoming_read_records(
+            service_url=service_url,
+            collection=collection,
+            label=label,
+            token=obj,
+            session=session,
+        )
 
+        # Get the first entry to find the total number of records
+        try:
+            first_record, _, _, _, total = next(record_source)
+        except StopIteration:
+            console.print(f'no records in incoming area [green]{label}[/green], skipping it')
+            continue
+
+        # Get the first entry an all other entries
+        for index, (record, _, _, _, total) in track(
+                zip(
+                    count(),
+                    chain(
+                        [(first_record, None, None, None, None)],
+                        cast(Iterable, record_source),
+                    ),
+                ),
+                description=f'processing [green]{label}[/green]',
+                total=total,
+                console=console,
+        ):
             if list_records:
                 output[label].append(record)
                 continue
@@ -204,43 +246,61 @@ def auto_curate(
                 class_name = re.search('([_A-Za-z0-9]*$)', record['schema_type']).group(0)
             except (IndexError, KeyError):
                 global stl_info
+                console.print(f'[yellow]Warning[/yellow]: ignoring record with pid [yellow]{record["pid"]}[/yellow] because `schema_type` attribute is missing.')
                 if not stl_info:
-                    logger.warning(
-                        f"""Could not find `schema_type` attribute in record with
-                            pid {record['pid']}. Please ensure that `schema_type` is stored in
-                            the records or that the associated incoming area store has a backend
-                            with a "Schema Type Layer", i.e., "record_dir+stl" or
-                            "sqlite+stl".""",
+                    console.print(
+                        '         [yellow]Please ensure that `schema_type` is stored in the records. Note: '
+                        'if the incoming area store has a backend with a "Schema Type Layer", i.e., '
+                        '"record_dir+stl" or "sqlite+stl", `schema_type` will not be stored on persistent '
+                        'storage and will not be returned when retrieving records from the incoming area. '
+                        'dump-things-service <= 5.4.0 circumvented the "Schema Type Layer", therefore they '
+                        'will return records without `schema_type` attributes on curator access to '
+                        'incoming areas or curated areas. Therefore it might be a good idea to NOT use a '
+                        '"Schema Type Layer" in collections that shall be auto-curated, when using '
+                        'dump-things-service <= 5.4.0.[/yellow]',
                     )
                     stl_info = True
-                else:
-                    logger.warning(f'ignoring record with pid {record["pid"]}, `schema_type` attribute is missing.')
                 continue
 
             if dry_run:
-                print(f'WRITE record "{record["pid"]}" of class "{class_name}" to "{destination_collection}@{destination_service_url}"')
-                print(f'DELETE record "{record["pid"]}" from inbox "{label}" of "{collection}@{service_url}"')
+                console.print(f'WRITE record [green]"{record["pid"]}"[/green] of class "{class_name}" to collection "{destination_collection}" on "{destination_service_url}"')
+                console.print(f'DELETE record [green]"{record["pid"]}"[/green] from inbox "{label}" of collection "{collection}" on "{service_url}"')
                 continue
 
             # Store record in destination collection
-            curated_write_record(
-                service_url=destination_service_url,
-                collection=destination_collection,
-                class_name=class_name,
-                record=record,
-                token=destination_token)
+            try:
+                curated_write_record(
+                    service_url=destination_service_url,
+                    collection=destination_collection,
+                    class_name=class_name,
+                    record=record,
+                    token=destination_token,
+                    session=session,
+                )
+            except HTTPError as e:
+                console.print(
+                    f'[red]Error[/red]: writing record with pid {record["pid"]} failed: {e}: {e.response.text}',
+                )
+                return 1
 
             # Delete record from incoming area
-            incoming_delete_record(
-                service_url=service_url,
-                collection=collection,
-                label=label,
-                pid=record['pid'],
-                token=obj,
-            )
+            try:
+                incoming_delete_record(
+                    service_url=service_url,
+                    collection=collection,
+                    label=label,
+                    pid=record['pid'],
+                    token=curator_token,
+                    session=session,
+                )
+            except HTTPError as e:
+                console.print(
+                    f'[red]ERROR[/red]: deleting record with pid {record["pid"]} failed: {e}: {e.response.text}',
+                )
+                return 1
 
     if output is not None:
-        print(json.dumps(output, ensure_ascii=False))
+        rprint(json.dumps(output, ensure_ascii=False))
 
     return 0
 

dump_things_pyclient/commands/dtc_plugins/clean_incoming.py

@@ -4,6 +4,7 @@ import rich_click as click
 
 from ...communicate import (
     HTTPError,
+    get_session,
     incoming_delete_record,
     incoming_read_records,
 )
@@ -71,11 +72,13 @@ def clean_incoming(
         click.echo('ERROR: token not provided', err=True)
         return 1
 
+    session = get_session()
     for record, _, _, _, _ in incoming_read_records(
             service_url=service_url,
             collection=collection,
             label=inbox_label,
             token=token,
+            session=session,
     ):
         if list_only:
             click.echo(json.dumps(record, ensure_ascii=False))
@@ -88,6 +91,6 @@ def clean_incoming(
             label=inbox_label,
             pid=record['pid'],
             token=token,
-
+            session=session,
         )
     return 0

dump_things_pyclient/commands/dtc_plugins/{delete_record.py → delete_records.py}

@@ -4,9 +4,12 @@ import sys
 from functools import partial
 
 import rich_click as click
+from rich.progress import track
+from rich.console import Console
 
 from ...communicate import (
     HTTPError,
+    get_session,
     collection_delete_record,
     curated_delete_record,
     incoming_delete_record,
@@ -18,6 +21,8 @@ subcommand_name = 'delete-records'
 
 logger = logging.getLogger('delete-records')
 
+console = Console(file=sys.stderr)
+
 
 @click.command(short_help='Delete records from a dump-things collection')
 @click.pass_obj
@@ -83,7 +88,7 @@ def cli(
             ignore_errors,
         )
     except HTTPError as e:
-        click.echo(f'ERROR: {e}: {e.response.text}', err=True)
+        console.print(f'[red]Error[/red]: {e}: {e.response.text}')
     return 1
 
 
@@ -102,16 +107,15 @@ def delete_records(
         click.echo(f'WARNING: no token provided', err=True)
 
     if incoming and curated:
-        click.echo(
-            'ERROR: -i/--incoming and -c/--curated are mutually exclusive',
-            err=True,
-        )
+        console.print('[red]Error[/red]: -i/--incoming and -c/--curated are mutually exclusive')
         return 1
 
+    session = get_session()
     kwargs = dict(
         service_url=service_url,
         collection=collection,
         token=token,
+        session=session,
     )
 
     if incoming == '-':
@@ -134,17 +138,18 @@ def delete_records(
     if not pids:
         pids = sys.stdin
 
-    for pid in pids:
+    for pid in track(pids, console=console):
         try:
             operation(
                 service_url=service_url,
                 collection=collection,
                 pid=pid.strip(),
                 token=token,
+                session=session,
             )
         except HTTPError as e:
+            console.print(f'[red]Error[/red]: while deleting pid {pid}: {e}, {e.response.text}')
             if ignore_errors:
-                click.echo(f'ERROR: while deleting pid {pid}: {e}', err=True)
                 continue
-            raise
+            return 1
     return 0

dump_things_pyclient/commands/dtc_plugins/export.py

@@ -1,4 +1,5 @@
 import json
+import sys
 from collections import defaultdict
 from itertools import count
 from pathlib import Path
@@ -8,10 +9,13 @@ from typing import (
 )
 
 import rich_click as click
+from rich.console import Console
+from rich.progress import track
 
 from ...communicate import (
     HTTPError,
     curated_read_records,
+    get_session,
     incoming_read_labels,
     incoming_read_records,
     server,
@@ -20,6 +24,8 @@ from ...communicate import (
 
 subcommand_name = 'export'
 
+console = Console(file=sys.stderr)
+
 
 @click.command(short_help='Export a collection to the file system')
 @click.pass_obj
@@ -76,9 +82,9 @@ def cli(
             ignore_errors,
         )
     except HTTPError as e:
-        click.echo(f'ERROR: {e}: {e.response.text}', err=True)
+        console.print(f'[red]Error[/red]: {e}: {e.response.text}')
     except ValueError as e:
-        click.echo(f'ERROR: {e}', err=True)
+        console.print(f'[red]Error[/red]: {e}')
     return 1
 
 
@@ -92,14 +98,15 @@ def export(
     token = obj
 
     if token is None:
-        click.echo(f'ERROR: no token provided', err=True)
+        console.print(f'[red]Error[/red]: no token provided')
         return 1
 
-    server_info = server(service_url)
+    session = get_session()
+    server_info = server(service_url, session=session)
     collection_info = ([c for c in server_info['collections'] if c['name'] == collection] or None)[0]
 
     if not collection_info:
-        click.echo(f'ERROR: no collection {collection} on service', err=True)
+        console.print(f'[red]Error[/red]: no collection {collection} on service')
         return 1
 
     description = {
@@ -116,6 +123,7 @@ def export(
     curated_destination = destination / 'curated'
     curated_destination.mkdir()
 
+    console.print('Exporting records from curated area')
     _store_records(
         map(
             lambda x: x[0],
@@ -123,6 +131,7 @@ def export(
                 service_url=service_url,
                 collection=collection,
                 token=token,
+                session=session,
             )
         ),
         curated_destination,
@@ -134,7 +143,9 @@ def export(
             service_url=service_url,
             collection=collection,
             token=token,
+            session=session,
     ):
+        console.print(f'Exporting records from incoming area: {label}')
         incoming_destination = destination / 'incoming' / label
         incoming_destination.mkdir(parents=True, exist_ok=False)
         _store_records(
@@ -145,6 +156,7 @@ def export(
                     collection=collection,
                     label=label,
                     token=token,
+                    session=session,
                 )
             ),
             incoming_destination,
@@ -162,16 +174,13 @@ def _store_records(
     created_dirs = set()
     class_counters = defaultdict(count)
 
-    for record in source:
+    for record in track(source, console=console):
         class_name = _de_prefix(record.get('schema_type', None))
         if class_name is None:
             if ignore_errors:
-                click.echo(
-                    f'WARNING: no `schema_type` in record `{record["pid"]}`',
-                    err=True
-                )
+                console.print(f'[red]Error[/red]: no `schema type` in record {record["pid"]}')
                 continue
-            msg = f'no `schema_type` in record `{record["pid"]}`'
+            msg = f'no `schema_type` in record {record["pid"]}'
             raise ValueError(msg)
 
         next_name_for_class = f'{next(class_counters[class_name]):09d}.json'

dump_things_pyclient/commands/dtc_plugins/get_records.py

@@ -1,7 +1,9 @@
 import json
+import sys
 from functools import partial
 
 import rich_click as click
+from rich.console import Console
 
 from ...communicate import (
     HTTPError,
@@ -11,6 +13,7 @@ from ...communicate import (
     curated_read_records,
     curated_read_records_of_class,
     curated_read_record_with_pid,
+    get_session,
     incoming_read_labels,
     incoming_read_records,
     incoming_read_records_of_class,
@@ -20,6 +23,8 @@ from ...communicate import (
 
 subcommand_name = 'get-records'
 
+console = Console(file=sys.stderr)
+
 
 @click.command(short_help='Get records from a dump-things collection')
 @click.pass_obj
@@ -48,7 +53,7 @@ subcommand_name = 'get-records'
 @click.option(
     '--incoming', '-i',
     metavar='LABEL',
-    help='read from the collection\'s inbox with label LABEL, if LABEL is "-", return labels of all collection inboxes and exit',
+    help='read from the collection\'s inbox with label LABEL, if LABEL is "-", print labels of all collection inboxes and exit',
 )
 @click.option(
     '--curated', '-c',
@@ -144,7 +149,7 @@ def cli(
             pagination,
         )
     except HTTPError as e:
-        click.echo(f'ERROR: {e}: {e.response.text}', err=True)
+        console.print(f'[red]Error[/red]: {e}: {e.response.text}')
     return 1
 
 
@@ -167,19 +172,18 @@ def get_records(
     token = obj
 
     if token is None:
-        click.echo(f'WARNING: no token provided', err=True)
+        console.print(f'[yellow]Warning[/yellow]: no token provided')
 
     if incoming and curated:
-        click.echo(
-            'ERROR: -i/--incoming and -c/--curated are mutually exclusive',
-            err=True,
-        )
+        console.print('[red]Error[/red]: -i/--incoming and -c/--curated are mutually exclusive')
         return 1
 
+    session = get_session()
     kwargs = dict(
         service_url=service_url,
         collection=collection,
         token=token,
+        session=session,
     )
 
     if incoming == '-':
@@ -187,7 +191,9 @@ def get_records(
         click.echo('\n'.join(
             map(
                 partial(json.dumps, ensure_ascii=False),
-                result)))
+                result
+            )
+        ))
         return 0
 
     elif pid:
@@ -251,8 +257,8 @@ def get_records(
 
     if pagination:
         for record in result:
-            print(json.dumps(record, ensure_ascii=False))
+            click.echo(json.dumps(record, ensure_ascii=False))
     else:
         for record in result:
-            print(json.dumps(record[0], ensure_ascii=False))
+            click.echo(json.dumps(record[0], ensure_ascii=False))
     return 0

dump_things_pyclient/commands/dtc_plugins/maintenance.py

@@ -0,0 +1,80 @@
+import logging
+import sys
+
+import rich_click as click
+from rich.console import Console
+
+from ...communicate import (
+    HTTPError,
+    get_session,
+    maintenance as communicate_maintenance,
+)
+
+
+logger = logging.getLogger('maintenance')
+
+console = Console(file=sys.stderr)
+
+subcommand_name = 'maintenance'
+
+
+@click.command(short_help='Activate or deactivate maintenance mode on a collection')
+@click.pass_obj
+@click.argument(
+    'service_url',
+    metavar='SERVICE_URL',
+)
+@click.argument(
+    'collection',
+    metavar='COLLECTION',
+)
+@click.argument(
+    'active',
+    metavar='ACTIVE',
+    type=click.Choice(['On', 'Off'], case_sensitive=False),
+)
+def cli(
+        obj,
+        service_url: str,
+        collection: str,
+        active: bool,
+):
+    """Activate or deactivate maintenance mode on collection COLLECTION on the
+    service SERVICE_URL. The argument ACTIVE should be either `On` or `Off`
+    (case-insensitive).
+
+    A token with curator rights is required.
+
+    This command expects a server version >= 5.4.0"""
+    try:
+        return maintenance(
+            obj,
+            service_url,
+            collection,
+            active,
+        )
+    except HTTPError as e:
+        click.echo(f'ERROR: {e}: {e.response.text}', err=True)
+    return 1
+
+
+def maintenance(
+        obj: str,
+        service_url: str,
+        collection: str,
+        active: bool,
+):
+    token = obj
+    if token is None:
+        console.print('[red]Error[/red]: no token provided')
+        return 1
+
+    session = get_session()
+    communicate_maintenance(
+        service_url=service_url,
+        collection=collection,
+        active=active,
+        token=token,
+        session=session,
+    )
+    return 0