dump-things-pyclient 0.1.4__py3-none-any.whl → 0.2.0__py3-none-any.whl

dump_things_pyclient/commands/dtc.py

@@ -0,0 +1,60 @@
+import logging
+import importlib
+import pkgutil
+from pathlib import Path
+
+import click
+
+
+dtc_plugins_dir = Path(__file__).parent / 'dtc_plugins'
+
+# This will add a stream handler
+logging.basicConfig(level=logging.WARNING)
+
+
+def load_subcommands(group):
+    """Load all sub-command plugins and register them with the group"""
+
+    for module_info in pkgutil.iter_modules([dtc_plugins_dir]):
+        try:
+            module = importlib.import_module(
+                '.' + module_info.name,
+                package='dump_things_pyclient.commands.dtc_plugins',
+            )
+        except:
+            logging.exception('failed to load plugin module %s', module_info)
+            exit(1)
+
+        # get the plugin attributes
+        plugin_cli = getattr(module, 'subcommand_name', None)
+        command_name = getattr(module, 'subcommand_name', None)
+
+        # skip non-plugin files
+        if plugin_cli is None or command_name is None:
+            continue
+
+        group.add_command(cmd=getattr(module, 'cli'), name=command_name)
+
+
+@click.group()
+@click.option('--token', envvar='DTC_TOKEN', default=None, help='provide a token on the command line, NOTE: on multiuser systems you should use the environment variable DTC_TOKEN instead')
+@click.option('--debug', envvar='DTC_DEBUG', default=False, is_flag=True, help='show debug output')
+@click.pass_context
+def cli(ctx, token: str, debug: bool):
+    initialize_logging(debug)
+    ctx.obj = token
+
+
+def initialize_logging(debug: bool):
+    logging.basicConfig(
+        level=logging.DEBUG if debug else logging.INFO,
+        force=True,
+    )
+
+
+# Load all command plugins from submodule .dtc_plugins`.
+load_subcommands(cli)
+
+
+if __name__ == '__main__':
+    cli()

dump_things_pyclient/commands/dtc_plugins/__init__.py

@@ -0,0 +1,24 @@
+"""Subcommands for dtc
+
+Each module implements a subcommand. To add a new subcommand, add a
+module with the following attributes:
+
+- `cli`: a `click.command`
+- `subcommand_name`: the name of the subcommand
+
+The following example shows the implementation of the subcommand `demo`
+
+
+```python
+import click
+
+@click.command()
+@click.pass_obj
+def cli(obj):
+    click.echo(f'demo with custom object: {obj}')
+
+subcommand_name = 'demo'
+```
+
+The parameter `obj` will contain a token --if given by the user-- or `None`.
+"""

dump_things_pyclient/commands/dtc_plugins/auto_curate.py

@@ -0,0 +1,248 @@
+import json
+import logging
+import re
+import sys
+
+import click
+
+from ...communicate import (
+    HTTPError,
+    curated_write_record,
+    incoming_delete_record,
+    incoming_read_labels,
+    incoming_read_records,
+)
+
+
+logger = logging.getLogger('auto-curate')
+
+stl_info = False
+
+
+@click.command(short_help='Move records from inbox to curate area of a collection')
+@click.pass_obj
+@click.argument(
+    'service_url',
+    metavar='SERVICE_URL',
+)
+@click.argument(
+    'collection',
+    metavar='COLLECTION',
+)
+@click.option(
+    '--destination-service-url',
+    metavar='DEST_SERVICE_URL',
+    help='select a different dump-thing-service, i.e. not SERVICE_URL, as destination for auto-curated records',
+)
+@click.option(
+    '--destination-collection',
+    metavar='DEST_COLLECTION',
+    help='select a different collection, i.e. not the COLLECTION of SERVICE_URL, as destination for auto-curated records',
+)
+@click.option(
+    '--destination-token',
+    metavar='DEST_TOKEN',
+    help='if provided, this token will be used the authenticate again DEST_SERVICE_URL, otherwise the token for SERVICE_URL will be used',
+)
+@click.option(
+    '--pid', '-p',
+    metavar='PID',
+    help='if provided, process only records that match the given PIDs. NOTE: matching does not involve CURIE-resolution',
+)
+@click.option(
+    '--exclude', '-e',
+    help='exclude an inbox on the source collection (repeatable)',
+    multiple=True,
+)
+@click.option(
+    '--include', '-i',
+    help='process only the given inbox, all other inboxes are ignored (repeatable, -e/--exclude is applied after inclusion)',
+    multiple=True,
+)
+@click.option(
+    '--list-labels', '-l',
+    help='list the inbox labels of the given source collection, do not perform any curation',
+    default=False,
+    is_flag=True,
+)
+@click.option(
+    '--list-records', '-r',
+    help='list records in the inboxes of the given source collection, do not perform any curation',
+    default=False,
+    is_flag=True,
+)
+@click.option(
+    '--dry-run', '-d',
+    help='if provided, do not alter any data, instead print what would be done',
+    default=False,
+    is_flag=True,
+)
+def cli(
+        obj,
+        service_url,
+        collection,
+        destination_service_url,
+        destination_collection,
+        destination_token,
+        pid,
+        exclude,
+        include,
+        list_labels,
+        list_records,
+        dry_run,
+):
+    """Automatically move records from the incoming areas of the collection
+    COLLECTION in the service SERVICE_URL to the curated area of the same
+    collection, or to the curated area of another collection, possibly on
+    another service.
+
+    A token is required and will be used to authenticate the requests.
+    The token must have curator-rights."""
+    try:
+        return auto_curate(
+            obj,
+            service_url,
+            collection,
+            destination_service_url,
+            destination_collection,
+            destination_token,
+            pid,
+            exclude,
+            include,
+            list_labels,
+            list_records,
+            dry_run,
+        )
+    except HTTPError as e:
+        print(f'ERROR: {e}: {e.response.text}', file=sys.stderr, flush=True)
+    return 1
+
+
+def auto_curate(
+        obj,
+        service_url,
+        collection,
+        destination_service_url,
+        destination_collection,
+        destination_token,
+        pid,
+        exclude,
+        include,
+        list_labels,
+        list_records,
+        dry_run,
+):
+    curator_token = obj
+
+    if curator_token is None:
+        print(
+            f'ERROR: no token was provided (use --token or DTC_TOKEN environment variable)',
+            file=sys.stderr,
+            flush=True,
+        )
+        return 1
+
+    click.echo(f'auto curate: {obj}')
+
+    output = None
+
+    # If --list-labels and --list-records are provided, keep only the latter,
+    # because it includes listing of labels
+    if list_records:
+        if list_labels:
+            logger.warning('`-l/--list-labels` and `-r/--list-records` defined, ignoring `-l/--list-labels`')
+            list_labels = False
+        output = {}
+
+    if list_labels:
+        output = []
+
+    for label in incoming_read_labels(
+            service_url=service_url,
+            collection=collection,
+            token=obj,
+    ):
+
+        if include and label not in include:
+            logger.debug('ignoring non-included incoming label: %s', label)
+            continue
+
+        if label in exclude:
+            logger.debug('ignoring excluded incoming label: %s', label)
+            continue
+
+        if list_labels:
+            output.append(label)
+            continue
+
+        if list_records:
+            output[label] = []
+
+        for record, _, _, _, _ in incoming_read_records(
+                service_url=service_url,
+                collection=collection,
+                label=label,
+                token=obj,
+        ):
+
+            if list_records:
+                output[label].append(record)
+                continue
+
+            if pid:
+                if record['pid'] not in pid:
+                    logger.debug(
+                        'ignoring record with non-matching pid: %s',
+                        record['pid'])
+                    continue
+
+            # Get the class name from the `schema_type` attribute. This requires
+            # that the schema type is either stored in the record or that the
+            # store has a "Schema Type Layer", i.e., the store type is
+            # `record_dir+stl`, or `sqlite+stl`.
+            try:
+                class_name = re.search('([_A-Za-z0-9]*$)', record['schema_type']).group(0)
+            except (IndexError, KeyError):
+                global stl_info
+                if not stl_info:
+                    logger.warning(
+                        f"""Could not find `schema_type` attribute in record with
+                            pid {record['pid']}. Please ensure that `schema_type` is stored in
+                            the records or that the associated incoming area store has a backend
+                            with a "Schema Type Layer", i.e., "record_dir+stl" or
+                            "sqlite+stl".""",
+                    )
+                    stl_info = True
+                else:
+                    logger.warning(f'ignoring record with pid {record["pid"]}, `schema_type` attribute is missing.')
+                continue
+
+            if dry_run:
+                print(f'WRITE record "{record["pid"]}" of class "{class_name}" to "{destination_collection}@{destination_service_url}"')
+                print(f'DELETE record "{record["pid"]}" from inbox "{label}" of "{collection}@{service_url}"')
+                continue
+
+            # Store record in destination collection
+            curated_write_record(
+                service_url=destination_service_url,
+                collection=destination_collection,
+                class_name=class_name,
+                record=record,
+                token=destination_token)
+
+            # Delete record from incoming area
+            incoming_delete_record(
+                service_url=service_url,
+                collection=collection,
+                label=label,
+                pid=record['pid'],
+                token=obj,
+            )
+
+    if output is not None:
+        print(json.dumps(output, ensure_ascii=False))
+
+    return 0
+
+
+subcommand_name = 'auto-curate'

dump_things_pyclient/commands/dtc_plugins/clean_incoming.py

@@ -0,0 +1,93 @@
+import json
+
+import click
+
+from ...communicate import (
+    HTTPError,
+    incoming_delete_record,
+    incoming_read_records,
+)
+
+
+subcommand_name = 'clean-incoming'
+
+
+@click.command(short_help='Remove records from an inbox of a dump-things collection')
+@click.pass_obj
+@click.argument(
+    'service_url',
+    metavar='SERVICE_URL',
+)
+@click.argument(
+    'collection',
+    metavar='COLLECTION',
+)
+@click.argument(
+    'inbox_label',
+    metavar='INBOX_LABEL',
+)
+@click.option(
+    '--list-only', '-l',
+    default=False,
+    is_flag=True,
+    help='only list records in the inbox, do not remove them',
+)
+def cli(
+        obj,
+        service_url,
+        collection,
+        inbox_label,
+        list_only,
+):
+    """Remove all records from an incoming areas of a collection on a dump-things-service
+
+    This command removes all records from the inbox with label INBOX_LABEL in
+    the collection COLLECTION on the dump-things service given by SERVICE_URL.
+
+    A token with curator rights has to be provided.
+    """
+    try:
+        return clean_incoming(
+            obj,
+            service_url,
+            collection,
+            inbox_label,
+            list_only,
+        )
+    except HTTPError as e:
+        click.echo(f'ERROR: {e}: {e.response.text}', err=True)
+    return 1
+
+
+def clean_incoming(
+        obj,
+        service_url,
+        collection,
+        inbox_label,
+        list_only,
+):
+    token = obj
+    if token is None:
+        click.echo('ERROR: token not provided', err=True)
+        return 1
+
+    for record, _, _, _, _ in incoming_read_records(
+            service_url=service_url,
+            collection=collection,
+            label=inbox_label,
+            token=token,
+    ):
+        if list_only:
+            click.echo(json.dumps(record, ensure_ascii=False))
+            continue
+
+        # Delete record from incoming area
+        incoming_delete_record(
+            service_url=service_url,
+            collection=collection,
+            label=inbox_label,
+            pid=record['pid'],
+            token=token,
+
+        )
+    return 0

dump_things_pyclient/commands/dtc_plugins/get_records.py

@@ -0,0 +1,262 @@
+import json
+from functools import partial
+
+import click
+
+from ...communicate import (
+    HTTPError,
+    collection_read_records,
+    collection_read_record_with_pid,
+    collection_read_records_of_class,
+    curated_read_records,
+    curated_read_records_of_class,
+    curated_read_record_with_pid,
+    incoming_read_labels,
+    incoming_read_records,
+    incoming_read_records_of_class,
+    incoming_read_record_with_pid,
+)
+
+
+subcommand_name = 'get-records'
+
+
+@click.command(short_help='Get records from a dump-things collection')
+@click.pass_obj
+@click.argument(
+    'service_url',
+    metavar='SERVICE_URL',
+)
+@click.argument(
+    'collection',
+    metavar='COLLECTION',
+)
+@click.option(
+    '--class', '-C', 'cls',
+    default=False,
+    is_flag=True,
+    help='only read records of this class, ignored if "--pid" is provided',
+)
+@click.option(
+    '--format', '-f', 'format_',
+    type=click.Choice(('json', 'ttl'), case_sensitive=False),
+    default='json',
+    help='request records in a specific format. (NOTE: not all endpoints support the "format"-parameter)',
+)
+@click.option(
+    '--pid', '-p',
+    help='the pid of the record that should be read',
+)
+@click.option(
+    '--incoming', '-i',
+    metavar='LABEL',
+    help='read from the collection inbox with label LABEL, if LABEL is "-", return labels of all collection inboxes',
+)
+@click.option(
+    '--curated', '-c',
+    default=False,
+    is_flag=True,
+    help='read from the curated area of the collection. (Note: requires a token with curator rights)',
+)
+@click.option(
+    '--matching', '-m',
+    default=False,
+    is_flag=True,
+    help='return only records that have a matching value (use % as wildcard). Ignored if "--pid" is provided. (Note: not all endpoints and backends support matching)',
+)
+@click.option(
+    '--page-size', '-s',
+    type=click.IntRange(1, 100),
+    default=100,
+    help='set the page size (default: 100). (ignored if "--pid" is provided)'
+)
+@click.option(
+    '--first-page', '-F',
+    type=click.INT,
+    default=1,
+    help='the first page to return (default: 1). (ignored if "--pid" is provided)'
+)
+@click.option(
+    '--last-page', '-l',
+    type=click.INT,
+    help='the last page to return, if not given, all pages will be returned. (ignored if "--pid" is provided)',
+    default=None,
+)
+@click.option(
+    '--stats',
+    default=False,
+    is_flag=True,
+    help='show the number of records and pages and exit. (ignored if "--pid" is provided)',
+)
+@click.option(
+    '--pagination', '-P',
+    default=False,
+    is_flag=True,
+    help='show pagination information (each record from an paginated endpoint is returned as [<record>, <current page number>, <total number of pages>, <page size>, <total number of items>]. (ignored if "--pid" is provided)',
+)
+def cli(
+        obj,
+        service_url,
+        collection,
+        cls,
+        format_,
+        pid,
+        incoming,
+        curated,
+        matching,
+        page_size,
+        first_page,
+        last_page,
+        stats,
+        pagination,
+):
+    """Get records from a collection on a dump-things-service
+
+    This command lists records that are stored in collection COLLECTION of the
+    dump-things service SERVICE_URL. By
+    default, all records that are readable with the given token, or the default
+    token, will be displayed. The output format is JSONL (JSON lines), where
+    every line contains a record or a record with paging information.  If `ttl`
+    is chosen as format of the output records, the record content will be a string
+    that contains a TTL-documents.
+
+    The command supports reading from the curated area only, reading from incoming
+    areas, or reading a record with a given PID.
+
+    Pagination information is returned for paginated results, when requested with
+    `-P/--pagination`. All results are paginated except "get a record with a given PID"
+    and "get the list of incoming zone labels".
+
+    For reading from curated or incoming areas, a token with curator rights has
+    to be provided.
+    """
+    try:
+        return get_records(
+            obj,
+            service_url,
+            collection,
+            cls,
+            format_,
+            pid,
+            incoming,
+            curated,
+            matching,
+            page_size,
+            first_page,
+            last_page,
+            stats,
+            pagination,
+        )
+    except HTTPError as e:
+        click.echo(f'ERROR: {e}: {e.response.text}', err=True)
+    return 1
+
+
+def get_records(
+        obj,
+        service_url,
+        collection,
+        cls,
+        format_,
+        pid,
+        incoming,
+        curated,
+        matching,
+        page_size,
+        first_page,
+        last_page,
+        stats,
+        pagination,
+):
+    token = obj
+
+    if token is None:
+        click.echo(f'WARNING: no token provided', err=True)
+
+    if incoming and curated:
+        click.echo(
+            'ERROR: -i/--incoming and -c/--curated are mutually exclusive',
+            err=True,
+        )
+        return 1
+
+    kwargs = dict(
+        service_url=service_url,
+        collection=collection,
+        token=token,
+    )
+
+    if incoming == '-':
+        result = incoming_read_labels(**kwargs)
+        print('\n'.join(
+            map(
+                partial(json.dumps, ensure_ascii=False),
+                result)))
+        return 0
+
+    elif pid:
+        for argument_value, argument_name in (
+                (matching, '-m/--matching'),
+                (page_size, '-s/--page_size'),
+                (first_page, '-f/--first_page'),
+                (last_page, '-l/--last_page'),
+                (stats, '--stats'),
+                (cls, '-c/--class'),
+        ):
+            if argument_value:
+                click.echo(
+                    f'WARNING: {argument_name} ignored because "-p/--pid" is provided',
+                    err=True,
+                )
+
+        kwargs['pid'] = pid
+        if curated:
+            result = curated_read_record_with_pid(**kwargs)
+        elif incoming:
+            kwargs['label'] = incoming
+            result = incoming_read_record_with_pid(**kwargs)
+        else:
+            kwargs['format'] = format_
+            result = collection_read_record_with_pid(**kwargs)
+        print(json.dumps(result, ensure_ascii=False))
+        return 0
+
+    elif cls:
+        kwargs.update(dict(
+            class_name=cls,
+            matching=matching,
+            page=first_page,
+            size=page_size,
+            last_page=last_page,
+        ))
+        if curated:
+            result = curated_read_records_of_class(**kwargs)
+        elif incoming:
+            kwargs['label'] = incoming
+            result = incoming_read_records_of_class(**kwargs)
+        else:
+            kwargs['format'] = format_
+            result = collection_read_records_of_class(**kwargs)
+    else:
+        kwargs.update(dict(
+            matching=matching,
+            page=first_page,
+            size=page_size or 100,
+            last_page=last_page,
+        ))
+        if curated:
+            result = curated_read_records(**kwargs)
+        elif incoming:
+            kwargs['label'] = incoming
+            result = incoming_read_records(**kwargs)
+        else:
+            kwargs['format'] = format_
+            result = collection_read_records(**kwargs)
+
+    if pagination:
+        for record in result:
+            print(json.dumps(record, ensure_ascii=False))
+    else:
+        for record in result:
+            print(json.dumps(record[0], ensure_ascii=False))
+    return 0