dump-things-pyclient 0.1.4__py3-none-any.whl → 0.2.1__py3-none-any.whl

dump_things_pyclient/commands/dtc_plugins/list_incoming.py

@@ -0,0 +1,86 @@
+import json
+
+import rich_click as click
+
+from ...communicate import (
+    HTTPError,
+    incoming_read_labels,
+    incoming_read_records,
+)
+
+
+subcommand_name = 'list-incoming'
+
+
+@click.command(short_help='List inboxes of a dump-things collection')
+@click.pass_obj
+@click.argument(
+    'service_url',
+    metavar='SERVICE_URL',
+)
+@click.argument(
+    'collection',
+    metavar='COLLECTION',
+)
+@click.option(
+    '--show-records', '-s',
+    default=False,
+    is_flag=True,
+    help='list records in inboxes',
+)
+def cli(
+        obj,
+        service_url,
+        collection,
+        show_records,
+):
+    """List labels of incoming areas of a collection on a dump-things-service
+
+    This command lists the labels of the incoming areas of the collection
+    COLLECTION on the dump-things service given by SERVICE_URL.
+
+    A token with curator rights has to be provided.
+    """
+    try:
+        return list_incoming(
+            obj,
+            service_url,
+            collection,
+            show_records,
+        )
+    except HTTPError as e:
+        click.echo(f'ERROR: {e}: {e.response.text}', err=True)
+    return 1
+
+
+def list_incoming(
+        obj,
+        service_url,
+        collection,
+        show_records,
+):
+    token = obj
+    if token is None:
+        click.echo('ERROR: token not provided', err=True)
+        return 1
+
+    result = {}
+    for label in incoming_read_labels(
+            service_url=service_url,
+            collection=collection,
+            token=token,
+    ):
+        result[label] = []
+        if show_records:
+            for record, _, _, _, _ in incoming_read_records(
+                    service_url=service_url,
+                    collection=collection,
+                    label=label,
+                    token=token,
+            ):
+                result[label].append(record)
+
+    if show_records is False:
+        result = list(result)
+    click.echo(json.dumps(result, indent=2, ensure_ascii=False))
+    return 0

dump_things_pyclient/commands/dtc_plugins/post_records.py

@@ -0,0 +1,116 @@
+import json
+import logging
+import sys
+
+import rich_click as click
+
+from ...communicate import (
+    HTTPError,
+    curated_write_record,
+    collection_write_record,
+)
+
+
+logger = logging.getLogger('post-records')
+
+
+@click.command(short_help='Post records to an inbox or the curated area of a dump-things collection')
+@click.pass_obj
+@click.argument(
+    'service_url',
+    metavar='SERVICE_URL',
+)
+@click.argument(
+    'collection',
+    metavar='COLLECTION',
+)
+@click.argument(
+    'cls',
+    metavar='CLASS',
+)
+@click.option(
+    '--curated',
+    default=False,
+    is_flag=True,
+    help='store record directly in curated area instead of an inbox. (Note: requires a token with curator rights)'
+)
+def cli(
+        obj,
+        service_url,
+        collection,
+        cls,
+        curated,
+):
+    """Read records of class CLASS from standard input and store them in
+    the collection COLLECTION on the service SERVICE_URL. Records should be
+    provided in JSON-lines format. Note: all records are assumed to be of class
+    CLASS. To submit records of multiple classes, the subcommand has to be
+    invoked multiple times, once for each class.
+
+    If the `--curated`-option is provided, the records will be stored directly
+    in the curated area of the collection without any alterations, i.e, no
+    annotations will be added.
+
+    If no `--curated`-option is provided, the record will be stored in the
+    inbox of the user that is associated with the token, and the record will be
+    annotated with the submission time and the user that performed
+    the submission.
+
+    A token is required and will be used to authenticate the requests.
+    If the `--curated`-option is provided, the token must have
+    curator-rights."""
+    try:
+        return post_records(
+            obj,
+            service_url,
+            collection,
+            cls,
+            curated,
+        )
+    except HTTPError as e:
+        click.echo(f'ERROR: {e}: {e.response.text}', err=True)
+    return 1
+
+
+def post_records(
+        obj,
+        service_url,
+        collection,
+        cls,
+        curated,
+):
+    token = obj
+    if token is None:
+        click.echo('ERROR: no token provided', err=True)
+        return 1
+
+    if curated:
+        write_record = curated_write_record
+    else:
+        write_record = collection_write_record
+
+    posted = False
+    for line in sys.stdin:
+        record = json.loads(line)
+        try:
+            write_record(
+                service_url=service_url,
+                collection=collection,
+                class_name=cls,
+                record=record,
+                token=token,
+            )
+        except Exception as e:
+            click.echo(f'ERROR: {e}', err=True)
+        else:
+            posted = True
+            click.echo('.', nl=False)
+
+    if posted:
+        # echo a final newline
+        click.echo('')
+
+    return 0
+
+
+subcommand_name = 'post-records'

dump_things_pyclient/commands/dtc_plugins/read_pages.py

@@ -0,0 +1,141 @@
+import json
+import logging
+
+import rich_click as click
+
+from ...communicate import (
+    HTTPError,
+    get_paginated,
+)
+
+
+logger = logging.getLogger('read-pages')
+
+
+@click.command(short_help='Read records from paginated dump-things endpoints')
+@click.pass_obj
+@click.argument(
+    'url',
+    metavar='URL',
+)
+@click.option(
+    '--page-size', '-s',
+    type=click.INT,
+    default=100,
+    help='set the page size (1 - 100) (default: 100)'
+)
+@click.option(
+    '--first-page', '-F',
+    type=click.INT,
+    default=1,
+    help='the first page to return (default: 1)'
+)
+@click.option(
+    '--last-page', '-l',
+    type=click.INT,
+    help='the last page to return (default: None (return all pages)',
+)
+@click.option(
+    '--stats',
+    is_flag=True,
+    default=False,
+    help='show information about  the number of records and pages and exit, the format is  is returned as [<total number of pages>, <page size>, <total number of items>]',
+)
+@click.option(
+    '--format', '-f', 'format_',
+    type=click.Choice(('json', 'ttl'), case_sensitive=False),
+    default='json',
+    help='request output records in a specific format. (NOTE: not all endpoints support the "format"-parameter)',
+)
+@click.option(
+    '--matching', '-m',
+    help='return only records that have a matching value (use % as wildcard). (NOTE: not all endpoints and storage-backends support matching.)',
+)
+@click.option(
+    '--pagination', '-P',
+    is_flag=True,
+    help='show pagination information (each record from an paginated endpoint is returned as [<record>, <current page number>, <total number of pages>, <page size>, <total number of items>]',
+)
+def cli(
+        obj,
+        url,
+        page_size,
+        first_page,
+        last_page,
+        stats,
+        format_,
+        matching,
+        pagination,
+):
+    """Read paginated endpoint
+
+    This command lists all records that are available via a paginated endpoints from
+    a dump-things-service, e.g., given by URL
+
+      https://<service-location>/<collection>/records/p/
+
+    """
+    try:
+        return read_pages(
+            obj,
+            url,
+            page_size,
+            first_page,
+            last_page,
+            stats,
+            format_,
+            matching,
+            pagination,
+        )
+    except HTTPError as e:
+        click.echo(f'ERROR: {e}: {e.response.text}', err=True)
+    return 1
+
+
+def read_pages(
+        obj,
+        url,
+        page_size,
+        first_page,
+        last_page,
+        stats,
+        format_,
+        matching,
+        pagination,
+):
+    token = obj
+
+    if token is None:
+        click.echo(f'WARNING: no token provided', err=True)
+
+    result = get_paginated(
+        url=url,
+        token=token,
+        first_page=first_page,
+        page_size=page_size,
+        last_page=last_page,
+        parameters={
+            'format': format_,
+            **(
+                {'matching': matching}
+                if matching is not None
+                else {}
+            ),
+        }
+    )
+
+    if stats:
+        record = next(result)
+        click.echo(json.dumps(record[2:], ensure_ascii=False))
+        return 0
+
+    if pagination:
+        for record in result:
+            click.echo(json.dumps(record, ensure_ascii=False))
+    else:
+        for record in result:
+            click.echo(json.dumps(record[0], ensure_ascii=False))
+    return 0
+
+
+subcommand_name = 'read-pages'

{dump_things_pyclient-0.1.4.dist-info → dump_things_pyclient-0.2.1.dist-info}/METADATA

@@ -1,11 +1,13 @@
 Metadata-Version: 2.4
 Name: dump-things-pyclient
-Version: 0.1.4
-Summary: A client library and some CLI command for dump-things-services
+Version: 0.2.1
+Summary: A client library and CLI commands for dump-things-services
 Author-email: Christian Mönch <christian.moench@web.de>
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
+Requires-Dist: click>=8.3.1
 Requires-Dist: requests>=2.32.5
+Requires-Dist: rich-click>=1.9.6
 Provides-Extra: ttl
 Requires-Dist: dump-things-service>=5.3.0; extra == "ttl"
 Provides-Extra: tests

dump_things_pyclient-0.2.1.dist-info/RECORD

@@ -0,0 +1,17 @@
+dump_things_pyclient/__init__.py,sha256=cn-U3TRIalN6aYHp1cMBRkQm1x98XBwquLFbgFEIf_Q,113
+dump_things_pyclient/communicate.py,sha256=CTkgEigu16BmLrlJScMQKhHLI_W4SywDb4HCwAICWzA,30523
+dump_things_pyclient/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dump_things_pyclient/commands/dtc.py,sha256=dxW5RuogqwhzfVujZ_EEsQMk8BcVMbZyMdg5c8EvYIA,1726
+dump_things_pyclient/commands/json2ttl.py,sha256=8BkvdjLWZ_H0L6fTmuR2M2MglKiMUiuNUcuWr_w6_dQ,2133
+dump_things_pyclient/commands/dtc_plugins/__init__.py,sha256=0YLByLiofhHkhJcDCkokldcCw3Jj0rsKJinRX4tt3Hc,514
+dump_things_pyclient/commands/dtc_plugins/auto_curate.py,sha256=3_SHXPQCXmY6GqTMTNVkKh5vvshfiZpMGFY0gvJxRbo,7411
+dump_things_pyclient/commands/dtc_plugins/clean_incoming.py,sha256=slk3xn1-DgMl88WZqgyemyscwof97TMXt3rley4mU1w,2086
+dump_things_pyclient/commands/dtc_plugins/get_records.py,sha256=YBRNo7HUCWZ-EOv8EU_yGMRRoXqqKy1XTcsfQ64ymuk,7508
+dump_things_pyclient/commands/dtc_plugins/list_incoming.py,sha256=tmM0Qs4MVwMMLyERsWCxWGTM90rSNOShLpHH32wObd8,1959
+dump_things_pyclient/commands/dtc_plugins/post_records.py,sha256=s3j9THe-RszKxyIISkQZRCTKplWWLlomHbS5dyRlep0,2908
+dump_things_pyclient/commands/dtc_plugins/read_pages.py,sha256=hpw7vtG7joIMrNqEqZFCwzbQFd3ATzv7iyySsX8nKWk,3385
+dump_things_pyclient-0.2.1.dist-info/METADATA,sha256=7TPuI97WTVrgJrPBWOzfYOFAOo7_lNXzbzPD-OkskrY,999
+dump_things_pyclient-0.2.1.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
+dump_things_pyclient-0.2.1.dist-info/entry_points.txt,sha256=UciGaqSUivgH8oFLO8vNhHqKLmVFhjdM1tdfqCwgOok,117
+dump_things_pyclient-0.2.1.dist-info/top_level.txt,sha256=Asvruw-SyLoYhWis1CFOx89RGxpjXoTZVGoq4JSGt88,21
+dump_things_pyclient-0.2.1.dist-info/RECORD,,

{dump_things_pyclient-0.1.4.dist-info → dump_things_pyclient-0.2.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

dump_things_pyclient-0.2.1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+dtc = dump_things_pyclient.commands.dtc:cli
+json2ttl = dump_things_pyclient.commands.json2ttl:main

dump_things_pyclient/commands/auto_curate.py

@@ -1,214 +0,0 @@
-from __future__ import annotations
-
-import argparse
-import json
-import logging
-import os
-import re
-import sys
-
-from ..communicate import (
-    HTTPError,
-    curated_write_record,
-    incoming_delete_record,
-    incoming_read_labels,
-    incoming_read_records,
-)
-
-
-logger = logging.getLogger('auto-curate')
-
-token_name = 'DUMPTHINGS_TOKEN'
-
-stl_info = False
-
-description=f"""
-Automatically move records from the incoming areas of a
-collection to the curated area of the same collection, or to
-the curated area of another collection.
-
-The environment variable "{token_name}" must contain a token
-which used to authenticate the requests. The token must have
-curator-rights.
-"""
-
-
-def _main():
-    argument_parser = argparse.ArgumentParser(
-        description=description,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    argument_parser.add_argument('service_url', metavar='SOURCE_SERVICE_URL')
-    argument_parser.add_argument('collection', metavar='SOURCE_COLLECTION')
-    argument_parser.add_argument(
-        '--destination-service-url',
-        default=None,
-        metavar='DEST_SERVICE_URL',
-        help='select a different dump-thing-service, i.e. not SOURCE_SERVICE_URL, as destination for auto-curated records',
-    )
-    argument_parser.add_argument(
-        '--destination-collection',
-        default=None,
-        metavar='DEST_COLLECTION',
-        help='select a different collection, i.e. not the SOURCE_COLLECTION of SOURCE_SERVICE_URL, as destination for auto-curated records',
-    ),
-    argument_parser.add_argument(
-        '--destination-token',
-        default=None,
-        metavar='DEST_TOKEN',
-        help='if provided, this token will be used for the destination service, otherwise ${CURATOR_TOKEN} will be used',
-    )
-    argument_parser.add_argument(
-        '-e', '--exclude',
-        action='append',
-        default=[],
-        help='exclude an inbox on the source collection (repeatable)',
-    )
-    argument_parser.add_argument(
-        '-i', '--include',
-        action='append',
-        default=[],
-        help='process only the given inbox, all other inboxes are ignored  (repeatable, -e/--exclude is applied after inclusion)',
-    )
-    argument_parser.add_argument(
-        '-l', '--list-labels',
-        action='store_true',
-        help='list the inbox labels of the given source collection, do not perform any curation',
-    )
-    argument_parser.add_argument(
-        '-r', '--list-records',
-        action='store_true',
-        help='list records in the inboxes of the given source collection, do not perform any curation',
-    )
-    argument_parser.add_argument(
-        '-p', '--pid',
-        action='append',
-        help='if provided, process only records that match the given PIDs. NOTE: matching does not involve CURIE-resolution!',
-    )
-    argument_parser.add_argument(
-        '-d', '--dry-run',
-        action='store_true',
-        help='if provided, do not alter any data, instead print what would be done',
-    )
-    arguments = argument_parser.parse_args()
-
-    curator_token = os.environ.get(token_name)
-    if curator_token is None:
-        print(f'ERROR: environment variable "{token_name}" not set', file=sys.stderr, flush=True)
-        return 1
-
-    destination_url = arguments.destination_service_url or arguments.service_url
-    destination_collection = arguments.destination_collection or arguments.collection
-    destination_token = arguments.destination_token or curator_token
-
-    output = None
-
-    # If --list-labels and --list-records are provided, keep only the latter,
-    # because it includes listing of labels
-    if arguments.list_records:
-        if arguments.list_labels:
-            print('WARNING: `-l/--list-labels` and `-r/--list-records` defined, ignoring `-l/--list-labels`', file=sys.stderr, flush=True)
-            arguments.list_labels = False
-        output = {}
-    if arguments.list_labels:
-        output = []
-
-    for label in incoming_read_labels(
-                 service_url=arguments.service_url,
-                 collection=arguments.collection,
-                 token=curator_token):
-
-        if arguments.include and label not in arguments.include:
-            logger.debug('ignoring non-included incoming label: %s', label)
-            continue
-
-        if label in arguments.exclude:
-            logger.debug('ignoring excluded incoming label: %s', label)
-            continue
-
-        if arguments.list_labels:
-            output.append(label)
-            continue
-
-        if arguments.list_records:
-            output[label] = []
-
-        for record, _, _, _, _ in incoming_read_records(
-                                  service_url=arguments.service_url,
-                                  collection=arguments.collection,
-                                  label=label,
-                                  token=curator_token):
-
-            if arguments.pid:
-                if record['pid'] not in arguments.pid:
-                    logger.debug(
-                        'ignoring record with non-matching pid: %s',
-                        record['pid'])
-                    continue
-
-            if arguments.list_records:
-                output[label].append(record)
-                continue
-
-            # Get the class name from the `schema_type` attribute. This requires
-            # that the schema type is either stored in the record or that the
-            # store has a "Schema Type Layer", i.e., the store type is
-            # `record_dir+stl`, or `sqlite+stl`.
-            try:
-                class_name = re.search('([_A-Za-z0-9]*$)', record['schema_type']).group(0)
-            except (IndexError, KeyError):
-                global stl_info
-                if not stl_info:
-                    print(
-                        f"""Could not find `schema_type` attribute in record with
-                            pid {record['pid']}. Please ensure that `schema_type` is stored in
-                            the records or that the associated incoming area store has a backend
-                            with a "Schema Type Layer", i.e., "record_dir+stl" or
-                            "sqlite+stl".""",
-                        file=sys.stderr,
-                        flush=True)
-                    stl_info = True
-                print(
-                    f'WARNING: ignoring record with pid {record["pid"]}, `schema_type` attribute is missing.',
-                    file=sys.stderr,
-                    flush=True)
-                continue
-
-            if arguments.dry_run:
-                print(f'WRITE record "{record["pid"]}" of class "{class_name}" to "{destination_collection}@{destination_url}"')
-                print(f'DELETE record "{record["pid"]}" from inbox "{label}" of "{arguments.collection}@{arguments.service_url}"')
-                continue
-
-            # Store record in destination collection
-            curated_write_record(
-                service_url=destination_url,
-                collection=destination_collection,
-                class_name=class_name,
-                record=record,
-                token=destination_token)
-
-            # Delete record from incoming area
-            incoming_delete_record(
-                service_url=arguments.service_url,
-                collection=arguments.collection,
-                label=label,
-                pid=record['pid'],
-                token=curator_token,
-            )
-
-    if output is not None:
-        print(json.dumps(output, ensure_ascii=False))
-
-    return 0
-
-
-def main():
-    try:
-        return _main()
-    except HTTPError as e:
-        print(f'ERROR: {e}: {e.response.text}', file=sys.stderr, flush=True)
-    return 1
-
-
-if __name__ == '__main__':
-    sys.exit(main())