dump-things-pyclient 0.2.2__py3-none-any.whl → 0.2.3__py3-none-any.whl

dump_things_pyclient/commands/dtc_plugins/delete_record.py

@@ -0,0 +1,150 @@
+import json
+import logging
+import sys
+from functools import partial
+
+import rich_click as click
+
+from ...communicate import (
+    HTTPError,
+    collection_delete_record,
+    curated_delete_record,
+    incoming_delete_record,
+    incoming_read_labels,
+)
+
+
+subcommand_name = 'delete-records'
+
+logger = logging.getLogger('delete-records')
+
+
+@click.command(short_help='Delete records from a dump-things collection')
+@click.pass_obj
+@click.argument(
+    'service_url',
+    metavar='SERVICE_URL',
+)
+@click.argument(
+    'collection',
+    metavar='COLLECTION',
+)
+@click.argument(
+    'pids',
+    metavar='PIDS',
+    nargs=-1,
+)
+@click.option(
+    '--curated', '-c',
+    default=False,
+    is_flag=True,
+    help='delete record from the curated area of the collection. (Note: requires a token with curator rights)',
+)
+@click.option(
+    '--incoming', '-i',
+    metavar='LABEL',
+    help='delete from the collection\'s inbox with label LABEL, if LABEL is "-", return labels of all collection inboxes and exit',
+)
+@click.option(
+    '--ignore-errors',
+    default=False,
+    is_flag=True,
+    help='ignore errors when deleting a pid and continue with remaining pids',
+)
+def cli(
+        obj,
+        service_url,
+        collection,
+        pids,
+        curated,
+        incoming,
+        ignore_errors,
+):
+    """Delete records from a collection on a dump-things-service
+
+    This command delete the records given by PIDS from the collection COLLECTION
+    of the dump-things service SERVICE_URL. If no PIDS are provided on the
+    command line, the pids that should be deleted are read from stdin (one pid
+    per line, lines are stripped).
+
+    By default, the records will be deleted from the inbox associated with the
+    token. If the option -c/--curated is given, the records are deleted from
+    the curated area of the collection (this requires a token with
+    curator rights).
+    """
+    try:
+        return delete_records(
+            obj,
+            service_url,
+            collection,
+            pids,
+            curated,
+            incoming,
+            ignore_errors,
+        )
+    except HTTPError as e:
+        click.echo(f'ERROR: {e}: {e.response.text}', err=True)
+    return 1
+
+
+def delete_records(
+        obj,
+        service_url,
+        collection,
+        pids,
+        curated,
+        incoming,
+        ignore_errors,
+):
+    token = obj
+
+    if token is None:
+        click.echo(f'WARNING: no token provided', err=True)
+
+    if incoming and curated:
+        click.echo(
+            'ERROR: -i/--incoming and -c/--curated are mutually exclusive',
+            err=True,
+        )
+        return 1
+
+    kwargs = dict(
+        service_url=service_url,
+        collection=collection,
+        token=token,
+    )
+
+    if incoming == '-':
+        result = incoming_read_labels(**kwargs)
+        click.echo('\n'.join(
+            map(
+                partial(json.dumps, ensure_ascii=False),
+                result
+            )
+        ))
+        return 0
+
+    if incoming:
+        operation = partial(incoming_delete_record, label=incoming)
+    elif curated:
+        operation = curated_delete_record
+    else:
+        operation = collection_delete_record
+
+    if not pids:
+        pids = sys.stdin
+
+    for pid in pids:
+        try:
+            operation(
+                service_url=service_url,
+                collection=collection,
+                pid=pid.strip(),
+                token=token,
+            )
+        except HTTPError as e:
+            if ignore_errors:
+                click.echo(f'ERROR: while deleting pid {pid}: {e}', err=True)
+                continue
+            raise
+    return 0

dump_things_pyclient/commands/dtc_plugins/export.py

@@ -0,0 +1,194 @@
+import json
+from collections import defaultdict
+from itertools import count
+from pathlib import Path
+from typing import (
+    Any,
+    Iterable,
+)
+
+import rich_click as click
+
+from ...communicate import (
+    HTTPError,
+    curated_read_records,
+    incoming_read_labels,
+    incoming_read_records,
+    server,
+)
+
+
+subcommand_name = 'export'
+
+
+@click.command(short_help='Export a collection to the file system')
+@click.pass_obj
+@click.argument(
+    'service_url',
+    metavar='SERVICE_URL',
+)
+@click.argument(
+    'collection',
+    metavar='COLLECTION',
+)
+@click.argument(
+    'destination',
+    type=click.Path(
+        exists=False,
+        file_okay=False,
+        dir_okay=True,
+        writable=True,
+        allow_dash=False,
+        path_type=Path,
+    ),
+    metavar='DESTINATION_DIR',
+)
+@click.option(
+    '--ignore-errors',
+    default=False,
+    is_flag=True,
+    help='ignore records with missing `schema_type` instead of raising an error',
+)
+def cli(
+        obj: Any,
+        service_url: str,
+        collection: str,
+        destination: Path,
+        ignore_errors,
+):
+    """Export a collection to disk
+
+    This command exports all records that are stored in curated area and in the
+    incoming areas of collection COLLECTION of the dump-things service
+    SERVICE_URL.
+
+    Exported records are written to the directory DESTINATION_DIR.
+    DESTINATION_DIR must not exist, `export` will create it.
+
+    A token with curator rights has to be provided.
+    """
+    try:
+        return export(
+            obj,
+            service_url,
+            collection,
+            destination,
+            ignore_errors,
+        )
+    except HTTPError as e:
+        click.echo(f'ERROR: {e}: {e.response.text}', err=True)
+    except ValueError as e:
+        click.echo(f'ERROR: {e}', err=True)
+    return 1
+
+
+def export(
+        obj: Any,
+        service_url: str,
+        collection: str,
+        destination: Path,
+        ignore_errors: bool,
+):
+    token = obj
+
+    if token is None:
+        click.echo(f'ERROR: no token provided', err=True)
+        return 1
+
+    server_info = server(service_url)
+    collection_info = ([c for c in server_info['collections'] if c['name'] == collection] or None)[0]
+
+    if not collection_info:
+        click.echo(f'ERROR: no collection {collection} on service', err=True)
+        return 1
+
+    description = {
+        'name': collection,
+        'schema': collection_info['schema'],
+    }
+
+    destination.mkdir(parents=True, exist_ok=False)
+    (destination / 'description.json').write_text(
+        json.dumps(description, ensure_ascii=False),
+    )
+
+    # Store the curated records
+    curated_destination = destination / 'curated'
+    curated_destination.mkdir()
+
+    _store_records(
+        map(
+            lambda x: x[0],
+            curated_read_records(
+                service_url=service_url,
+                collection=collection,
+                token=token,
+            )
+        ),
+        curated_destination,
+        ignore_errors,
+    )
+
+    # Store the incoming records
+    for label in incoming_read_labels(
+            service_url=service_url,
+            collection=collection,
+            token=token,
+    ):
+        incoming_destination = destination / 'incoming' / label
+        incoming_destination.mkdir(parents=True, exist_ok=False)
+        _store_records(
+            map(
+                lambda x: x[0],
+                incoming_read_records(
+                    service_url=service_url,
+                    collection=collection,
+                    label=label,
+                    token=token,
+                )
+            ),
+            incoming_destination,
+            ignore_errors,
+        )
+
+    return 0
+
+
+def _store_records(
+        source: Iterable,
+        destination: Path,
+        ignore_errors: bool = False,
+):
+    created_dirs = set()
+    class_counters = defaultdict(count)
+
+    for record in source:
+        class_name = _de_prefix(record.get('schema_type', None))
+        if class_name is None:
+            if ignore_errors:
+                click.echo(
+                    f'WARNING: no `schema_type` in record `{record["pid"]}`',
+                    err=True
+                )
+                continue
+            msg = f'no `schema_type` in record `{record["pid"]}`'
+            raise ValueError(msg)
+
+        next_name_for_class = f'{next(class_counters[class_name]):09d}.json'
+        file_dir, file_name = (
+            destination / class_name / next_name_for_class[:3],
+            next_name_for_class[3:]
+        )
+        if file_dir not in created_dirs:
+            file_dir.mkdir(parents=True, exist_ok=False)
+            created_dirs.add(file_dir)
+
+        (file_dir / file_name).write_text(
+            json.dumps(record, indent=2, ensure_ascii=False),
+        )
+
+
+def _de_prefix(
+        name: str,
+):
+    return name.split(':', 1)[-1]

dump_things_pyclient/commands/dtc_plugins/get_records.py

@@ -33,8 +33,6 @@ subcommand_name = 'get-records'
 )
 @click.option(
     '--class', '-C', 'cls',
-    default=False,
-    is_flag=True,
     help='only read records of this class, ignored if "--pid" is provided',
 )
 @click.option(
@@ -50,7 +48,7 @@ subcommand_name = 'get-records'
 @click.option(
     '--incoming', '-i',
     metavar='LABEL',
-    help='read from the collection inbox with label LABEL, if LABEL is "-", return labels of all collection inboxes',
+    help='read from the collection\'s inbox with label LABEL, if LABEL is "-", return labels of all collection inboxes and exit',
 )
 @click.option(
     '--curated', '-c',
@@ -60,8 +58,6 @@ subcommand_name = 'get-records'
 )
 @click.option(
     '--matching', '-m',
-    default=False,
-    is_flag=True,
     help='return only records that have a matching value (use % as wildcard). Ignored if "--pid" is provided. (Note: not all endpoints and backends support matching)',
 )
 @click.option(
@@ -188,7 +184,7 @@ def get_records(
 
     if incoming == '-':
         result = incoming_read_labels(**kwargs)
-        print('\n'.join(
+        click.echo('\n'.join(
             map(
                 partial(json.dumps, ensure_ascii=False),
                 result)))

dump_things_pyclient/communicate.py

@@ -677,6 +677,23 @@ def incoming_delete_record(
         params={'pid': pid})
 
 
+def server(
+        service_url: str,
+) -> JSON:
+    """Get server-information from the service
+
+    :param service_url: the base URL of the service, i.e., the URL up to
+           `/<collection>/...` or `/server`
+
+    :return: information returned by the `<service_url>/server` endpoint
+    """
+    url = (
+        (f'{service_url[:-1]}' if service_url.endswith('/') else service_url)
+        + '/server'
+    )
+    return _do_request(requests.get, url=url, token=None, params=None)
+
+
 def _get_from_url(url: str,
                   token: str | None,
                   params: dict[str, str] | None = None,

{dump_things_pyclient-0.2.2.dist-info → dump_things_pyclient-0.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dump-things-pyclient
-Version: 0.2.2
+Version: 0.2.3
 Summary: A client library and CLI commands for dump-things-services
 Author-email: Christian Mönch <christian.moench@web.de>
 Requires-Python: >=3.11

{dump_things_pyclient-0.2.2.dist-info → dump_things_pyclient-0.2.3.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dump_things_pyclient/__init__.py,sha256=cn-U3TRIalN6aYHp1cMBRkQm1x98XBwquLFbgFEIf_Q,113
-dump_things_pyclient/communicate.py,sha256=CTkgEigu16BmLrlJScMQKhHLI_W4SywDb4HCwAICWzA,30523
+dump_things_pyclient/communicate.py,sha256=R1lifyuP5Cm2oQdyaWxTsQAbiMBU9NxYA5-9yU4M3qA,31008
 dump_things_pyclient/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dump_things_pyclient/commands/dtc.py,sha256=dxW5RuogqwhzfVujZ_EEsQMk8BcVMbZyMdg5c8EvYIA,1726
 dump_things_pyclient/commands/json2ttl.py,sha256=8BkvdjLWZ_H0L6fTmuR2M2MglKiMUiuNUcuWr_w6_dQ,2133
@@ -7,12 +7,14 @@ dump_things_pyclient/commands/redirect.py,sha256=kl8pGj8khjxk4lhk8AJLfgtCIm5Ptje
 dump_things_pyclient/commands/dtc_plugins/__init__.py,sha256=0YLByLiofhHkhJcDCkokldcCw3Jj0rsKJinRX4tt3Hc,514
 dump_things_pyclient/commands/dtc_plugins/auto_curate.py,sha256=3_SHXPQCXmY6GqTMTNVkKh5vvshfiZpMGFY0gvJxRbo,7411
 dump_things_pyclient/commands/dtc_plugins/clean_incoming.py,sha256=slk3xn1-DgMl88WZqgyemyscwof97TMXt3rley4mU1w,2086
-dump_things_pyclient/commands/dtc_plugins/get_records.py,sha256=YBRNo7HUCWZ-EOv8EU_yGMRRoXqqKy1XTcsfQ64ymuk,7508
+dump_things_pyclient/commands/dtc_plugins/delete_record.py,sha256=TVXyGetob8d75zuSBGuRKjtQPAcRqqjw-CcwYYHSC28,3626
+dump_things_pyclient/commands/dtc_plugins/export.py,sha256=FF1DmSmIA3GIhosUrTrXy_qyypMSJUalxdLJ844XhIE,4850
+dump_things_pyclient/commands/dtc_plugins/get_records.py,sha256=00W-agALqO_qFhjq14MI9N6TnJCgscMioy-Ll1ClrB4,7451
 dump_things_pyclient/commands/dtc_plugins/list_incoming.py,sha256=tmM0Qs4MVwMMLyERsWCxWGTM90rSNOShLpHH32wObd8,1959
 dump_things_pyclient/commands/dtc_plugins/post_records.py,sha256=s3j9THe-RszKxyIISkQZRCTKplWWLlomHbS5dyRlep0,2908
 dump_things_pyclient/commands/dtc_plugins/read_pages.py,sha256=hpw7vtG7joIMrNqEqZFCwzbQFd3ATzv7iyySsX8nKWk,3385
-dump_things_pyclient-0.2.2.dist-info/METADATA,sha256=kvCtVMiZmCjtrJssE6O-WGGZBVrpVec2uoXdOcaj8vM,999
-dump_things_pyclient-0.2.2.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
-dump_things_pyclient-0.2.2.dist-info/entry_points.txt,sha256=U1QhQtk767G_OXdZwPdTXYbIPfcDU13Z2u1d6exX8uE,470
-dump_things_pyclient-0.2.2.dist-info/top_level.txt,sha256=Asvruw-SyLoYhWis1CFOx89RGxpjXoTZVGoq4JSGt88,21
-dump_things_pyclient-0.2.2.dist-info/RECORD,,
+dump_things_pyclient-0.2.3.dist-info/METADATA,sha256=hgtqkFRDuld7OTe2Alt-U-2YDqZurOqYT7PHphJ-uGQ,999
+dump_things_pyclient-0.2.3.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+dump_things_pyclient-0.2.3.dist-info/entry_points.txt,sha256=U1QhQtk767G_OXdZwPdTXYbIPfcDU13Z2u1d6exX8uE,470
+dump_things_pyclient-0.2.3.dist-info/top_level.txt,sha256=Asvruw-SyLoYhWis1CFOx89RGxpjXoTZVGoq4JSGt88,21
+dump_things_pyclient-0.2.3.dist-info/RECORD,,

{dump_things_pyclient-0.2.2.dist-info → dump_things_pyclient-0.2.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.10.1)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any