dump-things-pyclient 0.2.6__py3-none-any.whl → 0.2.8__py3-none-any.whl

dump_things_pyclient/commands/dtc_plugins/auto_curate.py

@@ -12,7 +12,6 @@ from typing import (
 )
 
 import rich_click as click
-from rich import print as rprint
 from rich.console import Console
 from rich.progress import track
 
@@ -300,7 +299,7 @@ def auto_curate(
                 return 1
 
     if output is not None:
-        rprint(json.dumps(output, ensure_ascii=False))
+        click.echo(json.dumps(output, ensure_ascii=False))
 
     return 0
 

dump_things_pyclient/commands/dtc_plugins/export.py

@@ -1,14 +1,19 @@
+import hashlib
 import json
 import sys
 from collections import defaultdict
-from itertools import count
+from itertools import (
+    chain,
+    count,
+)
 from pathlib import Path
 from typing import (
     Any,
-    Iterable,
+    Generator,
 )
 
 import rich_click as click
+import yaml
 from rich.console import Console
 from rich.progress import track
 
@@ -49,18 +54,34 @@ console = Console(file=sys.stderr)
     ),
     metavar='DESTINATION_DIR',
 )
+@click.option(
+    '--format', '-f', 'output_format',
+    type=click.Choice(('json', 'yaml'), case_sensitive=True),
+    default='json',
+    help='select output format for the exported records (default: json)',
+)
 @click.option(
     '--ignore-errors',
     default=False,
     is_flag=True,
     help='ignore records with missing `schema_type` instead of raising an error',
 )
+@click.option(
+    '--keep-schema-type', '-k',
+    default=False,
+    is_flag=True,
+    help='keep `schema_type`-attribute in records on file-system. By default the '
+         'schema_type-attribute is removed because the class is encoded in the '
+         'storage path of the records.'
+)
 def cli(
         obj: Any,
         service_url: str,
         collection: str,
         destination: Path,
-        ignore_errors,
+        output_format: str,
+        ignore_errors: bool,
+        keep_schema_type: bool,
 ):
     """Export a collection to disk
 
@@ -79,7 +100,9 @@ def cli(
             service_url,
             collection,
             destination,
+            output_format,
             ignore_errors,
+            keep_schema_type,
         )
     except HTTPError as e:
         console.print(f'[red]Error[/red]: {e}: {e.response.text}')
@@ -93,7 +116,9 @@ def export(
         service_url: str,
         collection: str,
         destination: Path,
+        output_format: str,
         ignore_errors: bool,
+        keep_schema_type: bool,
 ):
     token = obj
 
@@ -125,17 +150,17 @@ def export(
 
     console.print('Exporting records from curated area')
     _store_records(
-        map(
-            lambda x: x[0],
-            curated_read_records(
-                service_url=service_url,
-                collection=collection,
-                token=token,
-                session=session,
-            )
+        curated_read_records(
+            service_url=service_url,
+            collection=collection,
+            token=token,
+            session=session,
         ),
         curated_destination,
+        output_format,
         ignore_errors,
+        keep_schema_type,
+        source_name='curated area',
     )
 
     # Store the incoming records
@@ -149,55 +174,122 @@ def export(
         incoming_destination = destination / 'incoming' / label
         incoming_destination.mkdir(parents=True, exist_ok=False)
         _store_records(
-            map(
-                lambda x: x[0],
-                incoming_read_records(
-                    service_url=service_url,
-                    collection=collection,
-                    label=label,
-                    token=token,
-                    session=session,
-                )
+            incoming_read_records(
+                service_url=service_url,
+                collection=collection,
+                label=label,
+                token=token,
+                session=session,
             ),
             incoming_destination,
+            output_format,
             ignore_errors,
+            keep_schema_type,
+            source_name=f'incoming area: {label}'
         )
 
     return 0
 
 
 def _store_records(
-        source: Iterable,
+        source: Generator,
         destination: Path,
-        ignore_errors: bool = False,
+        output_format: str,
+        ignore_errors: bool,
+        keep_schema_type: bool,
+        source_name: str,
 ):
     created_dirs = set()
     class_counters = defaultdict(count)
 
-    for record in track(source, console=console):
-        class_name = _de_prefix(record.get('schema_type', None))
-        if class_name is None:
+    # Get the first result from the source to determine the total number
+    # of records.
+    try:
+        first_tuple = next(source)
+    except StopIteration:
+        console.print(f'no records in incoming [green]{source_name}[/green], skipping it')
+        return
+
+    total = first_tuple[4]
+    for record, _, _, _, _ in track(chain([first_tuple], source), total=total, console=console):
+        schema_type = record.get('schema_type', None)
+        if schema_type is None:
             if ignore_errors:
-                console.print(f'[red]Error[/red]: no `schema type` in record {record["pid"]}')
+                console.print(f'[red]Error[/red]: no `schema type` in record [red]{record["pid"]}[/red] in {source_name}')
                 continue
             msg = f'no `schema_type` in record {record["pid"]}'
             raise ValueError(msg)
 
-        next_name_for_class = f'{next(class_counters[class_name]):09d}.json'
+        class_name = _de_prefix(schema_type)
+        if not keep_schema_type:
+            del record['schema_type']
+
+        hash_dir, hash_name = _hash_p3(record['pid'])
         file_dir, file_name = (
-            destination / class_name / next_name_for_class[:3],
-            next_name_for_class[3:]
+            destination / class_name / hash_dir,
+            hash_name,
         )
         if file_dir not in created_dirs:
             file_dir.mkdir(parents=True, exist_ok=False)
             created_dirs.add(file_dir)
 
-        (file_dir / file_name).write_text(
-            json.dumps(record, indent=2, ensure_ascii=False),
-        )
+        try:
+            writer[output_format](
+                file_dir=file_dir,
+                file_name=file_name,
+                record=record,
+            )
+        except KeyError as e:
+            msg = f'unsupported output format: {output_format}'
+            raise ValueError(msg)
 
 
 def _de_prefix(
         name: str,
 ):
     return name.split(':', 1)[-1]
+
+
+def _get_hex_digest(
+        data: str,
+) -> str:
+    hash_context = hashlib.md5(data.encode())
+    return hash_context.hexdigest()
+
+
+def _hash_p3(
+        pid: str,
+) -> tuple[str, str]:
+    hex_digest = _get_hex_digest(pid)
+    return hex_digest[:3], hex_digest[3:]
+
+
+def write_json(
+        file_dir: Path,
+        file_name: str,
+        record: dict,
+):
+    (file_dir / (file_name + '.json')).write_text(
+        json.dumps(record, indent=2, ensure_ascii=False) + '\n',
+    )
+
+
+def write_yaml(
+        file_dir: Path,
+        file_name: str,
+        record: dict,
+):
+    (file_dir / (file_name + '.yaml')).write_text(
+        yaml.dump(
+            data=record,
+            sort_keys=False,
+            allow_unicode=True,
+            default_flow_style=False,
+        ),
+    )
+
+
+writer = {
+    'json': write_json,
+    'yaml': write_yaml,
+}

dump_things_pyclient/commands/dtc_plugins/get_records.py

@@ -68,13 +68,13 @@ console = Console(file=sys.stderr)
 @click.option(
     '--page-size', '-s',
     type=click.IntRange(1, 100),
-    default=100,
+    default=None,
     help='set the page size (default: 100). (ignored if "--pid" is provided)'
 )
 @click.option(
     '--first-page', '-F',
     type=click.INT,
-    default=1,
+    default=None,
     help='the first page to return (default: 1). (ignored if "--pid" is provided)'
 )
 @click.option(
@@ -206,9 +206,8 @@ def get_records(
                 (cls, '-c/--class'),
         ):
             if argument_value:
-                click.echo(
-                    f'WARNING: {argument_name} ignored because "-p/--pid" is provided',
-                    err=True,
+                console.print(
+                    f'[yellow]Warning[/yellow]: {argument_name} ignored because "-p/--pid" is provided',
                 )
 
         kwargs['pid'] = pid
@@ -242,7 +241,7 @@ def get_records(
     else:
         kwargs.update(dict(
             matching=matching,
-            page=first_page,
+            page=first_page or 1,
             size=page_size or 100,
             last_page=last_page,
         ))

{dump_things_pyclient-0.2.6.dist-info → dump_things_pyclient-0.2.8.dist-info}/METADATA

@@ -1,11 +1,12 @@
 Metadata-Version: 2.4
 Name: dump-things-pyclient
-Version: 0.2.6
+Version: 0.2.8
 Summary: A client library and CLI commands for dump-things-services
 Author-email: Christian Mönch <christian.moench@web.de>
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 Requires-Dist: click>=8.3.1
+Requires-Dist: pyyaml>=6.0.3
 Requires-Dist: requests>=2.32.5
 Requires-Dist: rich-click>=1.9.6
 Provides-Extra: ttl

{dump_things_pyclient-0.2.6.dist-info → dump_things_pyclient-0.2.8.dist-info}/RECORD

@@ -5,17 +5,17 @@ dump_things_pyclient/commands/dtc.py,sha256=dxW5RuogqwhzfVujZ_EEsQMk8BcVMbZyMdg5
 dump_things_pyclient/commands/json2ttl.py,sha256=8BkvdjLWZ_H0L6fTmuR2M2MglKiMUiuNUcuWr_w6_dQ,2133
 dump_things_pyclient/commands/redirect.py,sha256=kl8pGj8khjxk4lhk8AJLfgtCIm5PtjeMAl0J6K5FB7M,264
 dump_things_pyclient/commands/dtc_plugins/__init__.py,sha256=0YLByLiofhHkhJcDCkokldcCw3Jj0rsKJinRX4tt3Hc,514
-dump_things_pyclient/commands/dtc_plugins/auto_curate.py,sha256=dIpTJ4IqaORqMH0j-h7dr-7aoxjsJADNALpUKbVskcs,9997
+dump_things_pyclient/commands/dtc_plugins/auto_curate.py,sha256=fWylrzsrBWzJthWJNd_NAy3KSfxKfdFQun6EACpymnw,9968
 dump_things_pyclient/commands/dtc_plugins/clean_incoming.py,sha256=ikSPNTt254ax2tXhMK_gTgDCVkxMYJ_0NTAP8XsaRjk,2188
 dump_things_pyclient/commands/dtc_plugins/delete_records.py,sha256=SRQTHz4cWofI-RVx_p_mUex3amTaGZ9xP_S4F12Pw64,3849
-dump_things_pyclient/commands/dtc_plugins/export.py,sha256=s-uPWGLoZbyd4oyicNlk1_2f9MJWYDtSmLq7nTiKvE0,5232
-dump_things_pyclient/commands/dtc_plugins/get_records.py,sha256=5v3RUrNjEO4T5Ku0ZwGBy39dNxG5K-UcOwUigL8N4Vo,7615
+dump_things_pyclient/commands/dtc_plugins/export.py,sha256=Nbq-o1hq_6ZroBctKxzMyA1BSBnoqIkpSY8BO1aWuoA,7522
+dump_things_pyclient/commands/dtc_plugins/get_records.py,sha256=5yzkZFrYIlN6O-QmAZjr6L5SFhpncC6lgQaqgZzg_TE,7614
 dump_things_pyclient/commands/dtc_plugins/list_incoming.py,sha256=tmM0Qs4MVwMMLyERsWCxWGTM90rSNOShLpHH32wObd8,1959
 dump_things_pyclient/commands/dtc_plugins/maintenance.py,sha256=yTw1T_cvVTmwuzrTPteu6O6qiNCMxL5ZQoVF8yb72-M,1707
 dump_things_pyclient/commands/dtc_plugins/post_records.py,sha256=0676miD7VTMmokBAo7JdA9Dr9FZwhs0auNuRltulcBw,3469
 dump_things_pyclient/commands/dtc_plugins/read_pages.py,sha256=Libxf36L-0wUqAqfavotZPRMy5LjWJ37n_zSae1TgTA,3546
-dump_things_pyclient-0.2.6.dist-info/METADATA,sha256=9U_JvtQrlduHw2UZboU4uWfcbbAryqwBvnxCe3hoCg4,999
-dump_things_pyclient-0.2.6.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-dump_things_pyclient-0.2.6.dist-info/entry_points.txt,sha256=U1QhQtk767G_OXdZwPdTXYbIPfcDU13Z2u1d6exX8uE,470
-dump_things_pyclient-0.2.6.dist-info/top_level.txt,sha256=Asvruw-SyLoYhWis1CFOx89RGxpjXoTZVGoq4JSGt88,21
-dump_things_pyclient-0.2.6.dist-info/RECORD,,
+dump_things_pyclient-0.2.8.dist-info/METADATA,sha256=KAgBvgq-AlPkjmh-r5KwXkGNw9Wp0emCH3A8OCFNXJA,1028
+dump_things_pyclient-0.2.8.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+dump_things_pyclient-0.2.8.dist-info/entry_points.txt,sha256=U1QhQtk767G_OXdZwPdTXYbIPfcDU13Z2u1d6exX8uE,470
+dump_things_pyclient-0.2.8.dist-info/top_level.txt,sha256=Asvruw-SyLoYhWis1CFOx89RGxpjXoTZVGoq4JSGt88,21
+dump_things_pyclient-0.2.8.dist-info/RECORD,,