geometamaker 0.1.2__py3-none-any.whl → 0.2.0__py3-none-any.whl

geometamaker/__init__.py

@@ -1,7 +1,7 @@
 import importlib.metadata
 
 from .geometamaker import describe
-from .geometamaker import describe_dir
+from .geometamaker import describe_collection
 from .geometamaker import validate
 from .geometamaker import validate_dir
 from .config import Config
@@ -10,4 +10,4 @@ from .models import Profile
 
 __version__ = importlib.metadata.version('geometamaker')
 
-__all__ = ('describe', 'describe_dir', 'validate', 'validate_dir', 'Config', 'Profile')
+__all__ = ('describe', 'describe_collection', 'validate', 'validate_dir', 'Config', 'Profile')

geometamaker/cli.py

@@ -3,45 +3,147 @@ import os
 import sys
 
 import click
+import fsspec
+import numpy
 from pydantic import ValidationError
 
 import geometamaker
 
-ROOT_LOGGER = logging.getLogger()
-ROOT_LOGGER.setLevel(logging.DEBUG)
+LOGGER = logging.getLogger('geometamaker')
+LOGGER.setLevel(logging.DEBUG)
 HANDLER = logging.StreamHandler(sys.stdout)
 FORMATTER = logging.Formatter(
     fmt='%(asctime)s %(name)-18s %(levelname)-8s %(message)s',
     datefmt='%m/%d/%Y %H:%M:%S ')
 HANDLER.setFormatter(FORMATTER)
+LOGGER.addFilter(
+    lambda record: not record.__dict__.get(
+        geometamaker.geometamaker._NOT_FOR_CLI, False))
+
+
+# The recommended approach to allowing multiple ParamTypes
+# https://github.com/pallets/click/issues/1729
+class _ParamUnion(click.ParamType):
+    def __init__(self, types, report_all_errors=True):
+        """Union of click.ParamTypes.
+
+        Args:
+            types (list): List of click.ParamTypes to try to convert the value.
+            report_all_errors (bool): If True, all errors will be reported.
+                If False, only the last error will be reported.
+
+        """
+        self.types = types
+        self.report_all_errors = report_all_errors
+
+    def convert(self, value, param, ctx):
+        errors = []
+        for type_ in self.types:
+            try:
+                return type_.convert(value, param, ctx)
+            except click.BadParameter as e:
+                errors.append(e)
+                continue
+
+        if self.report_all_errors:
+            self.fail(errors)
+        else:
+            # If errors from different types are expected to
+            # be very similar, just report the last one.
+            self.fail(errors.pop())
+
+
+# https://click.palletsprojects.com/en/stable/parameters/#how-to-implement-custom-types
+class _URL(click.ParamType):
+    """A type that asserts a URL exists."""
+
+    name = "url"
+
+    def convert(self, value, param, ctx):
+        of = fsspec.open(value)
+        if not of.fs.exists(value):
+            self.fail(f'{value} does not exist', param, ctx)
+
+        return value
 
 
 @click.command(
     help='''Describe properties of a dataset given by FILEPATH and write this
     metadata to a .yml sidecar file. Or if FILEPATH is a directory, describe
     all datasets within.''',
-    short_help='Generate metadata for geospatial or tabular data, or zip archives.')
-@click.argument('filepath', type=click.Path(exists=True))
-@click.option('-r', '--recursive', is_flag=True, default=False,
-              help='if FILEPATH is a directory, describe files '
-                   'in all subdirectories')
-@click.option('-nw', '--no-write', is_flag=True, default=False,
-              help='Dump metadata to stdout instead of to a .yml file. '
-                   'This option is ignored if `filepath` is a directory')
-def describe(filepath, recursive, no_write):
+    short_help='Generate metadata for geospatial or tabular data, compressed'
+               ' archives, or collections of files in a directory.')
+@click.argument('filepath',
+                type=_ParamUnion([click.Path(exists=True), _URL()],
+                                 report_all_errors=False))
+@click.option('-nw', '--no-write',
+              is_flag=True,
+              default=False,
+              help='Dump metadata to stdout instead of to a .yml file.'
+                   ' This option is ignored when describing all files'
+                   ' in a directory.')
+@click.option('-st', '--stats',
+              is_flag=True,
+              default=False,
+              help='Compute raster band statistics.')
+@click.option('-d', '--depth',
+              default=numpy.iinfo(numpy.int16).max,
+              help='if FILEPATH is a directory, describe files in'
+                   ' subdirectories up to depth. Defaults to describing'
+                   ' all files.')
+@click.option('-x', '--exclude',
+              default=None,
+              help='Regular expression used to exclude files from being'
+                   ' described. Only used if FILEPATH is a directory.')
+@click.option('-a', '--all', 'all_files',
+              is_flag=True,
+              default=False,
+              help='Do not ignore files starting with .'
+                   ' Only used if FILEPATH is a directory.')
+@click.option('-co', '--collection-only',
+              is_flag=True,
+              default=False,
+              help='If FILEPATH is a directory, do not write metadata documents'
+                   ' for all files in the directory. Only create a single'
+                   ' *-metadata.yml document for the collection')
+def describe(filepath, depth, exclude, all_files, no_write, stats,
+             collection_only):
+    describing_single = True  # if filepath is a file, or collection_only=True
     if os.path.isdir(filepath):
-        if no_write:
-            click.echo('the -nw, or --no-write, flag is ignored when '
-                       'describing all files in a directory.')
-        geometamaker.describe_dir(
-            filepath, recursive=recursive)
+        resource = geometamaker.describe_collection(
+            filepath,
+            depth=depth,
+            exclude_regex=exclude,
+            exclude_hidden=(not all_files),
+            describe_files=(not collection_only),
+            compute_stats=stats)
+        describing_single = collection_only
     else:
-        resource = geometamaker.describe(filepath)
-        if no_write:
-            click.echo(geometamaker.utils.yaml_dump(
-                resource.model_dump(exclude=['metadata_path'])))
-        else:
-            resource.write()
+        resource = geometamaker.describe(filepath, compute_stats=stats)
+
+    if no_write and describing_single:
+        click.echo(geometamaker.utils.yaml_dump(
+            resource._dump_for_write()))
+        return
+
+    if no_write and not describing_single:
+        click.echo('the -nw, or --no-write, flag is ignored when '
+                   'describing all files in a directory.')
+    if resource._would_overwrite:
+        click.confirm(
+            f'\n{resource.metadata_path} is about to be overwritten'
+            ' because it is not a valid metadata document.\n'
+            'Are you sure want to continue?',
+            abort=True)
+    try:
+        # Users can abort at the confirm and manage their own backups.
+        resource.write(backup=False)
+    except OSError:
+        click.echo(
+            f'geometamaker could not write to {resource.metadata_path}\n'
+            'Try using the --no-write flag to print metadata to '
+            'stdout instead:')
+        click.echo(f'    geometamaker describe --no-write {filepath}')
 
 
 def echo_validation_error(error, filepath):
@@ -59,14 +161,17 @@ def echo_validation_error(error, filepath):
     help='''Validate a .yml metadata document given by FILEPATH.
     Or if FILEPATH is a directory, validate all documents within.''',
     short_help='Validate metadata documents for syntax or type errors.')
-@click.argument('filepath', type=click.Path(exists=True))
-@click.option('-r', '--recursive', is_flag=True, default=False,
-              help='if `filepath` is a directory, validate documents '
-                   'in all subdirectories.')
-def validate(filepath, recursive):
+@click.argument('filepath',
+                type=click.Path(exists=True))
+@click.option('-d', '--depth',
+              default=numpy.iinfo(numpy.int16).max,
+              help='if FILEPATH is a directory, validate files in'
+                   ' subdirectories up to depth. Defaults to validating'
+                   ' all files.')
+def validate(filepath, depth):
     if os.path.isdir(filepath):
         file_list, message_list = geometamaker.validate_dir(
-            filepath, recursive=recursive)
+            filepath, depth=depth)
         for filepath, msg in zip(file_list, message_list):
             if isinstance(msg, ValidationError):
                 echo_validation_error(msg, filepath)
@@ -96,7 +201,7 @@ def delete_config(ctx, param, value):
         return
     config = geometamaker.Config()
     click.confirm(
-        f'Are you sure you want to delete {config.config_path}?',
+        f'\nAre you sure you want to delete {config.config_path}?',
         abort=True)
     config.delete()
     ctx.exit()
@@ -141,7 +246,8 @@ def config(individual_name, email, organization, position_name,
     click.echo(f'saved profile information to {config.config_path}')
 
 
-@click.group()
+@click.group(
+    epilog='https://geometamaker.readthedocs.io/en/latest/ for more details')
 @click.option('-v', 'verbosity', count=True, default=2, required=False,
               help='''Override the default verbosity of logging. Use "-vvv" for
               debug-level logging. Omit this flag for default,
@@ -150,7 +256,7 @@ def config(individual_name, email, organization, position_name,
 def cli(verbosity):
     log_level = logging.ERROR - verbosity*10
     HANDLER.setLevel(log_level)
-    ROOT_LOGGER.addHandler(HANDLER)
+    LOGGER.addHandler(HANDLER)
 
 
 cli.add_command(describe)

geometamaker/config.py

@@ -7,7 +7,7 @@ from pydantic import ValidationError
 from . import models
 
 
-LOGGER = logging.getLogger(__name__)
+LOGGER = logging.getLogger('geometamaker')
 
 CONFIG_FILENAME = 'geometamaker_profile.yml'
 
@@ -33,9 +33,8 @@ class Config(object):
 
         try:
             self.profile = models.Profile.load(self.config_path)
-        except FileNotFoundError as err:
-            LOGGER.debug('config file does not exist', exc_info=err)
-            pass
+        except FileNotFoundError:
+            LOGGER.debug(f'config file does not exist at {self.config_path}')
         # an invalid profile should raise a ValidationError
         except ValidationError as err:
             LOGGER.warning('', exc_info=err)