metadata-crawler 2509.0.1__tar.gz → 2509.0.2__tar.gz

{metadata_crawler-2509.0.1 → metadata_crawler-2509.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metadata-crawler
-Version: 2509.0.1
+Version: 2509.0.2
 Summary: Crawl, extract and push climate metadata for indexing.
 Author-email: "DKRZ, Clint" <freva@dkrz.de>
 Requires-Python: >=3.11
@@ -34,7 +34,7 @@ Requires-Dist: numpy
 Requires-Dist: orjson
 Requires-Dist: pyarrow
 Requires-Dist: h5netcdf
-Requires-Dist: pydantic
+Requires-Dist: pydantic<2.12
 Requires-Dist: pyarrow
 Requires-Dist: rich
 Requires-Dist: rich-argparse
@@ -96,10 +96,12 @@ Provides-Extra: tests
 
 [![License](https://img.shields.io/badge/License-BSD-purple.svg)](LICENSE)
 [![PyPI](https://img.shields.io/pypi/pyversions/metadata-crawler.svg)](https://pypi.org/project/metadata-crawler/)
+[![Conda Version](https://img.shields.io/conda/vn/conda-forge/metadata-crawler.svg)](https://anaconda.org/conda-forge/metadata-crawler)
 [![Docs](https://readthedocs.org/projects/metadata-crawler/badge/?version=latest)](https://metadata-crawler.readthedocs.io/en/latest/?badge=latest)
 [![Tests](https://github.com/freva-org/metadata-crawler/actions/workflows/ci_job.yml/badge.svg)](https://github.com/freva-org/metadata-crawler/actions)
 [![Test-Coverage](https://codecov.io/gh/freva-org/metadata-crawler/graph/badge.svg?token=W2YziDnh2N)](https://codecov.io/gh/freva-org/metadata-crawler)
 
+
 Harvest, normalise, and index climate / earth-system metadata from **POSIX**,
 **S3/MinIO**, and **OpenStack Swift** using configurable **DRS dialects**
 (CMIP6, CMIP5, CORDEX, …). Output to a temporary **catalogue** (JSONLines)

{metadata_crawler-2509.0.1 → metadata_crawler-2509.0.2}/README.md

@@ -2,10 +2,12 @@
 
 [![License](https://img.shields.io/badge/License-BSD-purple.svg)](LICENSE)
 [![PyPI](https://img.shields.io/pypi/pyversions/metadata-crawler.svg)](https://pypi.org/project/metadata-crawler/)
+[![Conda Version](https://img.shields.io/conda/vn/conda-forge/metadata-crawler.svg)](https://anaconda.org/conda-forge/metadata-crawler)
 [![Docs](https://readthedocs.org/projects/metadata-crawler/badge/?version=latest)](https://metadata-crawler.readthedocs.io/en/latest/?badge=latest)
 [![Tests](https://github.com/freva-org/metadata-crawler/actions/workflows/ci_job.yml/badge.svg)](https://github.com/freva-org/metadata-crawler/actions)
 [![Test-Coverage](https://codecov.io/gh/freva-org/metadata-crawler/graph/badge.svg?token=W2YziDnh2N)](https://codecov.io/gh/freva-org/metadata-crawler)
 
+
 Harvest, normalise, and index climate / earth-system metadata from **POSIX**,
 **S3/MinIO**, and **OpenStack Swift** using configurable **DRS dialects**
 (CMIP6, CMIP5, CORDEX, …). Output to a temporary **catalogue** (JSONLines)

{metadata_crawler-2509.0.1 → metadata_crawler-2509.0.2}/docs/source/whatsnew.rst

@@ -3,6 +3,16 @@ What's new
 
 This document highlights major changes and additions across releases.
 
+v2509.0.2
+----------
+* Display progressbar for ingestion.
+* Improved logging.
+* Fix S3 "flat directory bug".
+* Drop dataset versioning if crawl path if past version position.
+* Fix "time" bug.
+* Add an optional index suffix for solr cores and mongo collections.
+* Add Multi threaded solr indexing.
+
 v2509.0.1
 ----------
 

{metadata_crawler-2509.0.1 → metadata_crawler-2509.0.2}/drs_config.toml

@@ -74,9 +74,6 @@ bbox = [0, 360, -60, 60]
 level_type = "2d"
 model = "sat"
 
-
-
-
 #[foo]
 #root_path = "./data/observations"
 #drs_format = "foo"

{metadata_crawler-2509.0.1 → metadata_crawler-2509.0.2}/pyproject.toml

@@ -41,7 +41,7 @@ dependencies = [
         "orjson",
         "pyarrow",
         "h5netcdf",
-        "pydantic",
+        "pydantic<2.12",
         "pyarrow",
         "rich",
         "rich-argparse",
@@ -230,11 +230,13 @@ deps = .
        flake8
        codespell
        pydocstyle
+       bandit[toml]
 commands =
     python3 -m isort --check --profile black -t py313 -l 79 src
     python3 -m flake8 src --count --max-complexity=10 --ignore=E203,F405,F403,E704,W503,C901 --max-line-length=88 --statistics --show-source
     codespell {posargs: --quiet-level=2} src/metadata_crawler
     python3 -m pydocstyle src/metadata_crawler
+    bandit -r src/metadata_crawler --skip B110 --severity-level medium --confidence-level high
 [testenv:types]
 deps = .
        mypy

{metadata_crawler-2509.0.1 → metadata_crawler-2509.0.2}/src/metadata_crawler/__init__.py

@@ -53,6 +53,7 @@ def index(
     *catalogue_files: Union[Path, str, List[str], List[Path]],
     batch_size: int = 2500,
     verbosity: int = 0,
+    log_suffix: Optional[str] = None,
     **kwargs: Any,
 ) -> None:
     """Index metadata in the indexing system.
@@ -68,6 +69,8 @@ def index(
         If the index system supports batch-sizes, the size of the batches.
     verbosity:
         Set the verbosity level.
+    log_suffix:
+        Add a suffix to the log file output.
 
     Other Parameters
     ^^^^^^^^^^^^^^^^
@@ -94,6 +97,7 @@ def index(
             *catalogue_files,
             batch_size=batch_size,
             verbosity=verbosity,
+            log_suffix=log_suffix,
             **kwargs,
         )
     )
@@ -103,6 +107,7 @@ def delete(
     index_system: str,
     batch_size: int = 2500,
     verbosity: int = 0,
+    log_suffix: Optional[str] = None,
     **kwargs: Any,
 ) -> None:
     """Delete metadata from the indexing system.
@@ -116,6 +121,8 @@ def delete(
         If the index system supports batch-sizes, the size of the batches.
     verbosity:
         Set the verbosity of the system.
+    log_suffix:
+        Add a suffix to the log file output.
 
     Other Parameters
     ^^^^^^^^^^^^^^^^
@@ -135,7 +142,11 @@ def delete(
             facets=[("project", "CMIP6"), ("institute", "MPI-M")],
         )
     """
-    uvloop.run(async_delete(index_system, batch_size=batch_size, **kwargs))
+    uvloop.run(
+        async_delete(
+            index_system, batch_size=batch_size, log_suffix=log_suffix, **kwargs
+        )
+    )
 
 
 def add(
@@ -155,6 +166,7 @@ def add(
     all_versions: str = IndexName().all,
     n_procs: Optional[int] = None,
     verbosity: int = 0,
+    log_suffix: Optional[str] = None,
     password: bool = False,
     fail_under: int = -1,
     **kwargs: Any,
@@ -204,6 +216,8 @@ def add(
         Set the number of parallel processes for collecting.
     verbosity:
         Set the verbosity of the system.
+    log_suffix:
+        Add a suffix to the log file output.
     fail_under:
          Fail if less than X of the discovered files could be indexed.
 
@@ -242,6 +256,7 @@ def add(
             n_procs=n_procs,
             storage_options=storage_options,
             verbosity=verbosity,
+            log_suffix=log_suffix,
             fail_under=fail_under,
             **kwargs,
         )

metadata_crawler-2509.0.2/src/metadata_crawler/_version.py

@@ -0,0 +1 @@
+__version__ = "2509.0.2"

{metadata_crawler-2509.0.1 → metadata_crawler-2509.0.2}/src/metadata_crawler/api/config.py

@@ -17,6 +17,7 @@ from typing import (
     List,
     Literal,
     Optional,
+    Tuple,
     Union,
     cast,
 )
@@ -285,7 +286,6 @@ class PathSpecs(BaseModel):
                     f"- needs: {len(self.file_parts)} has: {len(file_parts)})"
                 )
             )
-        _parts.setdefault("time", "fx")
         data.update({k: v for (k, v) in _parts.items() if k not in data})
         data.pop("_", None)
         return data
@@ -609,7 +609,9 @@ class DRSConfig(BaseModel, TemplateMixin):
                 case "conditional":
                     _rule = textwrap.dedent(rule.condition or "").strip()
                     s_cond = self.render_templates(_rule, data)
-                    cond = eval(s_cond, {}, getattr(self, "_model_dict", {}))
+                    cond = eval(
+                        s_cond, {}, getattr(self, "_model_dict", {})
+                    )  # nosec
                     result = rule.true if cond else rule.false
                 case "lookup":
                     args = cast(List[str], self.render_templates(rule.tree, data))
@@ -627,7 +629,7 @@ class DRSConfig(BaseModel, TemplateMixin):
                         self.render_templates(_call, data),
                         {},
                         getattr(self, "_model_dict", {}),
-                    )
+                    )  # nosec
             if result:
                 inp.metadata[facet] = result
 
@@ -666,7 +668,7 @@ class DRSConfig(BaseModel, TemplateMixin):
 
     def max_directory_tree_level(
         self, search_dir: str | Path, drs_type: str
-    ) -> int:
+    ) -> Tuple[int, bool]:
         """Get the maximum level for descending into directories.
 
         When searching for files in a directory we can only traverse the directory
@@ -686,6 +688,7 @@ class DRSConfig(BaseModel, TemplateMixin):
         version = cast(
             str, self.dialect[standard].facets.get("version", "version")
         )
+        is_versioned = True
         try:
             version_idx = self.dialect[standard].path_specs.dir_parts.index(
                 version
@@ -693,11 +696,12 @@ class DRSConfig(BaseModel, TemplateMixin):
         except ValueError:
             # No version given
             version_idx = len(self.dialect[standard].path_specs.dir_parts)
+            is_versioned = False
         if root_path == search_dir:
             current_pos = 0
         else:
             current_pos = len(search_dir.relative_to(root_path).parts)
-        return version_idx - current_pos
+        return version_idx - current_pos, is_versioned
 
     def is_complete(self, data: Dict[str, Any], standard: str) -> bool:
         """Check if all metadata that can be collected was collected."""

{metadata_crawler-2509.0.1 → metadata_crawler-2509.0.2}/src/metadata_crawler/api/drs_config.toml

@@ -78,6 +78,7 @@ multi_valued = true
 key         = "time"
 type        = "daterange"
 multi_valued = false
+default     = "fx"
 
 [drs_settings.index_schema.grid_label]
 key         = "grid_label"

{metadata_crawler-2509.0.1 → metadata_crawler-2509.0.2}/src/metadata_crawler/api/index.py

@@ -16,7 +16,7 @@ from typing import (
 )
 
 from ..logger import logger
-from ..utils import Console
+from ..utils import Console, IndexProgress
 from .config import SchemaField
 from .metadata_stores import CatalogueReader, IndexStore
 
@@ -40,6 +40,9 @@ class BaseIndex:
     batch_size:
         The amount for metadata that should be gathered `before` ingesting
         it into the catalogue.
+    progress:
+        Optional rich progress object that should display the progress of the
+        tasks.
 
     Attributes
     ^^^^^^^^^^
@@ -50,9 +53,11 @@ class BaseIndex:
         catalogue_file: Optional[Union[str, Path]] = None,
         batch_size: int = 2500,
         storage_options: Optional[Dict[str, Any]] = None,
+        progress: Optional[IndexProgress] = None,
         **kwargs: Any,
     ) -> None:
         self._store: Optional[IndexStore] = None
+        self.progress = progress or IndexProgress(total=-1)
         if catalogue_file is not None:
             _reader = CatalogueReader(
                 catalogue_file=catalogue_file or "",
@@ -92,6 +97,7 @@ class BaseIndex:
             logger.info("Indexing %s", index_name)
             async for batch in self._store.read(index_name):
                 yield batch
+                self.progress.update(len(batch))
                 num_items += len(batch)
             msg = f"Indexed {num_items:10,.0f} items for index {index_name}"
             Console.print(msg) if Console.is_terminal else print(msg)

{metadata_crawler-2509.0.1 → metadata_crawler-2509.0.2}/src/metadata_crawler/api/metadata_stores.py

@@ -473,10 +473,7 @@ class CatalogueReader:
     ) -> None:
         catalogue_file = str(catalogue_file)
         storage_options = storage_options or {}
-        fs, _ = IndexStore.get_fs(catalogue_file, **storage_options)
-        path = fs.unstrip_protocol(catalogue_file)
-        with fs.open(path) as stream:
-            cat = yaml.safe_load(stream.read())
+        cat = self.load_catalogue(catalogue_file, **storage_options)
         _schema_json = cat["metadata"]["schema"]
         schema = {s["key"]: SchemaField(**s) for k, s in _schema_json.items()}
         index_name = IndexName(**cat["metadata"]["index_names"])
@@ -493,6 +490,14 @@ class CatalogueReader:
             storage_options=storage_options,
         )
 
+    @staticmethod
+    def load_catalogue(path: Union[str, Path], **storage_options: Any) -> Any:
+        """Load a intake yaml catalogue (remote or local)."""
+        fs, _ = IndexStore.get_fs(str(path), **storage_options)
+        cat_path = fs.unstrip_protocol(path)
+        with fs.open(cat_path) as stream:
+            return yaml.safe_load(stream.read())
+
 
 class QueueConsumer:
     """Class that consumes the file discovery queue."""
@@ -722,6 +727,7 @@ class CatalogueWriter:
                     "latest": self.index_name.latest,
                     "all": self.index_name.all,
                 },
+                "indexed_objects": self.ingested_objects,
                 "schema": {
                     k: json.loads(s.model_dump_json())
                     for k, s in self.store.schema.items()

{metadata_crawler-2509.0.1 → metadata_crawler-2509.0.2}/src/metadata_crawler/api/mixin/template_mixin.py

@@ -6,7 +6,7 @@ from typing import Any, Dict, Mapping, Optional
 
 from jinja2 import Environment, Template, Undefined
 
-ENV = Environment(undefined=Undefined, autoescape=False)
+ENV = Environment(undefined=Undefined, autoescape=True)
 
 
 @lru_cache(maxsize=1024)

{metadata_crawler-2509.0.1 → metadata_crawler-2509.0.2}/src/metadata_crawler/backends/s3.py

@@ -71,11 +71,15 @@ class S3Path(PathTemplate):
         self, path: Union[str, Path, pathlib.Path]
     ) -> AsyncIterator[str]:
         """Retrieve sub directories of directory."""
-        path = str(path)
         client = await self._get_client()
-        for _content in await client._lsdir(path):
-            if _content.get("type", "") == "directory":
-                yield f'{_content.get("name", "")}'
+        path = str(path)
+        if await self.is_file(path):
+            yield path
+        else:
+            for _content in await client._lsdir(path):
+                size: int = _content.get("size") or 0
+                if _content.get("type", "") == "directory" or size > 0:
+                    yield _content.get("name", "")
 
     async def rglob(
         self, path: str | Path | pathlib.Path, glob_pattern: str = "*"

{metadata_crawler-2509.0.1 → metadata_crawler-2509.0.2}/src/metadata_crawler/cli.py

@@ -34,7 +34,6 @@ from .api.metadata_stores import CatalogueBackends, IndexName
 from .backends.intake import IntakePath
 from .logger import (
     THIS_NAME,
-    add_file_handle,
     apply_verbosity,
     logger,
 )
@@ -48,7 +47,9 @@ KwargValue = Union[
 
 
 def walk_catalogue(
-    path: str, storage_options: Optional[Dict[str, Any]] = None, **kwargs: Any
+    path: str,
+    storage_options: Optional[Dict[str, Any]] = None,
+    **kwargs: Any,
 ) -> int:
     """Recursively traverse an intake catalogue.
 
@@ -359,6 +360,13 @@ class ArgParse:
             action="append",
             nargs=2,
         )
+        parser.add_argument(
+            "-v",
+            "--verbose",
+            action="count",
+            default=self.verbose,
+            help="Increase the verbosity level.",
+        )
         parser.set_defaults(apply_func=walk_catalogue)
 
     def _index_submcommands(self) -> None:
@@ -391,8 +399,8 @@ class ArgParse:
                     "-b",
                     "--batch-size",
                     type=int,
-                    default=25_000,
-                    help="Set the batch size for ingestion.",
+                    default=5_000,
+                    help="Set the batch size for indexing.",
                 )
                 parser.add_argument(
                     "--storage_option",
@@ -494,7 +502,6 @@ class ArgParse:
                 "apply_func",
                 "verbose",
                 "version",
-                "log_suffix",
                 "storage_option",
                 "shadow",
             )
@@ -509,7 +516,6 @@ class ArgParse:
             self.kwargs["shadow"] = _flatten(args.shadow)
         self.kwargs["storage_options"] = so
         self.verbose = args.verbose
-        add_file_handle(args.log_suffix)
         self.kwargs["verbosity"] = self.verbose
         return args
 
@@ -519,7 +525,9 @@ def _run(
     **kwargs: KwargValue,
 ) -> None:
     """Apply the parsed method."""
-    old_level = apply_verbosity(getattr(parser, "verbose", 0))
+    old_level = apply_verbosity(
+        getattr(parser, "verbose", 0), suffix=getattr(parser, "log_suffix", None)
+    )
     try:
         parser.apply_func(**kwargs)
     except Exception as error:

{metadata_crawler-2509.0.1 → metadata_crawler-2509.0.2}/src/metadata_crawler/data_collector.py

@@ -15,6 +15,7 @@ from typing import (
     Dict,
     Iterator,
     Optional,
+    Tuple,
     Type,
     Union,
     cast,
@@ -33,7 +34,7 @@ from .utils import (
     print_performance,
 )
 
-ScanItem = tuple[str, str, bool]
+ScanItem = Tuple[str, str, bool, bool]
 
 
 class DataCollector:
@@ -138,6 +139,7 @@ class DataCollector:
         drs_type: str,
         search_dir: str,
         iterable: bool = True,
+        is_versioned: bool = True,
     ) -> None:
         if iterable:
             try:
@@ -161,7 +163,7 @@ class DataCollector:
                 await self.ingest_queue.put(
                     _inp, drs_type, name=self.index_name.all
                 )
-                if rank == 0:
+                if rank == 0 or is_versioned is False:
                     await self.ingest_queue.put(
                         _inp, drs_type, name=self.index_name.latest
                     )
@@ -176,16 +178,22 @@ class DataCollector:
             if item is None:  # sentinel -> exit
                 # do not task_done() for sentinel
                 break
-            drs_type, path, iterable = item
+            drs_type, path, iterable, is_versioned = item
             try:
-                await self._ingest_dir(drs_type, path, iterable=iterable)
+                await self._ingest_dir(
+                    drs_type, path, iterable=iterable, is_versioned=is_versioned
+                )
             except Exception as error:
                 logger.error(error)
             finally:
                 self._scan_queue.task_done()
 
     async def _iter_content(
-        self, drs_type: str, inp_dir: str, pos: int = 0
+        self,
+        drs_type: str,
+        inp_dir: str,
+        pos: int = 0,
+        is_versioned: bool = True,
     ) -> None:
         """Walk recursively until files or the version level is reached."""
         store = self.config.datasets[drs_type].backend
@@ -203,7 +211,6 @@ class DataCollector:
 
         iterable = False if suffix == ".zarr" else iterable
         op: Optional[Callable[..., Coroutine[Any, Any, None]]] = None
-
         if is_file and suffix in self.config.suffixes:
             op = self._ingest_dir
         elif pos <= 0 or suffix == ".zarr":
@@ -211,13 +218,17 @@ class DataCollector:
 
         if op is not None:
             # enqueue the heavy scan; workers will run _ingest_dir concurrently
-            await self._scan_queue.put((drs_type, inp_dir, iterable))
+            await self._scan_queue.put(
+                (drs_type, inp_dir, iterable, is_versioned)
+            )
             return
 
         # otherwise, recurse sequentially (cheap) — no task per directory
         try:
             async for sub in store.iterdir(inp_dir):
-                await self._iter_content(drs_type, sub, pos - 1)
+                await self._iter_content(
+                    drs_type, sub, pos - 1, is_versioned=is_versioned
+                )
         except Exception as error:
             logger.error(error)
 
@@ -239,10 +250,19 @@ class DataCollector:
 
             # produce scan items by walking roots sequentially
             for drs_type, path in self.search_objects:  # <- property is sync
-                pos = self.config.max_directory_tree_level(
+                pos, is_versioned = self.config.max_directory_tree_level(
                     path, drs_type=drs_type
                 )
-                await self._iter_content(drs_type, path, pos)
+                if pos < 0:
+                    logger.warning(
+                        "Can't define latest version of versioned dataset."
+                        " This might lead to unexpected results. Try adjusting"
+                        " your search path."
+                    )
+
+                await self._iter_content(
+                    drs_type, path, pos, is_versioned=is_versioned
+                )
 
             # wait until all queued scan items are processed
             await self._scan_queue.join()

{metadata_crawler-2509.0.1 → metadata_crawler-2509.0.2}/src/metadata_crawler/ingester/mongo.py

@@ -80,12 +80,13 @@ class MongoIndex(BaseIndex):
         await collection.bulk_write(ops, ordered=False)
 
     async def _index_collection(
-        self, db: AsyncIOMotorDatabase[Any], collection: str
+        self, db: AsyncIOMotorDatabase[Any], collection: str, suffix: str = ""
     ) -> None:
         """Index a collection."""
-        await db[collection].create_index(self.unique_index, unique=True)
+        col = collection + suffix
+        await db[col].create_index(self.unique_index, unique=True)
         async for chunk in self.get_metadata(collection):
-            await self._bulk_upsert(chunk, db[collection])
+            await self._bulk_upsert(chunk, db[col])
 
     async def _prep_db_connection(
         self, database: str, url: str
@@ -119,12 +120,24 @@ class MongoIndex(BaseIndex):
                 default="metadata",
             ),
         ] = "metadata",
+        index_suffix: Annotated[
+            Optional[str],
+            cli_parameter(
+                "--index-suffix",
+                help="Suffix for the latest and all version collections.",
+                type=str,
+            ),
+        ] = None,
     ) -> None:
         """Add metadata to the mongoDB metadata server."""
         db = await self._prep_db_connection(database, url or "")
         async with asyncio.TaskGroup() as tg:
             for collection in self.index_names:
-                tg.create_task(self._index_collection(db, collection))
+                tg.create_task(
+                    self._index_collection(
+                        db, collection, suffix=index_suffix or ""
+                    )
+                )
 
     async def close(self) -> None:
         """Close the mongoDB connection."""