metadata-crawler 2509.0.1__py3-none-any.whl → 2510.0.0__py3-none-any.whl

metadata_crawler/__init__.py

@@ -53,6 +53,7 @@ def index(
     *catalogue_files: Union[Path, str, List[str], List[Path]],
     batch_size: int = 2500,
     verbosity: int = 0,
+    log_suffix: Optional[str] = None,
     **kwargs: Any,
 ) -> None:
     """Index metadata in the indexing system.
@@ -68,6 +69,8 @@ def index(
         If the index system supports batch-sizes, the size of the batches.
     verbosity:
         Set the verbosity level.
+    log_suffix:
+        Add a suffix to the log file output.
 
     Other Parameters
     ^^^^^^^^^^^^^^^^
@@ -94,6 +97,7 @@ def index(
             *catalogue_files,
             batch_size=batch_size,
             verbosity=verbosity,
+            log_suffix=log_suffix,
             **kwargs,
         )
     )
@@ -103,6 +107,7 @@ def delete(
     index_system: str,
     batch_size: int = 2500,
     verbosity: int = 0,
+    log_suffix: Optional[str] = None,
     **kwargs: Any,
 ) -> None:
     """Delete metadata from the indexing system.
@@ -116,6 +121,8 @@ def delete(
         If the index system supports batch-sizes, the size of the batches.
     verbosity:
         Set the verbosity of the system.
+    log_suffix:
+        Add a suffix to the log file output.
 
     Other Parameters
     ^^^^^^^^^^^^^^^^
@@ -135,7 +142,11 @@ def delete(
             facets=[("project", "CMIP6"), ("institute", "MPI-M")],
         )
     """
-    uvloop.run(async_delete(index_system, batch_size=batch_size, **kwargs))
+    uvloop.run(
+        async_delete(
+            index_system, batch_size=batch_size, log_suffix=log_suffix, **kwargs
+        )
+    )
 
 
 def add(
@@ -155,6 +166,7 @@ def add(
     all_versions: str = IndexName().all,
     n_procs: Optional[int] = None,
     verbosity: int = 0,
+    log_suffix: Optional[str] = None,
     password: bool = False,
     fail_under: int = -1,
     **kwargs: Any,
@@ -204,6 +216,8 @@ def add(
         Set the number of parallel processes for collecting.
     verbosity:
         Set the verbosity of the system.
+    log_suffix:
+        Add a suffix to the log file output.
     fail_under:
          Fail if less than X of the discovered files could be indexed.
 
@@ -242,6 +256,7 @@ def add(
             n_procs=n_procs,
             storage_options=storage_options,
             verbosity=verbosity,
+            log_suffix=log_suffix,
             fail_under=fail_under,
             **kwargs,
         )

metadata_crawler/_version.py

@@ -1 +1 @@
-__version__ = "2509.0.1"
+__version__ = "2510.0.0"

metadata_crawler/api/config.py

@@ -17,6 +17,7 @@ from typing import (
     List,
     Literal,
     Optional,
+    Tuple,
     Union,
     cast,
 )
@@ -255,23 +256,31 @@ class CrawlerSettings(BaseModel):
 class PathSpecs(BaseModel):
     """Implementation of the Directory reference syntax."""
 
-    dir_parts: List[str] = Field(default_factory=list)
-    file_parts: List[str] = Field(default_factory=list)
+    dir_parts: Optional[List[str]] = None
+    file_parts: Optional[List[str]] = None
     file_sep: str = "_"
 
-    def get_metadata_from_path(self, rel_path: Path) -> Dict[str, Any]:
-        """Read path encoded metadata from path specs."""
+    def _get_metadata_from_dir(
+        self, data: Dict[str, Any], rel_path: Path
+    ) -> None:
         dir_parts = rel_path.parent.parts
-        file_parts = rel_path.name.split(self.file_sep)
-        if len(dir_parts) == len(self.dir_parts):
-            data: Dict[str, Any] = dict(zip(self.dir_parts, dir_parts))
-        else:
+        if self.dir_parts and len(dir_parts) == len(self.dir_parts):
+            _parts = dict(zip(self.dir_parts, dir_parts))
+        elif self.dir_parts:
             raise MetadataCrawlerException(
                 (
                     f"Number of dir parts for {rel_path.parent} do not match "
                     f"- needs: {len(self.dir_parts)} has: {len(dir_parts)}"
                 )
             ) from None
+        data.update({k: v for (k, v) in _parts.items() if k not in data})
+
+    def _get_metadata_from_filename(
+        self, data: Dict[str, Any], rel_path: Path
+    ) -> None:
+        if self.file_parts is None:
+            return
+        file_parts = rel_path.name.split(self.file_sep)
         if len(file_parts) == len(self.file_parts):
             _parts = dict(zip(self.file_parts, file_parts))
         elif (
@@ -285,8 +294,13 @@ class PathSpecs(BaseModel):
                     f"- needs: {len(self.file_parts)} has: {len(file_parts)})"
                 )
             )
-        _parts.setdefault("time", "fx")
         data.update({k: v for (k, v) in _parts.items() if k not in data})
+
+    def get_metadata_from_path(self, rel_path: Path) -> Dict[str, Any]:
+        """Read path encoded metadata from path specs."""
+        data: Dict[str, Any] = {}
+        self._get_metadata_from_dir(data, rel_path)
+        self._get_metadata_from_filename(data, rel_path)
         data.pop("_", None)
         return data
 
@@ -609,7 +623,9 @@ class DRSConfig(BaseModel, TemplateMixin):
                 case "conditional":
                     _rule = textwrap.dedent(rule.condition or "").strip()
                     s_cond = self.render_templates(_rule, data)
-                    cond = eval(s_cond, {}, getattr(self, "_model_dict", {}))
+                    cond = eval(
+                        s_cond, {}, getattr(self, "_model_dict", {})
+                    )  # nosec
                     result = rule.true if cond else rule.false
                 case "lookup":
                     args = cast(List[str], self.render_templates(rule.tree, data))
@@ -627,7 +643,7 @@ class DRSConfig(BaseModel, TemplateMixin):
                         self.render_templates(_call, data),
                         {},
                         getattr(self, "_model_dict", {}),
-                    )
+                    )  # nosec
             if result:
                 inp.metadata[facet] = result
 
@@ -666,7 +682,7 @@ class DRSConfig(BaseModel, TemplateMixin):
 
     def max_directory_tree_level(
         self, search_dir: str | Path, drs_type: str
-    ) -> int:
+    ) -> Tuple[int, bool]:
         """Get the maximum level for descending into directories.
 
         When searching for files in a directory we can only traverse the directory
@@ -686,18 +702,19 @@ class DRSConfig(BaseModel, TemplateMixin):
         version = cast(
             str, self.dialect[standard].facets.get("version", "version")
         )
+        is_versioned = True
+        dir_parts = self.dialect[standard].path_specs.dir_parts or []
         try:
-            version_idx = self.dialect[standard].path_specs.dir_parts.index(
-                version
-            )
+            version_idx = dir_parts.index(version)
         except ValueError:
             # No version given
-            version_idx = len(self.dialect[standard].path_specs.dir_parts)
+            version_idx = len(dir_parts)
+            is_versioned = False
         if root_path == search_dir:
             current_pos = 0
         else:
             current_pos = len(search_dir.relative_to(root_path).parts)
-        return version_idx - current_pos
+        return version_idx - current_pos, is_versioned
 
     def is_complete(self, data: Dict[str, Any], standard: str) -> bool:
         """Check if all metadata that can be collected was collected."""

metadata_crawler/api/drs_config.toml

@@ -78,6 +78,7 @@ multi_valued = true
 key         = "time"
 type        = "daterange"
 multi_valued = false
+default     = "fx"
 
 [drs_settings.index_schema.grid_label]
 key         = "grid_label"

metadata_crawler/api/index.py

@@ -4,19 +4,22 @@ from __future__ import annotations
 
 import abc
 from pathlib import Path
+from types import TracebackType
 from typing import (
     Any,
     AsyncIterator,
     Dict,
     List,
     Optional,
+    Self,
     Tuple,
+    Type,
     Union,
     cast,
 )
 
 from ..logger import logger
-from ..utils import Console
+from ..utils import Console, IndexProgress
 from .config import SchemaField
 from .metadata_stores import CatalogueReader, IndexStore
 
@@ -40,6 +43,9 @@ class BaseIndex:
     batch_size:
         The amount for metadata that should be gathered `before` ingesting
         it into the catalogue.
+    progress:
+        Optional rich progress object that should display the progress of the
+        tasks.
 
     Attributes
     ^^^^^^^^^^
@@ -50,9 +56,11 @@ class BaseIndex:
         catalogue_file: Optional[Union[str, Path]] = None,
         batch_size: int = 2500,
         storage_options: Optional[Dict[str, Any]] = None,
+        progress: Optional[IndexProgress] = None,
         **kwargs: Any,
     ) -> None:
         self._store: Optional[IndexStore] = None
+        self.progress = progress or IndexProgress(total=-1)
         if catalogue_file is not None:
             _reader = CatalogueReader(
                 catalogue_file=catalogue_file or "",
@@ -64,6 +72,16 @@ class BaseIndex:
 
     def __post_init__(self) -> None: ...
 
+    async def __aenter__(self) -> Self:
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None: ...
+
     @property
     def index_schema(self) -> Dict[str, SchemaField]:
         """Get the index schema."""
@@ -92,6 +110,7 @@ class BaseIndex:
             logger.info("Indexing %s", index_name)
             async for batch in self._store.read(index_name):
                 yield batch
+                self.progress.update(len(batch))
                 num_items += len(batch)
             msg = f"Indexed {num_items:10,.0f} items for index {index_name}"
             Console.print(msg) if Console.is_terminal else print(msg)

metadata_crawler/api/metadata_stores.py

@@ -473,10 +473,7 @@ class CatalogueReader:
     ) -> None:
         catalogue_file = str(catalogue_file)
         storage_options = storage_options or {}
-        fs, _ = IndexStore.get_fs(catalogue_file, **storage_options)
-        path = fs.unstrip_protocol(catalogue_file)
-        with fs.open(path) as stream:
-            cat = yaml.safe_load(stream.read())
+        cat = self.load_catalogue(catalogue_file, **storage_options)
         _schema_json = cat["metadata"]["schema"]
         schema = {s["key"]: SchemaField(**s) for k, s in _schema_json.items()}
         index_name = IndexName(**cat["metadata"]["index_names"])
@@ -493,6 +490,14 @@ class CatalogueReader:
             storage_options=storage_options,
         )
 
+    @staticmethod
+    def load_catalogue(path: Union[str, Path], **storage_options: Any) -> Any:
+        """Load a intake yaml catalogue (remote or local)."""
+        fs, _ = IndexStore.get_fs(str(path), **storage_options)
+        cat_path = fs.unstrip_protocol(path)
+        with fs.open(cat_path) as stream:
+            return yaml.safe_load(stream.read())
+
 
 class QueueConsumer:
     """Class that consumes the file discovery queue."""
@@ -722,6 +727,7 @@ class CatalogueWriter:
                     "latest": self.index_name.latest,
                     "all": self.index_name.all,
                 },
+                "indexed_objects": self.ingested_objects,
                 "schema": {
                     k: json.loads(s.model_dump_json())
                     for k, s in self.store.schema.items()

metadata_crawler/api/mixin/template_mixin.py

@@ -6,7 +6,7 @@ from typing import Any, Dict, Mapping, Optional
 
 from jinja2 import Environment, Template, Undefined
 
-ENV = Environment(undefined=Undefined, autoescape=False)
+ENV = Environment(undefined=Undefined, autoescape=True)
 
 
 @lru_cache(maxsize=1024)

metadata_crawler/backends/s3.py

@@ -71,11 +71,15 @@ class S3Path(PathTemplate):
         self, path: Union[str, Path, pathlib.Path]
     ) -> AsyncIterator[str]:
         """Retrieve sub directories of directory."""
-        path = str(path)
         client = await self._get_client()
-        for _content in await client._lsdir(path):
-            if _content.get("type", "") == "directory":
-                yield f'{_content.get("name", "")}'
+        path = str(path)
+        if await self.is_file(path):
+            yield path
+        else:
+            for _content in await client._lsdir(path):
+                size: int = _content.get("size") or 0
+                if _content.get("type", "") == "directory" or size > 0:
+                    yield _content.get("name", "")
 
     async def rglob(
         self, path: str | Path | pathlib.Path, glob_pattern: str = "*"

metadata_crawler/cli.py

@@ -34,7 +34,6 @@ from .api.metadata_stores import CatalogueBackends, IndexName
 from .backends.intake import IntakePath
 from .logger import (
     THIS_NAME,
-    add_file_handle,
     apply_verbosity,
     logger,
 )
@@ -48,7 +47,9 @@ KwargValue = Union[
 
 
 def walk_catalogue(
-    path: str, storage_options: Optional[Dict[str, Any]] = None, **kwargs: Any
+    path: str,
+    storage_options: Optional[Dict[str, Any]] = None,
+    **kwargs: Any,
 ) -> int:
     """Recursively traverse an intake catalogue.
 
@@ -359,6 +360,13 @@ class ArgParse:
             action="append",
             nargs=2,
         )
+        parser.add_argument(
+            "-v",
+            "--verbose",
+            action="count",
+            default=self.verbose,
+            help="Increase the verbosity level.",
+        )
         parser.set_defaults(apply_func=walk_catalogue)
 
     def _index_submcommands(self) -> None:
@@ -391,8 +399,8 @@ class ArgParse:
                     "-b",
                     "--batch-size",
                     type=int,
-                    default=25_000,
-                    help="Set the batch size for ingestion.",
+                    default=5_000,
+                    help="Set the batch size for indexing.",
                 )
                 parser.add_argument(
                     "--storage_option",
@@ -494,7 +502,6 @@ class ArgParse:
                 "apply_func",
                 "verbose",
                 "version",
-                "log_suffix",
                 "storage_option",
                 "shadow",
             )
@@ -509,7 +516,6 @@ class ArgParse:
             self.kwargs["shadow"] = _flatten(args.shadow)
         self.kwargs["storage_options"] = so
         self.verbose = args.verbose
-        add_file_handle(args.log_suffix)
         self.kwargs["verbosity"] = self.verbose
         return args
 
@@ -519,7 +525,9 @@ def _run(
     **kwargs: KwargValue,
 ) -> None:
     """Apply the parsed method."""
-    old_level = apply_verbosity(getattr(parser, "verbose", 0))
+    old_level = apply_verbosity(
+        getattr(parser, "verbose", 0), suffix=getattr(parser, "log_suffix", None)
+    )
     try:
         parser.apply_func(**kwargs)
     except Exception as error:

metadata_crawler/data_collector.py

@@ -15,6 +15,7 @@ from typing import (
     Dict,
     Iterator,
     Optional,
+    Tuple,
     Type,
     Union,
     cast,
@@ -33,7 +34,7 @@ from .utils import (
     print_performance,
 )
 
-ScanItem = tuple[str, str, bool]
+ScanItem = Tuple[str, str, bool, bool]
 
 
 class DataCollector:
@@ -138,6 +139,7 @@ class DataCollector:
         drs_type: str,
         search_dir: str,
         iterable: bool = True,
+        is_versioned: bool = True,
     ) -> None:
         if iterable:
             try:
@@ -161,7 +163,7 @@ class DataCollector:
                 await self.ingest_queue.put(
                     _inp, drs_type, name=self.index_name.all
                 )
-                if rank == 0:
+                if rank == 0 or is_versioned is False:
                     await self.ingest_queue.put(
                         _inp, drs_type, name=self.index_name.latest
                     )
@@ -176,16 +178,22 @@ class DataCollector:
             if item is None:  # sentinel -> exit
                 # do not task_done() for sentinel
                 break
-            drs_type, path, iterable = item
+            drs_type, path, iterable, is_versioned = item
             try:
-                await self._ingest_dir(drs_type, path, iterable=iterable)
+                await self._ingest_dir(
+                    drs_type, path, iterable=iterable, is_versioned=is_versioned
+                )
             except Exception as error:
                 logger.error(error)
             finally:
                 self._scan_queue.task_done()
 
     async def _iter_content(
-        self, drs_type: str, inp_dir: str, pos: int = 0
+        self,
+        drs_type: str,
+        inp_dir: str,
+        pos: int = 0,
+        is_versioned: bool = True,
     ) -> None:
         """Walk recursively until files or the version level is reached."""
         store = self.config.datasets[drs_type].backend
@@ -203,7 +211,6 @@ class DataCollector:
 
         iterable = False if suffix == ".zarr" else iterable
         op: Optional[Callable[..., Coroutine[Any, Any, None]]] = None
-
         if is_file and suffix in self.config.suffixes:
             op = self._ingest_dir
         elif pos <= 0 or suffix == ".zarr":
@@ -211,13 +218,17 @@ class DataCollector:
 
         if op is not None:
             # enqueue the heavy scan; workers will run _ingest_dir concurrently
-            await self._scan_queue.put((drs_type, inp_dir, iterable))
+            await self._scan_queue.put(
+                (drs_type, inp_dir, iterable, is_versioned)
+            )
             return
 
         # otherwise, recurse sequentially (cheap) — no task per directory
         try:
             async for sub in store.iterdir(inp_dir):
-                await self._iter_content(drs_type, sub, pos - 1)
+                await self._iter_content(
+                    drs_type, sub, pos - 1, is_versioned=is_versioned
+                )
         except Exception as error:
             logger.error(error)
 
@@ -239,10 +250,19 @@ class DataCollector:
 
             # produce scan items by walking roots sequentially
             for drs_type, path in self.search_objects:  # <- property is sync
-                pos = self.config.max_directory_tree_level(
+                pos, is_versioned = self.config.max_directory_tree_level(
                     path, drs_type=drs_type
                 )
-                await self._iter_content(drs_type, path, pos)
+                if pos < 0:
+                    logger.warning(
+                        "Can't define latest version of versioned dataset."
+                        " This might lead to unexpected results. Try adjusting"
+                        " your search path."
+                    )
+
+                await self._iter_content(
+                    drs_type, path, pos, is_versioned=is_versioned
+                )
 
             # wait until all queued scan items are processed
             await self._scan_queue.join()

metadata_crawler/ingester/mongo.py

@@ -80,12 +80,13 @@ class MongoIndex(BaseIndex):
         await collection.bulk_write(ops, ordered=False)
 
     async def _index_collection(
-        self, db: AsyncIOMotorDatabase[Any], collection: str
+        self, db: AsyncIOMotorDatabase[Any], collection: str, suffix: str = ""
     ) -> None:
         """Index a collection."""
-        await db[collection].create_index(self.unique_index, unique=True)
+        col = collection + suffix
+        await db[col].create_index(self.unique_index, unique=True)
         async for chunk in self.get_metadata(collection):
-            await self._bulk_upsert(chunk, db[collection])
+            await self._bulk_upsert(chunk, db[col])
 
     async def _prep_db_connection(
         self, database: str, url: str
@@ -119,12 +120,24 @@ class MongoIndex(BaseIndex):
                 default="metadata",
             ),
         ] = "metadata",
+        index_suffix: Annotated[
+            Optional[str],
+            cli_parameter(
+                "--index-suffix",
+                help="Suffix for the latest and all version collections.",
+                type=str,
+            ),
+        ] = None,
     ) -> None:
         """Add metadata to the mongoDB metadata server."""
         db = await self._prep_db_connection(database, url or "")
         async with asyncio.TaskGroup() as tg:
             for collection in self.index_names:
-                tg.create_task(self._index_collection(db, collection))
+                tg.create_task(
+                    self._index_collection(
+                        db, collection, suffix=index_suffix or ""
+                    )
+                )
 
     async def close(self) -> None:
         """Close the mongoDB connection."""

metadata_crawler/ingester/solr.py

@@ -5,9 +5,13 @@ from __future__ import annotations
 import asyncio
 import logging
 import os
-from typing import Annotated, Any, Dict, List, Optional
+import time
+from concurrent.futures import ThreadPoolExecutor
+from types import TracebackType
+from typing import Annotated, Any, Dict, List, Optional, Tuple, Type, cast
 
 import aiohttp
+import orjson
 
 from ..api.cli import cli_function, cli_parameter
 from ..api.index import BaseIndex
@@ -18,9 +22,26 @@ from ..logger import logger
 class SolrIndex(BaseIndex):
     """Ingest metadata into an apache solr server."""
 
+    senteniel: Optional[bytes] = None
+
     def __post_init__(self) -> None:
-        self.timeout = aiohttp.ClientTimeout(total=50)
+        self.timeout = aiohttp.ClientTimeout(
+            connect=10, sock_connect=10, sock_read=180, total=None
+        )
+        self.semaphore = asyncio.Event()
+        self.max_http_workers: int = 0
+        queue_max: int = 128
+        encode_workers: int = 4
         self._uri: str = ""
+        self.cpu_pool = ThreadPoolExecutor(max_workers=encode_workers)
+        self.producer_queue: asyncio.Queue[Tuple[str, Optional[bytes]]] = (
+            asyncio.Queue(maxsize=queue_max)
+        )
+        self.connector = aiohttp.TCPConnector(
+            ttl_dns_cache=300,
+            use_dns_cache=True,
+            enable_cleanup_closed=True,
+        )
 
     async def solr_url(self, server: str, core: str) -> str:
         """Construct the solr url from a given solr core."""
@@ -112,23 +133,98 @@ class SolrIndex(BaseIndex):
 
         return metadata
 
-    async def _index_core(self, server: str, core: str) -> None:
-        """Index data to a solr core."""
-        url = await self.solr_url(server, core)
-        async for chunk in self.get_metadata(core):
-            async with aiohttp.ClientSession(
-                timeout=self.timeout, raise_for_status=True
-            ) as session:
-                try:
-                    payload = list(map(self._convert, chunk))
-                    async with session.post(url, json=payload) as resp:
-                        logger.debug(await resp.text())
-                except Exception as error:
-                    logger.log(
-                        logging.WARNING,
-                        error,
-                        exc_info=logger.level < logging.INFO,
-                    )
+    def _encode_payload(self, chunk: List[Dict[str, Any]]) -> bytes:
+        """CPU-bound: convert docs and JSON-encode off the event loop."""
+        return orjson.dumps([self._convert(x) for x in chunk])
+
+    async def _post_chunk(
+        self,
+        session: aiohttp.ClientSession,
+        url: str,
+        body: bytes,
+    ) -> None:
+        """POST one batch with minimal overhead and simple retries."""
+        status = 500
+        t0 = time.perf_counter()
+        try:
+            async with session.post(
+                url, data=body, headers={"Content-Type": "application/json"}
+            ) as resp:
+                status = resp.status
+                await resp.read()
+
+        except Exception as error:
+            logger.log(
+                logging.WARNING,
+                error,
+                exc_info=logger.level < logging.INFO,
+            )
+            return
+        logger.debug(
+            "POST %s -> %i (index time: %.3f)",
+            url,
+            status,
+            time.perf_counter() - t0,
+        )
+
+    async def consumer(self, session: aiohttp.ClientSession) -> None:
+        """Consume the metadata read by the porducers."""
+        while True:
+            update_url, body = await self.producer_queue.get()
+            if body is self.senteniel:
+                self.producer_queue.task_done()
+                break
+            try:
+                await self._post_chunk(session, update_url, cast(bytes, body))
+            finally:
+                self.producer_queue.task_done()
+
+    async def _index_core(
+        self,
+        session: aiohttp.ClientSession,
+        server: str,
+        core: str,
+        suffix: str,
+        http_workers: int = 8,
+    ) -> None:
+        """Zero-copy-ish, backpressured, bounded-concurrency indexer.
+
+        - No per-batch commit.
+        - Bounded queue so tasks don't pile up.
+        - Constant number of worker tasks (not O(batches)).
+        """
+        base_url = await self.solr_url(server, core + suffix)
+        update_url = base_url.split("?", 1)[0]  # guard
+        loop = asyncio.get_running_loop()
+        async for batch in self.get_metadata(core):
+            body = await loop.run_in_executor(
+                self.cpu_pool, self._encode_payload, batch
+            )
+            await self.producer_queue.put((update_url, body))
+        commit_url = f"{update_url}?commit=true"
+        async with session.post(
+            commit_url,
+            data=b"[]",
+            headers={"Content-Type": "application/json"},
+        ) as resp:
+            if resp.status >= 400:
+                text = await resp.text()
+                logger.warning(
+                    "COMMIT %s -> %i: %s", commit_url, resp.status, text
+                )
+
+    async def __aexit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+
+        try:
+            self.producer_queue.shutdown()
+        except AttributeError:  # pragma: no cover
+            pass  # prgama: no cover
+        self.cpu_pool.shutdown()
 
     @cli_function(
         help="Add metadata to the apache solr metadata server.",
@@ -145,8 +241,41 @@ class SolrIndex(BaseIndex):
                 type=str,
             ),
         ] = None,
+        index_suffix: Annotated[
+            Optional[str],
+            cli_parameter(
+                "--index-suffix",
+                help="Suffix for the latest and all version collections.",
+                type=str,
+            ),
+        ] = None,
+        http_workers: Annotated[
+            int,
+            cli_parameter(
+                "--http-workers", help="Number of ingestion threads.", type=int
+            ),
+        ] = 8,
     ) -> None:
         """Add metadata to the apache solr metadata server."""
-        async with asyncio.TaskGroup() as tg:
-            for core in self.index_names:
-                tg.create_task(self._index_core(server or "", core))
+        async with aiohttp.ClientSession(
+            timeout=self.timeout, connector=self.connector, raise_for_status=True
+        ) as session:
+            consumers = [
+                asyncio.create_task(self.consumer(session))
+                for _ in range(http_workers)
+            ]
+            async with asyncio.TaskGroup() as tg:
+                for core in self.index_names:
+                    tg.create_task(
+                        self._index_core(
+                            session,
+                            server or "",
+                            core,
+                            suffix=index_suffix or "",
+                            http_workers=http_workers,
+                        )
+                    )
+            for _ in range(http_workers):
+                await self.producer_queue.put(("", self.senteniel))
+            await self.producer_queue.join()
+            await asyncio.gather(*consumers)

metadata_crawler/logger.py

@@ -11,7 +11,7 @@ import appdirs
 from rich.console import Console
 from rich.logging import RichHandler
 
-THIS_NAME = "data-crawler"
+THIS_NAME = "metadata-crawler"
 
 logging.basicConfig(
     level=logging.WARNING,
@@ -24,7 +24,7 @@ logging.config.dictConfig(
         # keep existing handlers
         "disable_existing_loggers": False,
         "root": {
-            "level": "WARNING",
+            "level": "CRITICAL",
             "handlers": ["default"],
         },
         "formatters": {
@@ -36,16 +36,12 @@ logging.config.dictConfig(
             "default": {
                 "class": "logging.StreamHandler",
                 "formatter": "standard",
-                "level": "WARNING",
+                "level": "CRITICAL",
             },
         },
     }
 )
 
-logging.getLogger("sqlalchemy").setLevel(logging.WARNING)
-logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
-logging.getLogger("sqlalchemy.pool").setLevel(logging.WARNING)
-
 
 class Logger(logging.Logger):
     """Custom Logger defining the logging behaviour."""
@@ -56,11 +52,14 @@ class Logger(logging.Logger):
     no_debug: list[str] = ["watchfiles", "httpcore", "pymongo", "pika"]
 
     def __init__(
-        self, name: Optional[str] = None, level: Optional[int] = None
+        self,
+        name: Optional[str] = None,
+        level: Optional[int] = None,
+        suffix: Optional[str] = None,
     ) -> None:
         """Instantiate this logger only once and for all."""
-        level = level or int(
-            cast(str, os.getenv("MDC_LOG_LEVEL", str(logging.WARNING)))
+        self.level = level or int(
+            cast(str, os.getenv("MDC_LOG_LEVEL", str(logging.CRITICAL)))
         )
         name = name or THIS_NAME
         logger_format = logging.Formatter(self.logfmt, self.datefmt)
@@ -78,11 +77,16 @@ class Logger(logging.Logger):
             ),
         )
         self._logger_stream_handle.setFormatter(logger_format)
-        self._logger_stream_handle.setLevel(level)
-        super().__init__(name, level)
+        self._logger_stream_handle.setLevel(self.level)
+        super().__init__(name, self.level)
 
         self.propagate = False
         self.handlers = [self._logger_stream_handle]
+        (
+            self.add_file_handle(suffix=suffix)
+            if os.getenv("MDC_LOG_INIT", "0") == "1"
+            else None
+        )
 
     def set_level(self, level: int) -> None:
         """Set the logger level to level."""
@@ -92,7 +96,7 @@ class Logger(logging.Logger):
                 log_level = min(level, logging.CRITICAL)
             handler.setLevel(log_level)
         self.setLevel(level)
-        logger.level = level
+        self.level = level
 
     def error(
         self,
@@ -105,28 +109,30 @@ class Logger(logging.Logger):
             kwargs.setdefault("exc_info", True)
         self._log(logging.ERROR, msg, args, **kwargs)
 
-
-logger = Logger()
+    def add_file_handle(
+        self,
+        suffix: Optional[str] = None,
+        level: int = logging.CRITICAL,
+    ) -> None:
+        """Add a file log handle to the logger."""
+        suffix = suffix or os.getenv("MDC_LOG_SUFFIX", "")
+        base_name = f"{THIS_NAME}-{suffix}" if suffix else THIS_NAME
+        log_dir = Path(os.getenv("MDC_LOG_DIR", appdirs.user_log_dir(THIS_NAME)))
+        log_dir.mkdir(exist_ok=True, parents=True)
+        logger_file_handle = RotatingFileHandler(
+            log_dir / f"{base_name}.log",
+            mode="a",
+            maxBytes=5 * 1024**2,
+            backupCount=5,
+            encoding="utf-8",
+            delay=False,
+        )
+        logger_file_handle.setFormatter(self.file_format)
+        logger_file_handle.setLevel(self.level)
+        self.addHandler(logger_file_handle)
 
 
-def add_file_handle(
-    suffix: Optional[str], log_level: int = logging.CRITICAL
-) -> None:
-    """Add a file log handle to the logger."""
-    base_name = f"{THIS_NAME}-{suffix}" if suffix else THIS_NAME
-    log_dir = Path(appdirs.user_log_dir(THIS_NAME))
-    log_dir.mkdir(exist_ok=True, parents=True)
-    logger_file_handle = RotatingFileHandler(
-        log_dir / f"{base_name}.log",
-        mode="a",
-        maxBytes=5 * 1024**2,
-        backupCount=5,
-        encoding="utf-8",
-        delay=False,
-    )
-    logger_file_handle.setFormatter(logger.file_format)
-    logger_file_handle.setLevel(min(log_level, logging.CRITICAL))
-    logger.addHandler(logger_file_handle)
+logger = Logger()
 
 
 def get_level_from_verbosity(verbosity: int) -> int:
@@ -134,9 +140,14 @@ def get_level_from_verbosity(verbosity: int) -> int:
     return max(logging.CRITICAL - 10 * verbosity, -1)
 
 
-def apply_verbosity(level: int) -> int:
+def apply_verbosity(
+    level: Optional[int] = None, suffix: Optional[str] = None
+) -> int:
     """Set the logging level of the handlers to a certain level."""
+    level = logger.level if level is None else level
     old_level = logger.level
     level = get_level_from_verbosity(level)
     logger.set_level(level)
+    logger.add_file_handle(suffix, level)
+
     return old_level

metadata_crawler/run.py

@@ -1,6 +1,5 @@
 """Apply the metadata collector."""
 
-import asyncio
 import os
 import time
 from fnmatch import fnmatch
@@ -9,15 +8,21 @@ from types import NoneType
 from typing import Any, Collection, Dict, List, Optional, Sequence, Union, cast
 
 import tomlkit
+import yaml
 from rich.prompt import Prompt
 
 from .api.config import CrawlerSettings, DRSConfig, strip_protocol
-from .api.metadata_stores import CatalogueBackendType, IndexName
+from .api.metadata_stores import (
+    CatalogueBackendType,
+    CatalogueReader,
+    IndexName,
+)
 from .data_collector import DataCollector
 from .logger import apply_verbosity, get_level_from_verbosity, logger
 from .utils import (
     Console,
     EmptyCrawl,
+    IndexProgress,
     MetadataCrawlerException,
     find_closest,
     load_plugins,
@@ -49,6 +54,20 @@ def _match(match: str, items: Collection[str]) -> List[str]:
     return out
 
 
+def _get_num_of_indexed_objects(
+    catalogue_files: FilesArg, storage_options: Optional[Dict[str, Any]] = None
+) -> int:
+    num_objects = 0
+    storage_options = storage_options or {}
+    for cat_file in _norm_files(catalogue_files):
+        try:
+            cat = CatalogueReader.load_catalogue(cat_file, **storage_options)
+            num_objects += cat.get("metadata", {}).get("indexed_objects", 0)
+        except (FileNotFoundError, IsADirectoryError, yaml.parser.ParserError):
+            pass
+    return num_objects
+
+
 def _get_search(
     config_file: Union[str, Path, Dict[str, Any], tomlkit.TOMLDocument],
     search_dirs: Optional[List[str]] = None,
@@ -86,14 +105,22 @@ async def async_call(
     batch_size: int = 2500,
     catalogue_files: Optional[Sequence[Union[Path, str]]] = None,
     verbosity: int = 0,
+    log_suffix: Optional[str] = None,
+    num_objects: int = 0,
     *args: Any,
     **kwargs: Any,
 ) -> None:
-    """Index metadata."""
+    """Add / Delete metadata from index."""
     env = cast(os._Environ[str], os.environ.copy())
-    old_level = apply_verbosity(verbosity)
+    old_level = apply_verbosity(verbosity, suffix=log_suffix)
+
     try:
+        progress = IndexProgress(total=num_objects)
+        os.environ["MDC_LOG_INIT"] = "1"
         os.environ["MDC_LOG_LEVEL"] = str(get_level_from_verbosity(verbosity))
+        os.environ["MDC_LOG_SUFFIX"] = (
+            log_suffix or os.getenv("MDC_LOG_SUFFIX") or ""
+        )
         backends = load_plugins("metadata_crawler.ingester")
         try:
             cls = backends[index_system]
@@ -103,22 +130,22 @@ async def async_call(
             )
             raise ValueError(msg) from None
         flat_files = _norm_files(catalogue_files)
-        _event_loop = asyncio.get_event_loop()
         flat_files = flat_files or [""]
-        futures = []
         storage_options = kwargs.pop("storage_options", {})
+        progress.start()
         for cf in flat_files:
-            obj = cls(
+            async with cls(
                 batch_size=batch_size,
                 catalogue_file=cf or None,
                 storage_options=storage_options,
-            )
-            func = getattr(obj, method)
-            future = _event_loop.create_task(func(**kwargs))
-            futures.append(future)
-        await asyncio.gather(*futures)
+                progress=progress,
+            ) as obj:
+                func = getattr(obj, method)
+                await func(**kwargs)
+
     finally:
         os.environ = env
+        progress.stop()
         logger.set_level(old_level)
 
 
@@ -127,6 +154,7 @@ async def async_index(
     *catalogue_files: Union[Path, str, List[str], List[Path]],
     batch_size: int = 2500,
     verbosity: int = 0,
+    log_suffix: Optional[str] = None,
     **kwargs: Any,
 ) -> None:
     """Index metadata in the indexing system.
@@ -142,6 +170,8 @@ async def async_index(
         If the index system supports batch-sizes, the size of the batches.
     verbosity:
         Set the verbosity of the system.
+    log_suffix:
+        Add a suffix to the log file output.
 
     Other Parameters
     ^^^^^^^^^^^^^^^^
@@ -168,6 +198,11 @@ async def async_index(
         "index",
         batch_size=batch_size,
         verbosity=verbosity,
+        log_suffix=log_suffix,
+        num_objects=_get_num_of_indexed_objects(
+            kwargs["catalogue_files"],
+            storage_options=kwargs.get("storage_options"),
+        ),
         **kwargs,
     )
 
@@ -176,6 +211,7 @@ async def async_delete(
     index_system: str,
     batch_size: int = 2500,
     verbosity: int = 0,
+    log_suffix: Optional[str] = None,
     **kwargs: Any,
 ) -> None:
     """Delete metadata from the indexing system.
@@ -188,6 +224,8 @@ async def async_delete(
         If the index system supports batch-sizes, the size of the batches.
     verbosity:
         Set the verbosity of the system.
+    log_suffix:
+        Add a suffix to the log file output.
 
     Other Parameters
     ^^^^^^^^^^^^^^^^^
@@ -212,6 +250,7 @@ async def async_delete(
         "delete",
         batch_size=batch_size,
         verbosity=verbosity,
+        log_suffix=log_suffix,
         **kwargs,
     )
 
@@ -236,6 +275,7 @@ async def async_add(
     password: bool = False,
     n_procs: Optional[int] = None,
     verbosity: int = 0,
+    log_suffix: Optional[str] = None,
     fail_under: int = -1,
     **kwargs: Any,
 ) -> None:
@@ -282,6 +322,8 @@ async def async_add(
         Set the number of parallel processes for collecting.
     verbosity:
         Set the verbosity of the system.
+    log_suffix:
+        Add a suffix to the log file output.
     fail_under:
         Fail if less than X of the discovered files could be indexed.
 
@@ -305,9 +347,13 @@ async def async_add(
 
     """
     env = cast(os._Environ[str], os.environ.copy())
-    old_level = apply_verbosity(verbosity)
+    old_level = apply_verbosity(verbosity, suffix=log_suffix)
     try:
+        os.environ["MDC_LOG_INIT"] = "1"
         os.environ["MDC_LOG_LEVEL"] = str(get_level_from_verbosity(verbosity))
+        os.environ["MDC_LOG_SUFFIX"] = (
+            log_suffix or os.getenv("MDC_LOG_SUFFIX") or ""
+        )
         config_file = config_file or os.environ.get(
             "EVALUATION_SYSTEM_CONFIG_DIR"
         )
@@ -316,7 +362,7 @@ async def async_add(
                 "You must give a config file/directory"
             )
         st = time.time()
-        passwd = ""
+        passwd: Optional[str] = None
         if password:  # pragma: no cover
             passwd = Prompt.ask(
                 "[b]Enter the password", password=True

metadata_crawler/utils.py

@@ -32,6 +32,7 @@ import rich.console
 import rich.spinner
 from dateutil.parser import isoparse
 from rich.live import Live
+from rich.progress import Progress, TaskID
 
 from .logger import logger
 
@@ -330,6 +331,76 @@ def timedelta_to_str(seconds: Union[int, float]) -> str:
     return " ".join(out[::-1])
 
 
+class IndexProgress:
+    """A helper that displays the progress of index Tasks."""
+
+    def __init__(
+        self,
+        total: int = 0,
+        interactive: Optional[bool] = None,
+        text: str = "Indexing: ",
+    ) -> None:
+        if interactive is None:
+            self._interactive = bool(
+                int(os.getenv("MDC_INTERACTIVE", str(int(Console.is_terminal))))
+            )
+        else:
+            self._interactive = interactive
+        self._log_interval = int(os.getenv("MDC_LOG_INTERVAL", "30"))
+        self.text = text
+        self._done = 0
+        self._task: TaskID = TaskID(0)
+        self._total = total
+        self._start = self._last_log = time.time()
+        self._progress = Progress()
+        self._last_printed_percent: float = -1.0
+
+    def start(self) -> None:
+        """Start the progress bar."""
+        self._start = self._last_log = time.time()
+
+        if self._interactive:
+            self._task = self._progress.add_task(
+                f"[green] {self.text}", total=self._total or None
+            )
+            self._progress.start()
+
+    def stop(self) -> None:
+        """Stop the progress bar."""
+        if self._interactive:
+            self._progress.stop()
+        else:
+            self._text_update()
+
+    def _text_update(self, bar_width: int = 40) -> None:
+        elapsed = timedelta(seconds=int(time.time() - self._start))
+        log_interval = timedelta(seconds=int(time.time() - self._last_log))
+        if self._total > 0:
+            filled = int((self._last_printed_percent / 100) * bar_width)
+            bar = "#" * filled + "-" * (bar_width - filled)
+            text = f"{self.text} [{bar}] {self._last_printed_percent:>6,.02f}%"
+        else:
+            text = f"{self.text} [{self._done:>12,}]"
+        if log_interval.total_seconds() >= self._log_interval:
+            print(f"{text} ({elapsed})", flush=True)
+            self._last_log = time.time()
+
+    def update(self, inc: int) -> None:
+        """Update the status progress bar by an increment."""
+        self._done += inc
+
+        if self._interactive is True:
+            desc = f"{self.text} [{self._done:>10d}]" if self._done == 0 else None
+            self._progress.update(self._task, advance=inc, description=desc)
+            return
+
+        frac = self._done / max(self._total, 1)
+        pct = frac * 100
+        if pct > self._last_printed_percent or self._total == 0:
+            self._last_printed_percent = pct
+            self._text_update()
+
+
 @daemon
 def print_performance(
     print_status: EventLike,

{metadata_crawler-2509.0.1.dist-info → metadata_crawler-2510.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metadata-crawler
-Version: 2509.0.1
+Version: 2510.0.0
 Summary: Crawl, extract and push climate metadata for indexing.
 Author-email: "DKRZ, Clint" <freva@dkrz.de>
 Requires-Python: >=3.11
@@ -34,7 +34,7 @@ Requires-Dist: numpy
 Requires-Dist: orjson
 Requires-Dist: pyarrow
 Requires-Dist: h5netcdf
-Requires-Dist: pydantic
+Requires-Dist: pydantic<2.12
 Requires-Dist: pyarrow
 Requires-Dist: rich
 Requires-Dist: rich-argparse
@@ -96,10 +96,12 @@ Provides-Extra: tests
 
 [![License](https://img.shields.io/badge/License-BSD-purple.svg)](LICENSE)
 [![PyPI](https://img.shields.io/pypi/pyversions/metadata-crawler.svg)](https://pypi.org/project/metadata-crawler/)
+[![Conda Version](https://img.shields.io/conda/vn/conda-forge/metadata-crawler.svg)](https://anaconda.org/conda-forge/metadata-crawler)
 [![Docs](https://readthedocs.org/projects/metadata-crawler/badge/?version=latest)](https://metadata-crawler.readthedocs.io/en/latest/?badge=latest)
 [![Tests](https://github.com/freva-org/metadata-crawler/actions/workflows/ci_job.yml/badge.svg)](https://github.com/freva-org/metadata-crawler/actions)
 [![Test-Coverage](https://codecov.io/gh/freva-org/metadata-crawler/graph/badge.svg?token=W2YziDnh2N)](https://codecov.io/gh/freva-org/metadata-crawler)
 
+
 Harvest, normalise, and index climate / earth-system metadata from **POSIX**,
 **S3/MinIO**, and **OpenStack Swift** using configurable **DRS dialects**
 (CMIP6, CMIP5, CORDEX, …). Output to a temporary **catalogue** (JSONLines)

metadata_crawler-2510.0.0.dist-info/RECORD

@@ -0,0 +1,34 @@
+metadata_crawler/__init__.py,sha256=dT4ZOngmtO-7fiWqdo80JmeRacG09fy1T8C0bZpFR6Q,7167
+metadata_crawler/__main__.py,sha256=4m56VOh7bb5xmZqb09fFbquke8g6KZfMbb3CUdBA60M,163
+metadata_crawler/_version.py,sha256=oJIpBtzsOuKTbnMbTB3ZHAqVHS0O9r3O0d2lf9lUGfE,25
+metadata_crawler/cli.py,sha256=qi77QXtuwO1N3MvLbacdaOZwzpT22FJMpnnp1k6yj-Y,17347
+metadata_crawler/data_collector.py,sha256=7N0zQcxjsqITUVr0JnkFu_beMzrTW-paaw69ESC9rkQ,9063
+metadata_crawler/logger.py,sha256=wNImwUVw0ycvIYrxzthWAgOCujJZhVDCSiCH5KKX5EA,4743
+metadata_crawler/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+metadata_crawler/run.py,sha256=_6mx29Co1HwfPNFWtzTR65CNlopqubj-McmavRM7i80,12869
+metadata_crawler/utils.py,sha256=Nm1DkyBD8PyBOP-EUf-Vqs-mLQUPu-6gWPgvNkGDmq8,14124
+metadata_crawler/api/__init__.py,sha256=UUF0_FKgfqgcXYmknxB0Wt1jaLNaf-w_q0tWVJhgV0M,28
+metadata_crawler/api/cli.py,sha256=pgj3iB_Irt74VbG3ZKStLRHKYY_I4bZpbOW1famKDnQ,1498
+metadata_crawler/api/config.py,sha256=4c9O0xmVwduEEGlNjQcIh1nV5HzXNjXNqPi3tEQkpGw,28814
+metadata_crawler/api/drs_config.toml,sha256=c3Gc8MGH22xlDOLH_y2TXiiEydmhjzvish-fQi5aGRA,10622
+metadata_crawler/api/index.py,sha256=0yqtXYOyWJJKKkCkIJbUUVG1w2Wt_icYJjXJPZZjSvU,4715
+metadata_crawler/api/metadata_stores.py,sha256=UekPl16KlaF7xiD4X7KVo3EMWz9KE-MT7gKxvgZyvXU,24016
+metadata_crawler/api/storage_backend.py,sha256=jdZZ_3SZcP3gJgw_NmPPdpDEx4D7qfLJDABfupTH9p0,7803
+metadata_crawler/api/mixin/__init__.py,sha256=4Y0T1eM4vLlgFazuC1q2briqx67LyfeCpY_pCICTnjk,197
+metadata_crawler/api/mixin/lookup_mixin.py,sha256=WxJ-ZNs8DcIXS9ThSoIZiepD07jfmLlzyTp65-Z1fLc,3558
+metadata_crawler/api/mixin/lookup_tables.py,sha256=za63xfZB0EvAm66uTTYo52zC0z7Y6VL8DUrP6CJ-DnQ,308683
+metadata_crawler/api/mixin/path_mixin.py,sha256=WKpesEjlwVSJ-VdoYYLEY5oBSAQTsvuv1B38ragAVIM,1247
+metadata_crawler/api/mixin/template_mixin.py,sha256=hxQXiP_JND3fuxBNcs1pZ7cvP-k-lTm5MQg40t0kF54,5105
+metadata_crawler/backends/__init__.py,sha256=yrk1L00ubQlMj3yXI73PPbhAahDKp792PJB-xcXUJIM,35
+metadata_crawler/backends/intake.py,sha256=TkvzBU8Rk49L0Y8e7Exz2nE3iLSWrBAwZnpEJtdlNR8,6595
+metadata_crawler/backends/posix.py,sha256=6sjAoCQHiOOjp_Hvwxn247wHBnoAJYUGequqphyZWaA,3409
+metadata_crawler/backends/s3.py,sha256=2ki-O_rRIb5dJVS9KyMmDDPczGOQTBUa-hmImllqeeE,4602
+metadata_crawler/backends/swift.py,sha256=az3ctF_npadjzAybX65CQbDLGoxRnk0ZR7vByo6lQOM,10954
+metadata_crawler/ingester/__init__.py,sha256=Y-c9VkQWMHDLb9WagwITCaEODlYa4p8xW-BkzzSRZXw,55
+metadata_crawler/ingester/mongo.py,sha256=Ntt3zKVtAX6wDB5aQYCoYrkVWrnvJU2oJJyfYGW30lU,6546
+metadata_crawler/ingester/solr.py,sha256=kpUAnI5iSsvNGagM_gqbTJZr8HNpYSFZFvNOcbHXB9o,9528
+metadata_crawler-2510.0.0.dist-info/entry_points.txt,sha256=4LzS7pbqwUPTD6C-iW42vuhXdtsOJmKXqFZpdpaKwF8,428
+metadata_crawler-2510.0.0.dist-info/licenses/LICENSE,sha256=GAUualebvSlegSVqb86FUqHrHM8WyM145__Nm2r_dfA,1496
+metadata_crawler-2510.0.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+metadata_crawler-2510.0.0.dist-info/METADATA,sha256=EdZwF0Y_U8NFQFTUcy6WbI8l2WYq59Ynp_L6S3ys1v4,13006
+metadata_crawler-2510.0.0.dist-info/RECORD,,

metadata_crawler-2509.0.1.dist-info/RECORD

@@ -1,34 +0,0 @@
-metadata_crawler/__init__.py,sha256=7gEpJjS9FpR6MHRY_Ztk8ORJ8JQ7WZUTV2TfLkaYgqs,6741
-metadata_crawler/__main__.py,sha256=4m56VOh7bb5xmZqb09fFbquke8g6KZfMbb3CUdBA60M,163
-metadata_crawler/_version.py,sha256=lJ4gM5yptFnF64LPHdDVhj6Mppmsw36i5KAr1dVXO5Y,25
-metadata_crawler/cli.py,sha256=meY5ZfR5VEW5ZorOPWO_b4MyIIQy0wTTPs9OkJ1WnfA,17180
-metadata_crawler/data_collector.py,sha256=9CVr4arKJspyLNLuF2MfkmY_r8x74Mw8hAaDSMouQUA,8372
-metadata_crawler/logger.py,sha256=5Lc0KdzH2HdWkidW-MASW8Pfy7vTMnzPv1-e2V3Any0,4407
-metadata_crawler/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metadata_crawler/run.py,sha256=w1kV4D63dS3mdgDTQj2ngzeSCjZPphWg1HwIJeJ6ATE,11345
-metadata_crawler/utils.py,sha256=QNr_9jZkuuQOrkuO46PrFhUfwLmfCJCq9gWUwwARfyM,11580
-metadata_crawler/api/__init__.py,sha256=UUF0_FKgfqgcXYmknxB0Wt1jaLNaf-w_q0tWVJhgV0M,28
-metadata_crawler/api/cli.py,sha256=pgj3iB_Irt74VbG3ZKStLRHKYY_I4bZpbOW1famKDnQ,1498
-metadata_crawler/api/config.py,sha256=j__JDKYTOR8kYC--HaHlYXfz38rzEhtUvHdO5Bh_j2E,28250
-metadata_crawler/api/drs_config.toml,sha256=90lQaSC2VdJ8OUoc6j27kg6d2OnfxR5a_KZH3W-FZV4,10603
-metadata_crawler/api/index.py,sha256=8g5HdSxluKtCwU45P0w_7LDIaSf200JbB-ekGJiI18c,4130
-metadata_crawler/api/metadata_stores.py,sha256=oWewL6XRmNZ6i5WxYI8Lm2jfpwLqBCGP2p4j3wLLNpQ,23735
-metadata_crawler/api/storage_backend.py,sha256=jdZZ_3SZcP3gJgw_NmPPdpDEx4D7qfLJDABfupTH9p0,7803
-metadata_crawler/api/mixin/__init__.py,sha256=4Y0T1eM4vLlgFazuC1q2briqx67LyfeCpY_pCICTnjk,197
-metadata_crawler/api/mixin/lookup_mixin.py,sha256=WxJ-ZNs8DcIXS9ThSoIZiepD07jfmLlzyTp65-Z1fLc,3558
-metadata_crawler/api/mixin/lookup_tables.py,sha256=za63xfZB0EvAm66uTTYo52zC0z7Y6VL8DUrP6CJ-DnQ,308683
-metadata_crawler/api/mixin/path_mixin.py,sha256=WKpesEjlwVSJ-VdoYYLEY5oBSAQTsvuv1B38ragAVIM,1247
-metadata_crawler/api/mixin/template_mixin.py,sha256=_qDp5n_CPnSYPMBsTia44b1ybBqrJEi-M1NaRkQ0z3U,5106
-metadata_crawler/backends/__init__.py,sha256=yrk1L00ubQlMj3yXI73PPbhAahDKp792PJB-xcXUJIM,35
-metadata_crawler/backends/intake.py,sha256=TkvzBU8Rk49L0Y8e7Exz2nE3iLSWrBAwZnpEJtdlNR8,6595
-metadata_crawler/backends/posix.py,sha256=6sjAoCQHiOOjp_Hvwxn247wHBnoAJYUGequqphyZWaA,3409
-metadata_crawler/backends/s3.py,sha256=DPz_bOyOlUveCwkSLVatwU_mcxUbFvygU_Id1AZVIMA,4455
-metadata_crawler/backends/swift.py,sha256=az3ctF_npadjzAybX65CQbDLGoxRnk0ZR7vByo6lQOM,10954
-metadata_crawler/ingester/__init__.py,sha256=Y-c9VkQWMHDLb9WagwITCaEODlYa4p8xW-BkzzSRZXw,55
-metadata_crawler/ingester/mongo.py,sha256=lpWIZ8mo6S8oY887uz2l6Y9pir0sUVEkfgOdDxrjIMM,6142
-metadata_crawler/ingester/solr.py,sha256=EoKS3kFeDTLf9zP22s2DhQGP81T6rTXVWDNT2wWKFkk,5242
-metadata_crawler-2509.0.1.dist-info/entry_points.txt,sha256=4LzS7pbqwUPTD6C-iW42vuhXdtsOJmKXqFZpdpaKwF8,428
-metadata_crawler-2509.0.1.dist-info/licenses/LICENSE,sha256=GAUualebvSlegSVqb86FUqHrHM8WyM145__Nm2r_dfA,1496
-metadata_crawler-2509.0.1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-metadata_crawler-2509.0.1.dist-info/METADATA,sha256=dT5Kd5_sBAccA_Qj9O64zksuF7u2iaj-DXqqEDtUDqc,12864
-metadata_crawler-2509.0.1.dist-info/RECORD,,