metadata-crawler 2509.0.2__py3-none-any.whl → 2510.0.0__py3-none-any.whl

metadata_crawler/_version.py

@@ -1 +1 @@
-__version__ = "2509.0.2"
+__version__ = "2510.0.0"

metadata_crawler/api/config.py

@@ -256,23 +256,31 @@ class CrawlerSettings(BaseModel):
 class PathSpecs(BaseModel):
     """Implementation of the Directory reference syntax."""
 
-    dir_parts: List[str] = Field(default_factory=list)
-    file_parts: List[str] = Field(default_factory=list)
+    dir_parts: Optional[List[str]] = None
+    file_parts: Optional[List[str]] = None
     file_sep: str = "_"
 
-    def get_metadata_from_path(self, rel_path: Path) -> Dict[str, Any]:
-        """Read path encoded metadata from path specs."""
+    def _get_metadata_from_dir(
+        self, data: Dict[str, Any], rel_path: Path
+    ) -> None:
         dir_parts = rel_path.parent.parts
-        file_parts = rel_path.name.split(self.file_sep)
-        if len(dir_parts) == len(self.dir_parts):
-            data: Dict[str, Any] = dict(zip(self.dir_parts, dir_parts))
-        else:
+        if self.dir_parts and len(dir_parts) == len(self.dir_parts):
+            _parts = dict(zip(self.dir_parts, dir_parts))
+        elif self.dir_parts:
             raise MetadataCrawlerException(
                 (
                     f"Number of dir parts for {rel_path.parent} do not match "
                     f"- needs: {len(self.dir_parts)} has: {len(dir_parts)}"
                 )
             ) from None
+        data.update({k: v for (k, v) in _parts.items() if k not in data})
+
+    def _get_metadata_from_filename(
+        self, data: Dict[str, Any], rel_path: Path
+    ) -> None:
+        if self.file_parts is None:
+            return
+        file_parts = rel_path.name.split(self.file_sep)
         if len(file_parts) == len(self.file_parts):
             _parts = dict(zip(self.file_parts, file_parts))
         elif (
@@ -287,6 +295,12 @@ class PathSpecs(BaseModel):
                 )
             )
         data.update({k: v for (k, v) in _parts.items() if k not in data})
+
+    def get_metadata_from_path(self, rel_path: Path) -> Dict[str, Any]:
+        """Read path encoded metadata from path specs."""
+        data: Dict[str, Any] = {}
+        self._get_metadata_from_dir(data, rel_path)
+        self._get_metadata_from_filename(data, rel_path)
         data.pop("_", None)
         return data
 
@@ -689,13 +703,12 @@ class DRSConfig(BaseModel, TemplateMixin):
             str, self.dialect[standard].facets.get("version", "version")
         )
         is_versioned = True
+        dir_parts = self.dialect[standard].path_specs.dir_parts or []
         try:
-            version_idx = self.dialect[standard].path_specs.dir_parts.index(
-                version
-            )
+            version_idx = dir_parts.index(version)
         except ValueError:
             # No version given
-            version_idx = len(self.dialect[standard].path_specs.dir_parts)
+            version_idx = len(dir_parts)
             is_versioned = False
         if root_path == search_dir:
             current_pos = 0

metadata_crawler/api/index.py

@@ -4,13 +4,16 @@ from __future__ import annotations
 
 import abc
 from pathlib import Path
+from types import TracebackType
 from typing import (
     Any,
     AsyncIterator,
     Dict,
     List,
     Optional,
+    Self,
     Tuple,
+    Type,
     Union,
     cast,
 )
@@ -69,6 +72,16 @@ class BaseIndex:
 
     def __post_init__(self) -> None: ...
 
+    async def __aenter__(self) -> Self:
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None: ...
+
     @property
     def index_schema(self) -> Dict[str, SchemaField]:
         """Get the index schema."""

metadata_crawler/ingester/solr.py

@@ -7,7 +7,8 @@ import logging
 import os
 import time
 from concurrent.futures import ThreadPoolExecutor
-from typing import Annotated, Any, Dict, List, Optional, cast
+from types import TracebackType
+from typing import Annotated, Any, Dict, List, Optional, Tuple, Type, cast
 
 import aiohttp
 import orjson
@@ -21,9 +22,26 @@ from ..logger import logger
 class SolrIndex(BaseIndex):
     """Ingest metadata into an apache solr server."""
 
+    senteniel: Optional[bytes] = None
+
     def __post_init__(self) -> None:
-        self.timeout = aiohttp.ClientTimeout(total=50)
+        self.timeout = aiohttp.ClientTimeout(
+            connect=10, sock_connect=10, sock_read=180, total=None
+        )
+        self.semaphore = asyncio.Event()
+        self.max_http_workers: int = 0
+        queue_max: int = 128
+        encode_workers: int = 4
         self._uri: str = ""
+        self.cpu_pool = ThreadPoolExecutor(max_workers=encode_workers)
+        self.producer_queue: asyncio.Queue[Tuple[str, Optional[bytes]]] = (
+            asyncio.Queue(maxsize=queue_max)
+        )
+        self.connector = aiohttp.TCPConnector(
+            ttl_dns_cache=300,
+            use_dns_cache=True,
+            enable_cleanup_closed=True,
+        )
 
     async def solr_url(self, server: str, core: str) -> str:
         """Construct the solr url from a given solr core."""
@@ -149,8 +167,25 @@ class SolrIndex(BaseIndex):
             time.perf_counter() - t0,
         )
 
+    async def consumer(self, session: aiohttp.ClientSession) -> None:
+        """Consume the metadata read by the porducers."""
+        while True:
+            update_url, body = await self.producer_queue.get()
+            if body is self.senteniel:
+                self.producer_queue.task_done()
+                break
+            try:
+                await self._post_chunk(session, update_url, cast(bytes, body))
+            finally:
+                self.producer_queue.task_done()
+
     async def _index_core(
-        self, server: str, core: str, suffix: str, http_workers: int = 8
+        self,
+        session: aiohttp.ClientSession,
+        server: str,
+        core: str,
+        suffix: str,
+        http_workers: int = 8,
     ) -> None:
         """Zero-copy-ish, backpressured, bounded-concurrency indexer.
 
@@ -160,70 +195,36 @@ class SolrIndex(BaseIndex):
         """
         base_url = await self.solr_url(server, core + suffix)
         update_url = base_url.split("?", 1)[0]  # guard
-
-        queue_max: int = 128
-        encode_workers: int = 4
-
-        timeout = aiohttp.ClientTimeout(
-            connect=10, sock_connect=10, sock_read=180, total=None
-        )
-        connector = aiohttp.TCPConnector(
-            limit_per_host=http_workers,
-            ttl_dns_cache=300,
-            enable_cleanup_closed=True,
-        )
-
         loop = asyncio.get_running_loop()
-        cpu_pool = ThreadPoolExecutor(max_workers=encode_workers)
-        q: asyncio.Queue[Optional[bytes]] = asyncio.Queue(maxsize=queue_max)
-        SENTINEL: Optional[bytes] = None
-
-        async def producer() -> None:
-            async for batch in self.get_metadata(core):
-                body = await loop.run_in_executor(
-                    cpu_pool, self._encode_payload, batch
+        async for batch in self.get_metadata(core):
+            body = await loop.run_in_executor(
+                self.cpu_pool, self._encode_payload, batch
+            )
+            await self.producer_queue.put((update_url, body))
+        commit_url = f"{update_url}?commit=true"
+        async with session.post(
+            commit_url,
+            data=b"[]",
+            headers={"Content-Type": "application/json"},
+        ) as resp:
+            if resp.status >= 400:
+                text = await resp.text()
+                logger.warning(
+                    "COMMIT %s -> %i: %s", commit_url, resp.status, text
                 )
-                await q.put(body)
-            for _ in range(http_workers):
-                await q.put(SENTINEL)
 
-        async def consumer(
-            worker_id: int, session: aiohttp.ClientSession
-        ) -> None:
-            while True:
-                body = await q.get()
-                if body is SENTINEL:
-                    q.task_done()
-                    break
-                try:
-                    await self._post_chunk(session, update_url, cast(bytes, body))
-                finally:
-                    q.task_done()
-
-        async with aiohttp.ClientSession(
-            timeout=timeout, connector=connector, raise_for_status=True
-        ) as session:
-            consumers = [
-                asyncio.create_task(consumer(i, session))
-                for i in range(http_workers)
-            ]
-            prod_task = asyncio.create_task(producer())
-            await prod_task
-            await q.join()
-            await asyncio.gather(*consumers)
+    async def __aexit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
 
-        commit_url = f"{update_url}?commit=true"
-        async with aiohttp.ClientSession(timeout=timeout) as session:
-            async with session.post(
-                commit_url,
-                data=b"[]",
-                headers={"Content-Type": "application/json"},
-            ) as resp:
-                if resp.status >= 400:
-                    text = await resp.text()
-                    logger.warning(
-                        "COMMIT %s -> %i: %s", commit_url, resp.status, text
-                    )
+        try:
+            self.producer_queue.shutdown()
+        except AttributeError:  # pragma: no cover
+            pass  # prgama: no cover
+        self.cpu_pool.shutdown()
 
     @cli_function(
         help="Add metadata to the apache solr metadata server.",
@@ -256,13 +257,25 @@ class SolrIndex(BaseIndex):
         ] = 8,
     ) -> None:
         """Add metadata to the apache solr metadata server."""
-        async with asyncio.TaskGroup() as tg:
-            for core in self.index_names:
-                tg.create_task(
-                    self._index_core(
-                        server or "",
-                        core,
-                        suffix=index_suffix or "",
-                        http_workers=http_workers,
+        async with aiohttp.ClientSession(
+            timeout=self.timeout, connector=self.connector, raise_for_status=True
+        ) as session:
+            consumers = [
+                asyncio.create_task(self.consumer(session))
+                for _ in range(http_workers)
+            ]
+            async with asyncio.TaskGroup() as tg:
+                for core in self.index_names:
+                    tg.create_task(
+                        self._index_core(
+                            session,
+                            server or "",
+                            core,
+                            suffix=index_suffix or "",
+                            http_workers=http_workers,
+                        )
                     )
-                )
+            for _ in range(http_workers):
+                await self.producer_queue.put(("", self.senteniel))
+            await self.producer_queue.join()
+            await asyncio.gather(*consumers)

metadata_crawler/run.py

@@ -1,6 +1,5 @@
 """Apply the metadata collector."""
 
-import asyncio
 import os
 import time
 from fnmatch import fnmatch
@@ -131,22 +130,18 @@ async def async_call(
             )
             raise ValueError(msg) from None
         flat_files = _norm_files(catalogue_files)
-        _event_loop = asyncio.get_event_loop()
         flat_files = flat_files or [""]
-        futures = []
         storage_options = kwargs.pop("storage_options", {})
         progress.start()
         for cf in flat_files:
-            obj = cls(
+            async with cls(
                 batch_size=batch_size,
                 catalogue_file=cf or None,
                 storage_options=storage_options,
                 progress=progress,
-            )
-            func = getattr(obj, method)
-            future = _event_loop.create_task(func(**kwargs))
-            futures.append(future)
-        await asyncio.gather(*futures)
+            ) as obj:
+                func = getattr(obj, method)
+                await func(**kwargs)
 
     finally:
         os.environ = env

{metadata_crawler-2509.0.2.dist-info → metadata_crawler-2510.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metadata-crawler
-Version: 2509.0.2
+Version: 2510.0.0
 Summary: Crawl, extract and push climate metadata for indexing.
 Author-email: "DKRZ, Clint" <freva@dkrz.de>
 Requires-Python: >=3.11

{metadata_crawler-2509.0.2.dist-info → metadata_crawler-2510.0.0.dist-info}/RECORD

@@ -1,17 +1,17 @@
 metadata_crawler/__init__.py,sha256=dT4ZOngmtO-7fiWqdo80JmeRacG09fy1T8C0bZpFR6Q,7167
 metadata_crawler/__main__.py,sha256=4m56VOh7bb5xmZqb09fFbquke8g6KZfMbb3CUdBA60M,163
-metadata_crawler/_version.py,sha256=9-K5oUNmfiY2VyddRsxyD-fcZp54m4x8eeX3XbXHEV0,25
+metadata_crawler/_version.py,sha256=oJIpBtzsOuKTbnMbTB3ZHAqVHS0O9r3O0d2lf9lUGfE,25
 metadata_crawler/cli.py,sha256=qi77QXtuwO1N3MvLbacdaOZwzpT22FJMpnnp1k6yj-Y,17347
 metadata_crawler/data_collector.py,sha256=7N0zQcxjsqITUVr0JnkFu_beMzrTW-paaw69ESC9rkQ,9063
 metadata_crawler/logger.py,sha256=wNImwUVw0ycvIYrxzthWAgOCujJZhVDCSiCH5KKX5EA,4743
 metadata_crawler/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metadata_crawler/run.py,sha256=ytkYZQGWQ1jAvm8_ZbVPfTydGoHTEAhKWbajlkt6oU4,13033
+metadata_crawler/run.py,sha256=_6mx29Co1HwfPNFWtzTR65CNlopqubj-McmavRM7i80,12869
 metadata_crawler/utils.py,sha256=Nm1DkyBD8PyBOP-EUf-Vqs-mLQUPu-6gWPgvNkGDmq8,14124
 metadata_crawler/api/__init__.py,sha256=UUF0_FKgfqgcXYmknxB0Wt1jaLNaf-w_q0tWVJhgV0M,28
 metadata_crawler/api/cli.py,sha256=pgj3iB_Irt74VbG3ZKStLRHKYY_I4bZpbOW1famKDnQ,1498
-metadata_crawler/api/config.py,sha256=MxxAN1y2FtHlUU42nBfQds5_8R_OSDdnHXsZANx6IFY,28373
+metadata_crawler/api/config.py,sha256=4c9O0xmVwduEEGlNjQcIh1nV5HzXNjXNqPi3tEQkpGw,28814
 metadata_crawler/api/drs_config.toml,sha256=c3Gc8MGH22xlDOLH_y2TXiiEydmhjzvish-fQi5aGRA,10622
-metadata_crawler/api/index.py,sha256=9hafNfNEbmw2tIVYq7jPagz7RaDtxXjs_L-YtFVvNJk,4411
+metadata_crawler/api/index.py,sha256=0yqtXYOyWJJKKkCkIJbUUVG1w2Wt_icYJjXJPZZjSvU,4715
 metadata_crawler/api/metadata_stores.py,sha256=UekPl16KlaF7xiD4X7KVo3EMWz9KE-MT7gKxvgZyvXU,24016
 metadata_crawler/api/storage_backend.py,sha256=jdZZ_3SZcP3gJgw_NmPPdpDEx4D7qfLJDABfupTH9p0,7803
 metadata_crawler/api/mixin/__init__.py,sha256=4Y0T1eM4vLlgFazuC1q2briqx67LyfeCpY_pCICTnjk,197
@@ -26,9 +26,9 @@ metadata_crawler/backends/s3.py,sha256=2ki-O_rRIb5dJVS9KyMmDDPczGOQTBUa-hmImllqe
 metadata_crawler/backends/swift.py,sha256=az3ctF_npadjzAybX65CQbDLGoxRnk0ZR7vByo6lQOM,10954
 metadata_crawler/ingester/__init__.py,sha256=Y-c9VkQWMHDLb9WagwITCaEODlYa4p8xW-BkzzSRZXw,55
 metadata_crawler/ingester/mongo.py,sha256=Ntt3zKVtAX6wDB5aQYCoYrkVWrnvJU2oJJyfYGW30lU,6546
-metadata_crawler/ingester/solr.py,sha256=cRHe47l3WFZEFLZkHD1q-aPVjimi8H03xgL994XO1Lg,8988
-metadata_crawler-2509.0.2.dist-info/entry_points.txt,sha256=4LzS7pbqwUPTD6C-iW42vuhXdtsOJmKXqFZpdpaKwF8,428
-metadata_crawler-2509.0.2.dist-info/licenses/LICENSE,sha256=GAUualebvSlegSVqb86FUqHrHM8WyM145__Nm2r_dfA,1496
-metadata_crawler-2509.0.2.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-metadata_crawler-2509.0.2.dist-info/METADATA,sha256=b32DEUfPeWaSKbhdZYw_1qi57-yIyS0Z2PhaaH4EDK8,13006
-metadata_crawler-2509.0.2.dist-info/RECORD,,
+metadata_crawler/ingester/solr.py,sha256=kpUAnI5iSsvNGagM_gqbTJZr8HNpYSFZFvNOcbHXB9o,9528
+metadata_crawler-2510.0.0.dist-info/entry_points.txt,sha256=4LzS7pbqwUPTD6C-iW42vuhXdtsOJmKXqFZpdpaKwF8,428
+metadata_crawler-2510.0.0.dist-info/licenses/LICENSE,sha256=GAUualebvSlegSVqb86FUqHrHM8WyM145__Nm2r_dfA,1496
+metadata_crawler-2510.0.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+metadata_crawler-2510.0.0.dist-info/METADATA,sha256=EdZwF0Y_U8NFQFTUcy6WbI8l2WYq59Ynp_L6S3ys1v4,13006
+metadata_crawler-2510.0.0.dist-info/RECORD,,