metadata-crawler 2509.0.2__tar.gz → 2510.0.1__tar.gz

{metadata_crawler-2509.0.2 → metadata_crawler-2510.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metadata-crawler
-Version: 2509.0.2
+Version: 2510.0.1
 Summary: Crawl, extract and push climate metadata for indexing.
 Author-email: "DKRZ, Clint" <freva@dkrz.de>
 Requires-Python: >=3.11

{metadata_crawler-2509.0.2 → metadata_crawler-2510.0.1}/docs/source/whatsnew.rst

@@ -3,6 +3,15 @@ What's new
 
 This document highlights major changes and additions across releases.
 
+v2510.0.1
+---------
+* Bug fixing
+
+v2510.0.0
+---------
+* Restructure solr ingest.
+* Make directory and filename parts in path specs optional.
+
 v2509.0.2
 ----------
 * Display progressbar for ingestion.

metadata_crawler-2510.0.1/src/metadata_crawler/_version.py

@@ -0,0 +1 @@
+__version__ = "2510.0.1"

{metadata_crawler-2509.0.2 → metadata_crawler-2510.0.1}/src/metadata_crawler/api/config.py

@@ -256,23 +256,38 @@ class CrawlerSettings(BaseModel):
 class PathSpecs(BaseModel):
     """Implementation of the Directory reference syntax."""
 
-    dir_parts: List[str] = Field(default_factory=list)
-    file_parts: List[str] = Field(default_factory=list)
+    dir_parts: Optional[List[str]] = None
+    file_parts: Optional[List[str]] = None
     file_sep: str = "_"
 
-    def get_metadata_from_path(self, rel_path: Path) -> Dict[str, Any]:
-        """Read path encoded metadata from path specs."""
+    def _get_metadata_from_dir(
+        self, data: Dict[str, Any], rel_path: Path
+    ) -> None:
         dir_parts = rel_path.parent.parts
-        file_parts = rel_path.name.split(self.file_sep)
-        if len(dir_parts) == len(self.dir_parts):
-            data: Dict[str, Any] = dict(zip(self.dir_parts, dir_parts))
-        else:
+
+        if self.dir_parts and len(dir_parts) == len(self.dir_parts):
+            data.update(
+                {
+                    k: v
+                    for (k, v) in zip(self.dir_parts, dir_parts)
+                    if k not in data
+                }
+            )
+        elif self.dir_parts:
             raise MetadataCrawlerException(
                 (
                     f"Number of dir parts for {rel_path.parent} do not match "
                     f"- needs: {len(self.dir_parts)} has: {len(dir_parts)}"
                 )
             ) from None
+
+    def _get_metadata_from_filename(
+        self, data: Dict[str, Any], rel_path: Path
+    ) -> None:
+        if self.file_parts is None:
+            return
+        file_parts = rel_path.name.split(self.file_sep)
+        _parts: Dict[str, str] = {}
         if len(file_parts) == len(self.file_parts):
             _parts = dict(zip(self.file_parts, file_parts))
         elif (
@@ -287,6 +302,12 @@ class PathSpecs(BaseModel):
                 )
             )
         data.update({k: v for (k, v) in _parts.items() if k not in data})
+
+    def get_metadata_from_path(self, rel_path: Path) -> Dict[str, Any]:
+        """Read path encoded metadata from path specs."""
+        data: Dict[str, Any] = {}
+        self._get_metadata_from_dir(data, rel_path)
+        self._get_metadata_from_filename(data, rel_path)
         data.pop("_", None)
         return data
 
@@ -689,13 +710,12 @@ class DRSConfig(BaseModel, TemplateMixin):
             str, self.dialect[standard].facets.get("version", "version")
         )
         is_versioned = True
+        dir_parts = self.dialect[standard].path_specs.dir_parts or []
         try:
-            version_idx = self.dialect[standard].path_specs.dir_parts.index(
-                version
-            )
+            version_idx = dir_parts.index(version)
         except ValueError:
             # No version given
-            version_idx = len(self.dialect[standard].path_specs.dir_parts)
+            version_idx = len(dir_parts)
             is_versioned = False
         if root_path == search_dir:
             current_pos = 0

{metadata_crawler-2509.0.2 → metadata_crawler-2510.0.1}/src/metadata_crawler/api/index.py

@@ -4,13 +4,16 @@ from __future__ import annotations
 
 import abc
 from pathlib import Path
+from types import TracebackType
 from typing import (
     Any,
     AsyncIterator,
     Dict,
     List,
     Optional,
+    Self,
     Tuple,
+    Type,
     Union,
     cast,
 )
@@ -69,6 +72,16 @@ class BaseIndex:
 
     def __post_init__(self) -> None: ...
 
+    async def __aenter__(self) -> Self:
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None: ...
+
     @property
     def index_schema(self) -> Dict[str, SchemaField]:
         """Get the index schema."""

{metadata_crawler-2509.0.2 → metadata_crawler-2510.0.1}/src/metadata_crawler/ingester/solr.py

@@ -7,7 +7,8 @@ import logging
 import os
 import time
 from concurrent.futures import ThreadPoolExecutor
-from typing import Annotated, Any, Dict, List, Optional, cast
+from types import TracebackType
+from typing import Annotated, Any, Dict, List, Optional, Tuple, Type, cast
 
 import aiohttp
 import orjson
@@ -21,9 +22,26 @@ from ..logger import logger
 class SolrIndex(BaseIndex):
     """Ingest metadata into an apache solr server."""
 
+    senteniel: Optional[bytes] = None
+
     def __post_init__(self) -> None:
-        self.timeout = aiohttp.ClientTimeout(total=50)
+        self.timeout = aiohttp.ClientTimeout(
+            connect=10, sock_connect=10, sock_read=180, total=None
+        )
+        self.semaphore = asyncio.Event()
+        self.max_http_workers: int = 0
+        queue_max: int = 128
+        encode_workers: int = 4
         self._uri: str = ""
+        self.cpu_pool = ThreadPoolExecutor(max_workers=encode_workers)
+        self.producer_queue: asyncio.Queue[Tuple[str, Optional[bytes]]] = (
+            asyncio.Queue(maxsize=queue_max)
+        )
+        self.connector = aiohttp.TCPConnector(
+            ttl_dns_cache=300,
+            use_dns_cache=True,
+            enable_cleanup_closed=True,
+        )
 
     async def solr_url(self, server: str, core: str) -> str:
         """Construct the solr url from a given solr core."""
@@ -149,8 +167,25 @@ class SolrIndex(BaseIndex):
             time.perf_counter() - t0,
         )
 
+    async def consumer(self, session: aiohttp.ClientSession) -> None:
+        """Consume the metadata read by the porducers."""
+        while True:
+            update_url, body = await self.producer_queue.get()
+            if body is self.senteniel:
+                self.producer_queue.task_done()
+                break
+            try:
+                await self._post_chunk(session, update_url, cast(bytes, body))
+            finally:
+                self.producer_queue.task_done()
+
     async def _index_core(
-        self, server: str, core: str, suffix: str, http_workers: int = 8
+        self,
+        session: aiohttp.ClientSession,
+        server: str,
+        core: str,
+        suffix: str,
+        http_workers: int = 8,
     ) -> None:
         """Zero-copy-ish, backpressured, bounded-concurrency indexer.
 
@@ -160,70 +195,36 @@ class SolrIndex(BaseIndex):
         """
         base_url = await self.solr_url(server, core + suffix)
         update_url = base_url.split("?", 1)[0]  # guard
-
-        queue_max: int = 128
-        encode_workers: int = 4
-
-        timeout = aiohttp.ClientTimeout(
-            connect=10, sock_connect=10, sock_read=180, total=None
-        )
-        connector = aiohttp.TCPConnector(
-            limit_per_host=http_workers,
-            ttl_dns_cache=300,
-            enable_cleanup_closed=True,
-        )
-
         loop = asyncio.get_running_loop()
-        cpu_pool = ThreadPoolExecutor(max_workers=encode_workers)
-        q: asyncio.Queue[Optional[bytes]] = asyncio.Queue(maxsize=queue_max)
-        SENTINEL: Optional[bytes] = None
-
-        async def producer() -> None:
-            async for batch in self.get_metadata(core):
-                body = await loop.run_in_executor(
-                    cpu_pool, self._encode_payload, batch
+        async for batch in self.get_metadata(core):
+            body = await loop.run_in_executor(
+                self.cpu_pool, self._encode_payload, batch
+            )
+            await self.producer_queue.put((update_url, body))
+        commit_url = f"{update_url}?commit=true"
+        async with session.post(
+            commit_url,
+            data=b"[]",
+            headers={"Content-Type": "application/json"},
+        ) as resp:
+            if resp.status >= 400:
+                text = await resp.text()
+                logger.warning(
+                    "COMMIT %s -> %i: %s", commit_url, resp.status, text
                 )
-                await q.put(body)
-            for _ in range(http_workers):
-                await q.put(SENTINEL)
 
-        async def consumer(
-            worker_id: int, session: aiohttp.ClientSession
-        ) -> None:
-            while True:
-                body = await q.get()
-                if body is SENTINEL:
-                    q.task_done()
-                    break
-                try:
-                    await self._post_chunk(session, update_url, cast(bytes, body))
-                finally:
-                    q.task_done()
-
-        async with aiohttp.ClientSession(
-            timeout=timeout, connector=connector, raise_for_status=True
-        ) as session:
-            consumers = [
-                asyncio.create_task(consumer(i, session))
-                for i in range(http_workers)
-            ]
-            prod_task = asyncio.create_task(producer())
-            await prod_task
-            await q.join()
-            await asyncio.gather(*consumers)
+    async def __aexit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
 
-        commit_url = f"{update_url}?commit=true"
-        async with aiohttp.ClientSession(timeout=timeout) as session:
-            async with session.post(
-                commit_url,
-                data=b"[]",
-                headers={"Content-Type": "application/json"},
-            ) as resp:
-                if resp.status >= 400:
-                    text = await resp.text()
-                    logger.warning(
-                        "COMMIT %s -> %i: %s", commit_url, resp.status, text
-                    )
+        try:
+            self.producer_queue.shutdown()
+        except AttributeError:  # pragma: no cover
+            pass  # prgama: no cover
+        self.cpu_pool.shutdown()
 
     @cli_function(
         help="Add metadata to the apache solr metadata server.",
@@ -256,13 +257,25 @@ class SolrIndex(BaseIndex):
         ] = 8,
     ) -> None:
         """Add metadata to the apache solr metadata server."""
-        async with asyncio.TaskGroup() as tg:
-            for core in self.index_names:
-                tg.create_task(
-                    self._index_core(
-                        server or "",
-                        core,
-                        suffix=index_suffix or "",
-                        http_workers=http_workers,
+        async with aiohttp.ClientSession(
+            timeout=self.timeout, connector=self.connector, raise_for_status=True
+        ) as session:
+            consumers = [
+                asyncio.create_task(self.consumer(session))
+                for _ in range(http_workers)
+            ]
+            async with asyncio.TaskGroup() as tg:
+                for core in self.index_names:
+                    tg.create_task(
+                        self._index_core(
+                            session,
+                            server or "",
+                            core,
+                            suffix=index_suffix or "",
+                            http_workers=http_workers,
+                        )
                     )
-                )
+            for _ in range(http_workers):
+                await self.producer_queue.put(("", self.senteniel))
+            await self.producer_queue.join()
+            await asyncio.gather(*consumers)

{metadata_crawler-2509.0.2 → metadata_crawler-2510.0.1}/src/metadata_crawler/run.py

@@ -1,6 +1,5 @@
 """Apply the metadata collector."""
 
-import asyncio
 import os
 import time
 from fnmatch import fnmatch
@@ -131,22 +130,18 @@ async def async_call(
             )
             raise ValueError(msg) from None
         flat_files = _norm_files(catalogue_files)
-        _event_loop = asyncio.get_event_loop()
         flat_files = flat_files or [""]
-        futures = []
         storage_options = kwargs.pop("storage_options", {})
         progress.start()
         for cf in flat_files:
-            obj = cls(
+            async with cls(
                 batch_size=batch_size,
                 catalogue_file=cf or None,
                 storage_options=storage_options,
                 progress=progress,
-            )
-            func = getattr(obj, method)
-            future = _event_loop.create_task(func(**kwargs))
-            futures.append(future)
-        await asyncio.gather(*futures)
+            ) as obj:
+                func = getattr(obj, method)
+                await func(**kwargs)
 
     finally:
         os.environ = env

metadata_crawler-2509.0.2/src/metadata_crawler/_version.py

@@ -1 +0,0 @@
-__version__ = "2509.0.2"