cellar-extractor 2.0.3__tar.gz → 2.0.4__tar.gz

{cellar_extractor-2.0.3/cellar_extractor.egg-info → cellar_extractor-2.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cellar-extractor
-Version: 2.0.3
+Version: 2.0.4
 Summary: Library for extracting CELLAR case law data from EUR-Lex
 Author-email: LawTech Lab <law-techlab@maastrichtuniversity.nl>
 License: Apache-2.0

{cellar_extractor-2.0.3 → cellar_extractor-2.0.4}/cellar_extractor/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '2.0.3'
-__version_tuple__ = version_tuple = (2, 0, 3)
+__version__ = version = '2.0.4'
+__version_tuple__ = version_tuple = (2, 0, 4)
 
-__commit_id__ = commit_id = 'g1aaeae809'
+__commit_id__ = commit_id = 'g792a41d4a'

{cellar_extractor-2.0.3 → cellar_extractor-2.0.4}/cellar_extractor/cellar.py

@@ -4,7 +4,12 @@ from concurrent.futures import ThreadPoolExecutor
 from tqdm import tqdm
 
 from cellar_extractor.cellar_extra_extract import extra_cellar
-from cellar_extractor.cellar_queries import get_all_eclis, get_raw_cellar_metadata
+from cellar_extractor.cellar_queries import (
+    get_all_eclis,
+    get_infocuria_document_metadata,
+    get_raw_cellar_metadata,
+    reconcile_document_metadata,
+)
 from cellar_extractor.json_to_csv import json_to_csv_returning
 from cellar_extractor.nodes_and_edges import get_nodes_and_edges
 from cellar_extractor.persistence import (
@@ -70,6 +75,7 @@ def get_cellar(
     output_path=None,
     return_data=None,
     save=None,
+    reconcile_infocuria=False,
 ):
     """
     Fetch base CELLAR metadata.
@@ -91,10 +97,24 @@ def get_cellar(
     logging.info(f"Up until the specified end date {ed}")
     eclis = get_all_eclis(starting_date=sd, ending_date=ed, limit=max_ecli)
     logging.info(f"Found {len(eclis)} ECLIs")
-    if len(eclis) == 0:
+    all_eclis = _fetch_metadata_batches(eclis)
+    if reconcile_infocuria:
+        infocuria_metadata = get_infocuria_document_metadata(
+            starting_date=sd,
+            ending_date=ed,
+            limit=max_ecli,
+        )
+        logging.info(
+            "Found %s ECLIs in the InfoCuria document catalogue",
+            len(infocuria_metadata),
+        )
+        all_eclis = reconcile_document_metadata(all_eclis, infocuria_metadata)
+        if max_ecli is not None and len(all_eclis) > max_ecli:
+            all_eclis = dict(list(all_eclis.items())[:max_ecli])
+
+    if len(all_eclis) == 0:
         logging.info(f"No data to download found between {sd} and {ed}")
         return False
-    all_eclis = _fetch_metadata_batches(eclis)
 
     result = _materialize_cellar_output(all_eclis, file_format)
     if save_enabled:
@@ -136,7 +156,14 @@ def get_cellar_extra(
     if not ed:
         ed = datetime.now().isoformat(timespec="seconds")
     save_enabled = resolve_save_enabled(save=save, save_file=save_file, default=True)
-    data = get_cellar(ed=ed, save=False, max_ecli=max_ecli, sd=sd, file_format="csv")
+    data = get_cellar(
+        ed=ed,
+        save=False,
+        max_ecli=max_ecli,
+        sd=sd,
+        file_format="csv",
+        reconcile_infocuria=True,
+    )
     if data is False:
         logging.warning("Cellar extraction unsuccessful")
         return False, False

{cellar_extractor-2.0.3 → cellar_extractor-2.0.4}/cellar_extractor/cellar_queries.py

@@ -1,6 +1,8 @@
+import re
 import time
 from datetime import date, datetime, timedelta
 
+import requests
 from SPARQLWrapper import SPARQLWrapper, JSON, POST
 
 # Literal placeholder CELLAR emits while a property is awaiting curation;
@@ -13,6 +15,17 @@ MAX_SORTED_TOP_LIMIT = 10000
 ECLI_WINDOW_DAYS = 366
 SPARQL_REQUEST_TIMEOUT_SECONDS = 30
 SPARQL_RETRY_BACKOFF_BASE_SECONDS = 0.5
+INFOCURIA_SEARCH_ENDPOINT = "https://infocuriaws.curia.europa.eu/elastic-connector/search"
+INFOCURIA_PAGE_SIZE = 100
+INFOCURIA_REQUEST_TIMEOUT_SECONDS = 60
+INFOCURIA_IDENTITY_FIELDS = {
+    "case-law_ecli",
+    "resource_legal_id_celex",
+    "work_date_document",
+    "resource_legal_type",
+    "resource_legal_id_sector",
+    "case-law_affaire_number",
+}
 
 
 def _query_with_retries(sparql, retries, error_message):
@@ -157,6 +170,192 @@ def get_all_eclis(starting_date=None, ending_date=None, limit=None, max_retries=
     return eclis
 
 
+def _normalize_infocuria_celex(value):
+    """Return a canonical primary-document CELEX from an InfoCuria hit.
+
+    InfoCuria writes numbered document variants as ``.01`` while EUR-Lex and
+    CELLAR use ``(01)``. Derived summary/information works are deliberately
+    rejected instead of being collapsed onto their base CELEX.
+    """
+    if value is None:
+        return ""
+    celex = str(value).replace(" ", "").strip()
+    if celex == "":
+        return ""
+    celex = celex.split(";", 1)[0]
+    if re.search(r"_(?:SUM|RES|INF)$", celex, flags=re.IGNORECASE):
+        return ""
+    celex = re.sub(r"\.(\d{2})$", r"(\1)", celex)
+    if not celex.startswith("6"):
+        return ""
+    return celex
+
+
+def _build_infocuria_search_payload(starting_date, ending_date, page_number, page_size):
+    start = page_number * page_size + 1
+    return {
+        "multiSearchTerms": [],
+        "searchTerm": "",
+        "ecli": "",
+        "publishedId": "",
+        "usualName": "",
+        "logicDocId": "",
+        "repJurExpand": False,
+        "pagination": {
+            "pageNumber": page_number,
+            "pageSize": page_size,
+            "from": start,
+            "to": start + page_size - 1,
+            "origin": "jurisprudence",
+        },
+        "sortTermList": [
+            {
+                "sortDirection": "ASC",
+                "sortTerm": "DOC_DATE",
+                "sortSourceTab": "jurisprudence",
+            }
+        ],
+        "filtersValue": [{"field": "docDate", "values": [starting_date, ending_date]}],
+        "advancedFiltersValue": [],
+        "language": "EN",
+        "isSearchExact": False,
+        "searchSources": ["document", "metadata"],
+        "tabName": "jurisprudence",
+        "isAllTabsRequest": False,
+    }
+
+
+def _query_infocuria_page(payload, max_retries):
+    last_error = None
+    for attempt in range(max_retries):
+        try:
+            response = requests.post(
+                INFOCURIA_SEARCH_ENDPOINT,
+                json=payload,
+                timeout=INFOCURIA_REQUEST_TIMEOUT_SECONDS,
+            )
+            response.raise_for_status()
+            result = response.json()
+            if not isinstance(result, dict):
+                raise ValueError("InfoCuria search response is not an object")
+            return result
+        except Exception as exc:
+            last_error = exc
+            if attempt < max_retries - 1:
+                time.sleep(SPARQL_RETRY_BACKOFF_BASE_SECONDS * (2**attempt))
+    raise RuntimeError(
+        "Failed to query InfoCuria document catalogue after retries"
+    ) from last_error
+
+
+def _infocuria_metadata_from_hit(hit):
+    content = hit.get("content", {}) if isinstance(hit, dict) else {}
+    if not isinstance(content, dict):
+        return None
+
+    ecli = str(content.get("ecli") or "").strip()
+    celex = _normalize_infocuria_celex(content.get("celex"))
+    document_date = str(content.get("docDate") or "").strip()
+    if not ecli.startswith("ECLI:EU:") or celex == "" or document_date == "":
+        return None
+
+    resource_type = celex[5:7] if len(celex) >= 7 else ""
+    metadata = {
+        "case-law_ecli": [ecli],
+        "resource_legal_id_celex": [celex],
+        "work_date_document": [document_date],
+        "resource_legal_id_sector": [celex[0]],
+        "metadata_catalog_source": ["infocuria"],
+    }
+    if resource_type:
+        metadata["resource_legal_type"] = [resource_type]
+    published_id = str(content.get("idPublished") or "").strip()
+    if published_id:
+        metadata["case-law_affaire_number"] = [published_id]
+    return ecli, metadata
+
+
+def get_infocuria_document_metadata(
+    starting_date=None,
+    ending_date=None,
+    limit=None,
+    max_retries=3,
+):
+    """Enumerate official InfoCuria documents for a date range.
+
+    CELLAR's SPARQL graph does not contain every document that is available
+    through EUR-Lex/InfoCuria, particularly procedural orders. InfoCuria is
+    therefore used as a second catalogue source. Results use the same
+    predicate-map shape as :func:`get_raw_cellar_metadata` so callers can
+    reconcile the sources before normal schema flattening.
+    """
+    metadata = {}
+    for window_start, window_end in _build_ecli_windows(
+        starting_date=starting_date,
+        ending_date=ending_date,
+    ):
+        page_number = 0
+        while True:
+            remaining = None if limit is None else limit - len(metadata)
+            if remaining is not None and remaining <= 0:
+                return metadata
+            page_size = INFOCURIA_PAGE_SIZE
+
+            payload = _build_infocuria_search_payload(
+                str(window_start)[:10],
+                str(window_end)[:10],
+                page_number,
+                page_size,
+            )
+            result = _query_infocuria_page(payload, max_retries=max_retries)
+            hits = result.get("searchHits", [])
+            if not isinstance(hits, list):
+                raise RuntimeError("InfoCuria catalogue returned invalid searchHits")
+
+            for hit in hits:
+                parsed = _infocuria_metadata_from_hit(hit)
+                if parsed is None:
+                    continue
+                ecli, values = parsed
+                # Multiple logical documents can share an ECLI. Their CELEX
+                # identity is normally identical; retaining the first hit is
+                # deterministic because the endpoint is date-sorted.
+                metadata.setdefault(ecli, values)
+                if limit is not None and len(metadata) >= limit:
+                    return metadata
+
+            total_hits = int(result.get("totalHits") or 0)
+            consumed = (page_number + 1) * page_size
+            if not hits or consumed >= total_hits:
+                break
+            page_number += 1
+    return metadata
+
+
+def reconcile_document_metadata(cellar_metadata, infocuria_metadata):
+    """Merge both catalogues, preferring InfoCuria for document identity.
+
+    Rich CELLAR metadata is retained. InfoCuria supplies missing documents
+    and corrects identity fields when CELLAR associates an ECLI with a stale
+    or sibling CELEX work.
+    """
+    reconciled = {
+        ecli: {key: list(values) for key, values in values_by_key.items()}
+        for ecli, values_by_key in (cellar_metadata or {}).items()
+    }
+    for ecli, values_by_key in (infocuria_metadata or {}).items():
+        if ecli not in reconciled:
+            reconciled[ecli] = {
+                key: list(values) for key, values in values_by_key.items()
+            }
+            continue
+        target = reconciled[ecli]
+        for key, values in values_by_key.items():
+            if key in INFOCURIA_IDENTITY_FIELDS or not target.get(key):
+                target[key] = list(values)
+    return reconciled
+
+
 def get_raw_cellar_metadata_by_celex(
     celex_ids,
     get_labels=True,

{cellar_extractor-2.0.3 → cellar_extractor-2.0.4}/cellar_extractor/eurlex_scraping.py

@@ -126,6 +126,7 @@ def _normalize_celex(celex):
         value = non_inf[0] if non_inf else options[0]
     if "_" in value:
         value = value.split("_")[0]
+    value = re.sub(r"\.(\d{2})$", r"(\1)", value)
     return value
 
 
@@ -814,7 +815,7 @@ def _collect_affecting_ids(content):
     return ids
 
 
-def _choose_best_document(doc_hits, language="EN"):
+def _choose_best_document(doc_hits, language="EN", celex=None):
     candidates = []
     for hit in doc_hits or []:
         content = hit.get("content", {}) if isinstance(hit, dict) else {}
@@ -827,6 +828,23 @@ def _choose_best_document(doc_hits, language="EN"):
     if len(candidates) == 0:
         return None
 
+    normalized_target = _normalize_celex(celex) if celex else ""
+    if normalized_target:
+        candidates_with_celex = [
+            doc for doc in candidates if _normalize_celex(doc.get("celex", ""))
+        ]
+        exact_candidates = [
+            doc
+            for doc in candidates_with_celex
+            if _normalize_celex(doc.get("celex", "")) == normalized_target
+        ]
+        if exact_candidates:
+            candidates = exact_candidates
+        elif candidates_with_celex:
+            # Selecting a judgment merely because it is the highest-ranked
+            # document can attach a sibling judgment to an order CELEX.
+            return None
+
     type_priority = {
         "ARRET": 0,
         "ORDONNANCE": 1,
@@ -1028,7 +1046,11 @@ def _get_case_data_sector6(celex, language="EN"):
         if isinstance(root_hit, dict)
         else []
     )
-    selected_doc = _choose_best_document(documents, language=language)
+    selected_doc = _choose_best_document(
+        documents,
+        language=language,
+        celex=normalized,
+    )
     if selected_doc is None:
         return _get_case_data_sector6_cellar_fallback(normalized, language=language)
 
@@ -1084,18 +1106,41 @@ def _get_case_data_sector6(celex, language="EN"):
     else:
         summary_source = ""
 
-    # Multi-language fanout: InfoCuria's `documents.searchHits` already
-    # carries every docLang variant of the procedure. Fetch each blob the
-    # same way the primary one was fetched and build a fulltexts list.
+    # Multi-language fanout must remain within the selected logical document.
+    # A procedure can contain judgments, orders, opinions, and notices. The
+    # selected document's groupByLogicalId list is the authoritative language
+    # family; older responses without that field fall back to documents with
+    # the same logicDocId.
     fulltexts: list = []
     seen_langs: set = set()
     session = _get_http_session()
-    for doc in documents or []:
-        if not isinstance(doc, dict):
-            continue
-        content = doc.get("content", {}) if isinstance(doc, dict) else {}
-        if not isinstance(content, dict):
-            continue
+    selected_logic_id = str(selected_doc.get("logicDocId", ""))
+    selected_group = selected_doc.get("groupByLogicalId") or []
+    variant_contents = []
+    if isinstance(selected_group, list) and selected_group:
+        for variant in selected_group:
+            if not isinstance(variant, dict):
+                continue
+            variant_contents.append(
+                {
+                    "docLang": variant.get("docLang"),
+                    "docFormats": variant.get("formats") or [],
+                    "logicDocId": selected_logic_id,
+                    "idProcedure": variant.get("idProcedure")
+                    or selected_doc.get("idProcedure"),
+                }
+            )
+    else:
+        for doc in documents or []:
+            content = doc.get("content", {}) if isinstance(doc, dict) else {}
+            if not isinstance(content, dict):
+                continue
+            if str(content.get("logicDocId", "")) == selected_logic_id:
+                variant_contents.append(content)
+    if not variant_contents:
+        variant_contents = [selected_doc]
+
+    for content in variant_contents:
         variant_lang = content.get("docLang") or ""
         variant_lang_upper = str(variant_lang).upper()
         if variant_lang_upper == "" or variant_lang_upper in seen_langs:

{cellar_extractor-2.0.3 → cellar_extractor-2.0.4/cellar_extractor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cellar-extractor
-Version: 2.0.3
+Version: 2.0.4
 Summary: Library for extracting CELLAR case law data from EUR-Lex
 Author-email: LawTech Lab <law-techlab@maastrichtuniversity.nl>
 License: Apache-2.0

cellar_extractor-2.0.4/cellar_extractor.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "2.0.4",
+  "distance": 0,
+  "node": "g792a41d4a8dc60a4b19153a4e294b31ffb487d1d",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-09-30"
+}

{cellar_extractor-2.0.3 → cellar_extractor-2.0.4}/tests/test_cellar.py

@@ -25,7 +25,9 @@ def test_get_cellar_csv_in_memory(monkeypatch):
     monkeypatch.setattr(
         cellar,
         "get_all_eclis",
-        lambda starting_date, ending_date, limit=None: ["E1", "E2"][:limit] if limit else ["E1", "E2"],
+        lambda starting_date, ending_date, limit=None: ["E1", "E2"][:limit]
+        if limit
+        else ["E1", "E2"],
     )
     monkeypatch.setattr(
         cellar,
@@ -50,7 +52,9 @@ def test_get_cellar_csv_in_memory(monkeypatch):
 
 
 def test_get_cellar_json_in_memory(monkeypatch):
-    monkeypatch.setattr(cellar, "get_all_eclis", lambda starting_date, ending_date, limit=None: ["E1"])
+    monkeypatch.setattr(
+        cellar, "get_all_eclis", lambda starting_date, ending_date, limit=None: ["E1"]
+    )
     monkeypatch.setattr(
         cellar,
         "get_raw_cellar_metadata",
@@ -71,7 +75,9 @@ def test_get_cellar_json_in_memory(monkeypatch):
 
 def test_get_cellar_json_save_file(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)
-    monkeypatch.setattr(cellar, "get_all_eclis", lambda starting_date, ending_date, limit=None: ["E1"])
+    monkeypatch.setattr(
+        cellar, "get_all_eclis", lambda starting_date, ending_date, limit=None: ["E1"]
+    )
     monkeypatch.setattr(
         cellar,
         "get_raw_cellar_metadata",
@@ -94,7 +100,9 @@ def test_get_cellar_json_save_file(monkeypatch, tmp_path):
 
 def test_get_cellar_in_memory_does_not_create_default_output_dir(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)
-    monkeypatch.setattr(cellar, "get_all_eclis", lambda starting_date, ending_date, limit=None: ["E1"])
+    monkeypatch.setattr(
+        cellar, "get_all_eclis", lambda starting_date, ending_date, limit=None: ["E1"]
+    )
     monkeypatch.setattr(
         cellar,
         "get_raw_cellar_metadata",
@@ -113,7 +121,9 @@ def test_get_cellar_in_memory_does_not_create_default_output_dir(monkeypatch, tm
 
 
 def test_get_cellar_save_file_supports_custom_output_path(monkeypatch, tmp_path):
-    monkeypatch.setattr(cellar, "get_all_eclis", lambda starting_date, ending_date, limit=None: ["E1"])
+    monkeypatch.setattr(
+        cellar, "get_all_eclis", lambda starting_date, ending_date, limit=None: ["E1"]
+    )
     monkeypatch.setattr(
         cellar,
         "get_raw_cellar_metadata",
@@ -136,8 +146,12 @@ def test_get_cellar_save_file_supports_custom_output_path(monkeypatch, tmp_path)
     assert result.loc[0, "celex"] == "62025CJ0001"
 
 
-def test_get_cellar_save_to_output_dir_creates_only_requested_parents(monkeypatch, tmp_path):
-    monkeypatch.setattr(cellar, "get_all_eclis", lambda starting_date, ending_date, limit=None: ["E1"])
+def test_get_cellar_save_to_output_dir_creates_only_requested_parents(
+    monkeypatch, tmp_path
+):
+    monkeypatch.setattr(
+        cellar, "get_all_eclis", lambda starting_date, ending_date, limit=None: ["E1"]
+    )
     monkeypatch.setattr(
         cellar,
         "get_raw_cellar_metadata",
@@ -189,7 +203,9 @@ def test_get_cellar_preserves_ecli_order_across_parallel_metadata_batches(monkey
     monkeypatch.setattr(
         cellar,
         "get_all_eclis",
-        lambda starting_date, ending_date, limit=None: eclis[:limit] if limit else eclis,
+        lambda starting_date, ending_date, limit=None: eclis[:limit]
+        if limit
+        else eclis,
     )
 
     def _fake_get_raw_cellar_metadata(batch):
@@ -206,7 +222,9 @@ def test_get_cellar_preserves_ecli_order_across_parallel_metadata_batches(monkey
             for index, ecli in enumerate(batch)
         }
 
-    monkeypatch.setattr(cellar, "get_raw_cellar_metadata", _fake_get_raw_cellar_metadata)
+    monkeypatch.setattr(
+        cellar, "get_raw_cellar_metadata", _fake_get_raw_cellar_metadata
+    )
 
     df = cellar.get_cellar(
         ed="2025-01-02T00:00:00",
@@ -221,10 +239,22 @@ def test_get_cellar_preserves_ecli_order_across_parallel_metadata_batches(monkey
 
 def test_get_cellar_extra_in_memory_calls_extra(monkeypatch):
     base_df = pd.DataFrame({"ecli": ["E1"], "celex": ["62025CJ0001"]})
-    monkeypatch.setattr(cellar, "get_cellar", lambda **kwargs: base_df)
     called = {}
 
-    def _fake_extra(data, threads, username, password, metadata_output_path=None, fulltext_output_path=None):
+    def _fake_get_cellar(**kwargs):
+        called["reconcile_infocuria"] = kwargs.get("reconcile_infocuria")
+        return base_df
+
+    monkeypatch.setattr(cellar, "get_cellar", _fake_get_cellar)
+
+    def _fake_extra(
+        data,
+        threads,
+        username,
+        password,
+        metadata_output_path=None,
+        fulltext_output_path=None,
+    ):
         called["threads"] = threads
         called["username"] = username
         called["password"] = password
@@ -247,6 +277,7 @@ def test_get_cellar_extra_in_memory_calls_extra(monkeypatch):
     assert len(data) == 1
     assert len(fulltext) == 1
     assert called == {
+        "reconcile_infocuria": True,
         "threads": 4,
         "username": "user",
         "password": "pass",
@@ -260,7 +291,14 @@ def test_get_cellar_extra_save_file_calls_extra_with_path(monkeypatch, tmp_path)
     monkeypatch.setattr(cellar, "get_cellar", lambda **kwargs: base_df)
     called = {}
 
-    def _fake_extra(data, threads, username, password, metadata_output_path=None, fulltext_output_path=None):
+    def _fake_extra(
+        data,
+        threads,
+        username,
+        password,
+        metadata_output_path=None,
+        fulltext_output_path=None,
+    ):
         called["metadata_output_path"] = metadata_output_path
         called["fulltext_output_path"] = fulltext_output_path
         called["threads"] = threads
@@ -278,11 +316,15 @@ def test_get_cellar_extra_save_file_calls_extra_with_path(monkeypatch, tmp_path)
         threads=3,
     )
 
-    assert str(called["metadata_output_path"]).replace("\\", "/").endswith(
-        "data/cellar_extra_2025-01-01_2025-01-02T00_00_00.csv"
+    assert (
+        str(called["metadata_output_path"])
+        .replace("\\", "/")
+        .endswith("data/cellar_extra_2025-01-01_2025-01-02T00_00_00.csv")
     )
-    assert str(called["fulltext_output_path"]).replace("\\", "/").endswith(
-        "data/cellar_extra_2025-01-01_2025-01-02T00_00_00_fulltext.json"
+    assert (
+        str(called["fulltext_output_path"])
+        .replace("\\", "/")
+        .endswith("data/cellar_extra_2025-01-01_2025-01-02T00_00_00_fulltext.json")
     )
     assert called["threads"] == 3
 
@@ -292,7 +334,14 @@ def test_get_cellar_extra_supports_independent_output_paths(monkeypatch, tmp_pat
     monkeypatch.setattr(cellar, "get_cellar", lambda **kwargs: base_df)
     called = {}
 
-    def _fake_extra(data, threads, username, password, metadata_output_path=None, fulltext_output_path=None):
+    def _fake_extra(
+        data,
+        threads,
+        username,
+        password,
+        metadata_output_path=None,
+        fulltext_output_path=None,
+    ):
         called["metadata_output_path"] = metadata_output_path
         called["fulltext_output_path"] = fulltext_output_path
         return data, [{"celex": "62025CJ0001"}]
@@ -315,7 +364,9 @@ def test_get_cellar_extra_supports_independent_output_paths(monkeypatch, tmp_pat
     assert output[1] == [{"celex": "62025CJ0001"}]
 
 
-def test_get_cellar_extra_in_memory_does_not_create_default_output_dir(monkeypatch, tmp_path):
+def test_get_cellar_extra_in_memory_does_not_create_default_output_dir(
+    monkeypatch, tmp_path
+):
     monkeypatch.chdir(tmp_path)
     base_df = pd.DataFrame({"ecli": ["E1"], "celex": ["62025CJ0001"]})
     monkeypatch.setattr(cellar, "get_cellar", lambda **kwargs: base_df)
@@ -337,7 +388,9 @@ def test_get_cellar_extra_in_memory_does_not_create_default_output_dir(monkeypat
 
 
 def test_get_cellar_save_file_alias_still_works(monkeypatch):
-    monkeypatch.setattr(cellar, "get_all_eclis", lambda starting_date, ending_date, limit=None: ["E1"])
+    monkeypatch.setattr(
+        cellar, "get_all_eclis", lambda starting_date, ending_date, limit=None: ["E1"]
+    )
     monkeypatch.setattr(
         cellar,
         "get_raw_cellar_metadata",

{cellar_extractor-2.0.3 → cellar_extractor-2.0.4}/tests/test_cellar_queries_local.py

@@ -103,7 +103,9 @@ def test_get_raw_cellar_metadata_filters_requested_eclis(monkeypatch):
             "bindings": [
                 {
                     "ecli": {"value": "ECLI:EU:C:2025:1"},
-                    "p": {"value": "http://publications.europa.eu/ontology/cdm#case-law_ecli"},
+                    "p": {
+                        "value": "http://publications.europa.eu/ontology/cdm#case-law_ecli"
+                    },
                     "o": {"value": "ECLI:EU:C:2025:1"},
                 }
             ]
@@ -215,3 +217,107 @@ def test_get_all_eclis_applies_large_limit_locally_after_chunking(monkeypatch):
 
     assert result == ["ECLI:EU:C:2025:1", "ECLI:EU:C:2025:2", "ECLI:EU:C:2025:3"]
     assert len(fake.queries) == 2
+
+
+class _FakeResponse:
+    def __init__(self, payload):
+        self.payload = payload
+
+    def raise_for_status(self):
+        return None
+
+    def json(self):
+        return self.payload
+
+
+def test_infocuria_catalog_paginates_and_normalizes_document_variants(monkeypatch):
+    calls = []
+    pages = [
+        {
+            "totalHits": 3,
+            "searchHits": [
+                {
+                    "content": {
+                        "ecli": "ECLI:EU:F:2011:62",
+                        "celex": "62011FO0005.01",
+                        "docDate": "2011-05-12",
+                        "docTypeCode": "ORDONNANCE",
+                        "idPublished": "F-5/11",
+                    }
+                },
+                {
+                    "content": {
+                        "ecli": "ECLI:EU:C:2011:1",
+                        "celex": "62011CJ0001_SUM",
+                        "docDate": "2011-05-13",
+                    }
+                },
+            ],
+        },
+        {
+            "totalHits": 3,
+            "searchHits": [
+                {
+                    "content": {
+                        "ecli": "ECLI:EU:C:2011:2",
+                        "celex": "62011CO0002",
+                        "docDate": "2011-05-14",
+                        "idPublished": "C-2/11",
+                    }
+                }
+            ],
+        },
+    ]
+
+    def _post(url, json, timeout):
+        calls.append((url, json, timeout))
+        return _FakeResponse(pages[len(calls) - 1])
+
+    monkeypatch.setattr(cellar_queries.requests, "post", _post)
+    monkeypatch.setattr(cellar_queries, "INFOCURIA_PAGE_SIZE", 2)
+
+    result = cellar_queries.get_infocuria_document_metadata(
+        starting_date="2011-05-01",
+        ending_date="2011-05-31",
+    )
+
+    assert set(result) == {"ECLI:EU:F:2011:62", "ECLI:EU:C:2011:2"}
+    assert result["ECLI:EU:F:2011:62"]["resource_legal_id_celex"] == ["62011FO0005(01)"]
+    assert result["ECLI:EU:F:2011:62"]["metadata_catalog_source"] == ["infocuria"]
+    assert result["ECLI:EU:C:2011:2"]["resource_legal_type"] == ["CO"]
+    assert calls[0][1]["pagination"]["from"] == 1
+    assert calls[1][1]["pagination"]["from"] == 3
+    assert calls[0][1]["filtersValue"] == [
+        {"field": "docDate", "values": ["2011-05-01", "2011-05-31"]}
+    ]
+
+
+def test_reconcile_document_metadata_adds_and_overrides_infocuria_identity():
+    cellar = {
+        "ECLI:EU:T:2014:1": {
+            "case-law_ecli": ["ECLI:EU:T:2014:1"],
+            "resource_legal_id_celex": ["62013TO0505(01)"],
+            "work_date_document": ["2014-01-10"],
+            "subject_matter": ["Staff cases"],
+        }
+    }
+    infocuria = {
+        "ECLI:EU:T:2014:1": {
+            "case-law_ecli": ["ECLI:EU:T:2014:1"],
+            "resource_legal_id_celex": ["62013TO0505(02)"],
+            "work_date_document": ["2014-01-10"],
+        },
+        "ECLI:EU:T:2014:166": {
+            "case-law_ecli": ["ECLI:EU:T:2014:166"],
+            "resource_legal_id_celex": ["62013TO0505(03)"],
+            "work_date_document": ["2014-04-02"],
+        },
+    }
+
+    result = cellar_queries.reconcile_document_metadata(cellar, infocuria)
+
+    assert result["ECLI:EU:T:2014:1"]["resource_legal_id_celex"] == ["62013TO0505(02)"]
+    assert result["ECLI:EU:T:2014:1"]["subject_matter"] == ["Staff cases"]
+    assert result["ECLI:EU:T:2014:166"]["resource_legal_id_celex"] == [
+        "62013TO0505(03)"
+    ]

{cellar_extractor-2.0.3 → cellar_extractor-2.0.4}/tests/test_infocuria_adapter.py

@@ -40,6 +40,51 @@ def test_choose_best_document_prefers_target_language():
     assert selected["logicDocId"] == "id_2"
 
 
+def test_choose_best_document_requires_requested_celex_before_type_priority():
+    docs = [
+        {
+            "content": {
+                "docLang": "EN",
+                "docFormats": ["HTML"],
+                "logicDocId": "id_judgment",
+                "idProcedure": "C/0444/11/00000000RP/01/P/01",
+                "docTypeCode": "ARRET",
+                "celex": "62011CJ0444",
+            }
+        },
+        {
+            "content": {
+                "docLang": "EN",
+                "docFormats": ["HTML"],
+                "logicDocId": "id_order",
+                "idProcedure": "C/0444/11/00000000RP/01/P/01",
+                "docTypeCode": "ORDONNANCE",
+                "celex": "62011CO0444",
+            }
+        },
+    ]
+
+    selected = eurlex_scraping._choose_best_document(
+        docs,
+        language="EN",
+        celex="62011CO0444",
+    )
+
+    assert selected["logicDocId"] == "id_order"
+    assert (
+        eurlex_scraping._choose_best_document(
+            docs,
+            language="EN",
+            celex="62011CC0444",
+        )
+        is None
+    )
+
+
+def test_normalize_celex_converts_infocuria_dot_variant():
+    assert eurlex_scraping.normalize_celex("62011FO0005.01") == "62011FO0005(01)"
+
+
 def test_extract_summary_from_documents_prefers_summary_marker():
     docs = [
         {
@@ -103,9 +148,13 @@ def test_get_case_data_by_celex_id_builds_blob_request(monkeypatch):
                         "content": {
                             "matCodeML": [{"label": [{"en": "Environment"}]}],
                             "matCode": ["ENVI"],
-                            "advocateML": [{"code": "KOK", "label": [{"en": "Kokott"}]}],
+                            "advocateML": [
+                                {"code": "KOK", "label": [{"en": "Kokott"}]}
+                            ],
                             "avg": "KOK",
-                            "reportingJudgeML": [{"code": "SGE", "label": [{"en": "Spielmann"}]}],
+                            "reportingJudgeML": [
+                                {"code": "SGE", "label": [{"en": "Spielmann"}]}
+                            ],
                             "reportingJudge": "SGE",
                             "joinAffairs": ["C-2/20"],
                             "procedureResultTypeML": [{"label": [{"en": "Judgment"}]}],
@@ -158,7 +207,8 @@ def test_get_case_data_by_celex_id_builds_blob_request(monkeypatch):
     # noops — supplementation behaviour itself is covered in
     # test_sector6_cellar_supplement.py.
     monkeypatch.setattr(
-        eurlex_scraping, "_fetch_sector8_work_uri",
+        eurlex_scraping,
+        "_fetch_sector8_work_uri",
         lambda celex, sector="8": "",
     )
 

{cellar_extractor-2.0.3 → cellar_extractor-2.0.4}/tests/test_multilang_fulltext.py

@@ -130,6 +130,15 @@ def test_sector6_infocuria_returns_fulltexts_list_with_all_languages(monkeypatch
                                             "docTypeCode": "ARRET",
                                         }
                                     },
+                                    {
+                                        "content": {
+                                            "docLang": "IT",
+                                            "docFormats": ["HTML"],
+                                            "logicDocId": "id_unrelated_opinion",
+                                            "idProcedure": "C/0131/24/00000000RP/01/P/01",
+                                            "docTypeCode": "CONCL",
+                                        }
+                                    },
                                 ]
                             }
                         },
@@ -249,7 +258,9 @@ def test_sector6_cellar_fallback_returns_all_languages(monkeypatch):
         {"item_url": "u_it", "format": "xhtml", "language": "IT"},
         {"item_url": "u_fr", "format": "html", "language": "FR"},
     ]
-    monkeypatch.setattr(eurlex_scraping, "_post_json", lambda url, payload, retries=3: [])
+    monkeypatch.setattr(
+        eurlex_scraping, "_post_json", lambda url, payload, retries=3: []
+    )
     monkeypatch.setattr(
         eurlex_scraping, "_fetch_sector8_work_uri", lambda celex, sector="8": work_uri
     )
@@ -357,6 +368,8 @@ def test_build_fulltext_records_normalizes_composite_celex():
             "missing_reasons": "",
         }
     ]
+
+
 def test_build_fulltext_records_falls_back_to_single_when_no_fulltexts_list():
     """Backwards-compat: legacy infocuria_data dicts that don't carry a
     `fulltexts` list (e.g. from a plug-in or stub) still produce exactly

cellar_extractor-2.0.3/cellar_extractor.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "2.0.3",
-  "distance": 0,
-  "node": "g1aaeae8096046eeeb4e5ab1b7507f92cda10057c",
-  "dirty": false,
-  "branch": "HEAD",
-  "node_date": "2026-09-15"
-}