cellar-extractor 2.0.2__tar.gz → 2.0.3__tar.gz

{cellar_extractor-2.0.2/cellar_extractor.egg-info → cellar_extractor-2.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cellar-extractor
-Version: 2.0.2
+Version: 2.0.3
 Summary: Library for extracting CELLAR case law data from EUR-Lex
 Author-email: LawTech Lab <law-techlab@maastrichtuniversity.nl>
 License: Apache-2.0
@@ -127,6 +127,11 @@ df = cell.get_cellar(
 
 Returns a dataframe with base metadata such as CELEX, ECLI, type, dates, and subject-matter-related fields.
 
+For direct manifestation access, use
+`get_cellar_manifestations_by_celex()`. It canonicalizes composite and
+`_SUM`/`_RES`/`_INF` identifiers to the base work before querying, preventing
+derived summaries or notices from being returned as judgment full text.
+
 You can also save explicitly to a custom path instead of the default `data/` location:
 
 ```python
@@ -153,6 +158,20 @@ extra_df, fulltext = cell.get_cellar_extra(
 )
 ```
 
+For targeted CELEX repair or supplementation, use the public CELLAR
+manifestation API. It unions every CELLAR work sharing the CELEX before choosing
+the best downloadable item per language:
+
+```python
+import cellar_extractor as cell
+
+_, manifestations = cell.get_cellar_manifestations_by_celex(
+    "62020CJ0414", sector="6"
+)
+english = [m for m in manifestations if m["language"] == "EN"]
+fulltexts = cell.extract_cellar_fulltexts(english)
+```
+
 Returns:
 
 - `extra_df`: enriched dataframe
@@ -303,6 +322,8 @@ Imported from [`cellar_extractor/__init__.py`](/Users/davidwickerhf/Projects/wor
 | `get_cellar(...)` | Fetch base CELLAR metadata (case law only) |
 | `get_cellar_extra(...)` | Fetch enriched metadata + full text (case law only) |
 | `get_legislation_by_celex_id(celex, language="EN")` | Fetch sector 3 / sector 0 legislation XHTML by CELEX |
+| `get_cellar_manifestations_by_celex(celex, sector="8")` | Resolve every CELLAR work for a CELEX and return their deduplicated manifestation union |
+| `extract_cellar_fulltexts(manifestations, source_label="CELLAR_ITEM")` | Download the best manifestation per language as fulltext records |
 | `get_nodes_and_edges_lists(df, only_local=False)` | Build citation graph lists |
 | `filter_subject_matter(df, phrase)` | Filter dataframe by subject phrase |
 | `FetchOperativePart` | Extract operative part from a single case document |

{cellar_extractor-2.0.2 → cellar_extractor-2.0.3}/README.md

@@ -92,6 +92,11 @@ df = cell.get_cellar(
 
 Returns a dataframe with base metadata such as CELEX, ECLI, type, dates, and subject-matter-related fields.
 
+For direct manifestation access, use
+`get_cellar_manifestations_by_celex()`. It canonicalizes composite and
+`_SUM`/`_RES`/`_INF` identifiers to the base work before querying, preventing
+derived summaries or notices from being returned as judgment full text.
+
 You can also save explicitly to a custom path instead of the default `data/` location:
 
 ```python
@@ -118,6 +123,20 @@ extra_df, fulltext = cell.get_cellar_extra(
 )
 ```
 
+For targeted CELEX repair or supplementation, use the public CELLAR
+manifestation API. It unions every CELLAR work sharing the CELEX before choosing
+the best downloadable item per language:
+
+```python
+import cellar_extractor as cell
+
+_, manifestations = cell.get_cellar_manifestations_by_celex(
+    "62020CJ0414", sector="6"
+)
+english = [m for m in manifestations if m["language"] == "EN"]
+fulltexts = cell.extract_cellar_fulltexts(english)
+```
+
 Returns:
 
 - `extra_df`: enriched dataframe
@@ -268,6 +287,8 @@ Imported from [`cellar_extractor/__init__.py`](/Users/davidwickerhf/Projects/wor
 | `get_cellar(...)` | Fetch base CELLAR metadata (case law only) |
 | `get_cellar_extra(...)` | Fetch enriched metadata + full text (case law only) |
 | `get_legislation_by_celex_id(celex, language="EN")` | Fetch sector 3 / sector 0 legislation XHTML by CELEX |
+| `get_cellar_manifestations_by_celex(celex, sector="8")` | Resolve every CELLAR work for a CELEX and return their deduplicated manifestation union |
+| `extract_cellar_fulltexts(manifestations, source_label="CELLAR_ITEM")` | Download the best manifestation per language as fulltext records |
 | `get_nodes_and_edges_lists(df, only_local=False)` | Build citation graph lists |
 | `filter_subject_matter(df, phrase)` | Filter dataframe by subject phrase |
 | `FetchOperativePart` | Extract operative part from a single case document |

{cellar_extractor-2.0.2 → cellar_extractor-2.0.3}/cellar_extractor/__init__.py

@@ -9,6 +9,9 @@ from cellar_extractor.cellar import get_cellar_extra
 from cellar_extractor.cellar import get_nodes_and_edges_lists
 from cellar_extractor.cellar import filter_subject_matter
 from cellar_extractor.eurlex_scraping import get_legislation_by_celex_id
+from cellar_extractor.eurlex_scraping import get_cellar_manifestations_by_celex
+from cellar_extractor.eurlex_scraping import extract_cellar_fulltexts
+from cellar_extractor.eurlex_scraping import normalize_celex
 from cellar_extractor.operative_extractions import FetchOperativePart
 from cellar_extractor.operative_extractions import Writing
 import logging

{cellar_extractor-2.0.2 → cellar_extractor-2.0.3}/cellar_extractor/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '2.0.2'
-__version_tuple__ = version_tuple = (2, 0, 2)
+__version__ = version = '2.0.3'
+__version_tuple__ = version_tuple = (2, 0, 3)
 
-__commit_id__ = commit_id = 'gf354af594'
+__commit_id__ = commit_id = 'g1aaeae809'

{cellar_extractor-2.0.2 → cellar_extractor-2.0.3}/cellar_extractor/eurlex_scraping.py

@@ -129,6 +129,17 @@ def _normalize_celex(celex):
     return value
 
 
+def normalize_celex(celex):
+    """Return the canonical work CELEX used for full-text retrieval.
+
+    CELLAR metadata can bundle a primary document with derived information,
+    summary, and résumé works (for example ``62020CJ0414_SUM;62020CJ0414``).
+    Full-text callers must resolve the unsuffixed base work so a derived work
+    can never occupy the judgment's language slot.
+    """
+    return _normalize_celex(celex)
+
+
 def _sleep_with_backoff(attempt, base=0.2):
     time.sleep(base * (attempt + 1) + random.uniform(0.0, 0.1))
 
@@ -413,6 +424,25 @@ def _fetch_sector8_items_for_celex(celex, sector="8"):
     return work_uris, candidates
 
 
+def get_cellar_manifestations_by_celex(celex, sector="8"):
+    """Return all CELLAR works and manifestation candidates for a CELEX.
+
+    A CELEX can resolve to multiple CELLAR works with different language
+    coverage.  This public entry point deliberately returns the union across
+    every matching work so callers do not need to depend on the extractor's
+    private sector-8 helpers.
+
+    The return value is ``(work_uris, manifestations)``.  Each manifestation
+    contains ``item_url``, ``format``, and ``language`` keys and is deduplicated
+    on that triple.  ``sector`` defaults to ``"8"`` and may be set to ``"6"``
+    for CJEU documents.
+    """
+    canonical_celex = normalize_celex(celex)
+    if not canonical_celex:
+        return [], []
+    return _fetch_sector8_items_for_celex(canonical_celex, sector=sector)
+
+
 def _fetch_sector8_work_uri(celex, sector="8"):
     """Resolve a CELEX to its CELLAR work URI.
 
@@ -553,6 +583,17 @@ def _fanout_fulltexts_from_candidates(candidates, source_label):
     return out
 
 
+def extract_cellar_fulltexts(manifestations, source_label="CELLAR_ITEM"):
+    """Download the best manifestation per language as fulltext records.
+
+    ``manifestations`` is the candidate list returned by
+    :func:`get_cellar_manifestations_by_celex`.  Empty bodies are omitted and
+    each returned dictionary contains ``text``, ``html``, ``text_source``,
+    ``text_language``, and ``text_format``.
+    """
+    return _fanout_fulltexts_from_candidates(manifestations, source_label)
+
+
 def _get_case_data_sector8(celex, language="EN"):
     work_uris, main_candidates = _fetch_sector8_items_for_celex(celex)
 
@@ -1104,15 +1145,14 @@ def _get_case_data_sector6(celex, language="EN"):
             }
         )
 
-    # Supplement InfoCuria fulltexts with any languages CELLAR has that
-    # InfoCuria didn't expose. InfoCuria's documents.searchHits typically
-    # carries only the procedural language plus EN; CELLAR's CDM model
-    # exposes all 23 EU-official manifestations via expression_uses_language.
-    # We keep InfoCuria's entries where languages overlap (the court's own
-    # publication is typically higher fidelity) and append CELLAR-sourced
-    # entries for any missing languages. Metadata fields (judge, advocate,
-    # directory_codes, etc.) remain InfoCuria-sourced — CELLAR cannot
-    # populate them.
+    # Merge InfoCuria fulltexts with every language CELLAR has. CELLAR is
+    # authoritative when both sources expose the same language because its
+    # manifestation belongs to the canonical CELEX work. InfoCuria's
+    # procedure search has returned a different document under the judgment
+    # slot in production (AG opinions, procedural orders, and notices), so an
+    # existing InfoCuria language must not block the canonical manifestation.
+    # Metadata fields (judge, advocate, directory_codes, etc.) remain
+    # InfoCuria-sourced — CELLAR cannot populate them.
     #
     # All-in-one try/except: a CELLAR-side failure must never kill the
     # InfoCuria-sourced row we already built.
@@ -1122,18 +1162,36 @@ def _get_case_data_sector6(celex, language="EN"):
             cellar_fulltexts = _fanout_fulltexts_from_candidates(
                 cellar_candidates, source_label="CELLAR_ITEM"
             )
+            by_language = {
+                entry.get("text_language", "").upper(): entry
+                for entry in fulltexts
+                if entry.get("text_language", "")
+            }
             for entry in cellar_fulltexts:
                 entry_lang = entry.get("text_language", "").upper()
-                if not entry_lang or entry_lang in seen_langs:
+                if not entry_lang:
                     continue
                 seen_langs.add(entry_lang)
-                fulltexts.append(entry)
+                by_language[entry_lang] = entry
+            fulltexts = list(by_language.values())
     except Exception:
         # CELLAR supplementation is best-effort. If anything goes wrong
         # (SPARQL timeout, network hiccup, schema drift on the manifestation
         # graph) the InfoCuria-only result is still returned.
         pass
 
+    primary = next(
+        (
+            entry
+            for entry in fulltexts
+            if entry.get("text_language", "").upper() == str(language).upper()
+        ),
+        None,
+    )
+    if primary is not None:
+        text = primary.get("text", "")
+        html = primary.get("html", "")
+
     return {
         "html": html,
         "text": text,
@@ -1146,9 +1204,9 @@ def _get_case_data_sector6(celex, language="EN"):
         "affecting_ids": affecting_ids,
         "affecting_string": affecting_string,
         "citations_extra": citations_extra,
-        "text_source": "INFOCURIA_BLOB_HTML" if text != "" else "",
-        "text_language": str(doc_lang).upper() if doc_lang else language.upper(),
-        "text_format": "html" if text != "" else "",
+        "text_source": primary.get("text_source", "") if primary else "",
+        "text_language": primary.get("text_language", "") if primary else "",
+        "text_format": primary.get("text_format", "") if primary else "",
         "summary_source": summary_source,
         "summary_language": "EN" if summary != "" else "",
         "sector": "6",

{cellar_extractor-2.0.2 → cellar_extractor-2.0.3/cellar_extractor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cellar-extractor
-Version: 2.0.2
+Version: 2.0.3
 Summary: Library for extracting CELLAR case law data from EUR-Lex
 Author-email: LawTech Lab <law-techlab@maastrichtuniversity.nl>
 License: Apache-2.0
@@ -127,6 +127,11 @@ df = cell.get_cellar(
 
 Returns a dataframe with base metadata such as CELEX, ECLI, type, dates, and subject-matter-related fields.
 
+For direct manifestation access, use
+`get_cellar_manifestations_by_celex()`. It canonicalizes composite and
+`_SUM`/`_RES`/`_INF` identifiers to the base work before querying, preventing
+derived summaries or notices from being returned as judgment full text.
+
 You can also save explicitly to a custom path instead of the default `data/` location:
 
 ```python
@@ -153,6 +158,20 @@ extra_df, fulltext = cell.get_cellar_extra(
 )
 ```
 
+For targeted CELEX repair or supplementation, use the public CELLAR
+manifestation API. It unions every CELLAR work sharing the CELEX before choosing
+the best downloadable item per language:
+
+```python
+import cellar_extractor as cell
+
+_, manifestations = cell.get_cellar_manifestations_by_celex(
+    "62020CJ0414", sector="6"
+)
+english = [m for m in manifestations if m["language"] == "EN"]
+fulltexts = cell.extract_cellar_fulltexts(english)
+```
+
 Returns:
 
 - `extra_df`: enriched dataframe
@@ -303,6 +322,8 @@ Imported from [`cellar_extractor/__init__.py`](/Users/davidwickerhf/Projects/wor
 | `get_cellar(...)` | Fetch base CELLAR metadata (case law only) |
 | `get_cellar_extra(...)` | Fetch enriched metadata + full text (case law only) |
 | `get_legislation_by_celex_id(celex, language="EN")` | Fetch sector 3 / sector 0 legislation XHTML by CELEX |
+| `get_cellar_manifestations_by_celex(celex, sector="8")` | Resolve every CELLAR work for a CELEX and return their deduplicated manifestation union |
+| `extract_cellar_fulltexts(manifestations, source_label="CELLAR_ITEM")` | Download the best manifestation per language as fulltext records |
 | `get_nodes_and_edges_lists(df, only_local=False)` | Build citation graph lists |
 | `filter_subject_matter(df, phrase)` | Filter dataframe by subject phrase |
 | `FetchOperativePart` | Extract operative part from a single case document |

cellar_extractor-2.0.3/cellar_extractor.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "2.0.3",
+  "distance": 0,
+  "node": "g1aaeae8096046eeeb4e5ab1b7507f92cda10057c",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-09-15"
+}

{cellar_extractor-2.0.2 → cellar_extractor-2.0.3}/tests/test_metadata_hygiene_local.py

@@ -6,6 +6,7 @@
    at the metadata assembly point, in both the celex- and ecli-keyed loops.
 """
 
+import cellar_extractor as cell
 import cellar_extractor.cellar_queries as cq
 import cellar_extractor.eurlex_scraping as es
 
@@ -132,3 +133,36 @@ def test_items_for_celex_unions_across_all_works(monkeypatch):
     assert len(uris) == 2
     langs = sorted(c["language"] for c in cands)
     assert langs == ["DE", "EN", "FR", "NL"]  # union, DE deduped
+
+
+def test_public_manifestation_and_fulltext_api(monkeypatch):
+    manifestations = [
+        {"item_url": "u_en", "format": "xhtml", "language": "EN"},
+        {"item_url": "u_fr", "format": "xhtml", "language": "FR"},
+    ]
+    monkeypatch.setattr(
+        es,
+        "_fetch_sector8_items_for_celex",
+        lambda celex, sector="8": ([f"http://cellar/{sector}/{celex}"], manifestations),
+    )
+
+    works, found = cell.get_cellar_manifestations_by_celex(
+        "62024CJ0001_SUM;62024CJ0001", sector="6"
+    )
+    assert works == ["http://cellar/6/62024CJ0001"]
+    assert found == manifestations
+    assert cell.normalize_celex("62024CJ0001_SUM;62024CJ0001") == "62024CJ0001"
+
+    monkeypatch.setattr(
+        es,
+        "_fanout_fulltexts_from_candidates",
+        lambda candidates, source_label: [
+            {"text_language": candidate["language"], "text_source": source_label}
+            for candidate in candidates
+        ],
+    )
+    rows = cell.extract_cellar_fulltexts(found, source_label="CELLAR_ITEM")
+    assert rows == [
+        {"text_language": "EN", "text_source": "CELLAR_ITEM"},
+        {"text_language": "FR", "text_source": "CELLAR_ITEM"},
+    ]

{cellar_extractor-2.0.2 → cellar_extractor-2.0.3}/tests/test_sector6_cellar_supplement.py

@@ -7,8 +7,8 @@ languages for the same work.
 
 After this change, sector 6 always supplements InfoCuria's fulltexts with
 CELLAR's manifestation graph (unconditionally — not env-var gated), so the
-``fulltexts`` list contains every language CELLAR has, with InfoCuria's
-entries kept verbatim where both sources have the same language.
+``fulltexts`` list contains every language CELLAR has, with canonical CELLAR
+manifestations replacing InfoCuria entries where both sources overlap.
 """
 
 from __future__ import annotations
@@ -154,9 +154,8 @@ def test_infocuria_success_is_supplemented_with_cellar_languages(monkeypatch):
     assert langs == {"EN", "FR", "DE", "IT", "NL", "ES", "PT"}
 
 
-def test_infocuria_entries_are_preserved_when_languages_overlap(monkeypatch):
-    """When both sources have the same language, keep InfoCuria's entry —
-    it's the court's own publication and typically has higher fidelity."""
+def test_cellar_entries_replace_infocuria_when_languages_overlap(monkeypatch):
+    """Canonical CELLAR works win overlaps to prevent wrong-doc captures."""
     eurlex_scraping._get_case_data_cached.cache_clear()
     _patch_infocuria(monkeypatch, doc_langs=["EN", "FR"])
     _patch_cellar(monkeypatch, languages=["EN", "FR", "DE"])
@@ -164,14 +163,11 @@ def test_infocuria_entries_are_preserved_when_languages_overlap(monkeypatch):
     data = eurlex_scraping._get_case_data_sector6("62024CJ0001", language="EN")
 
     by_lang = {entry["text_language"]: entry for entry in data["fulltexts"]}
-    # The EN + FR entries should still be from InfoCuria.
-    assert by_lang["EN"]["text_source"] == "INFOCURIA_BLOB_HTML"
-    assert "infocuria body in EN" in by_lang["EN"]["text"]
-    assert by_lang["FR"]["text_source"] == "INFOCURIA_BLOB_HTML"
-    assert "infocuria body in FR" in by_lang["FR"]["text"]
-    # DE only existed in CELLAR — comes through with CELLAR_ITEM source.
-    assert by_lang["DE"]["text_source"] == "CELLAR_ITEM"
-    assert "cellar body" in by_lang["DE"]["text"]
+    for language in ("EN", "FR", "DE"):
+        assert by_lang[language]["text_source"] == "CELLAR_ITEM"
+        assert "cellar body" in by_lang[language]["text"]
+    assert data["text_source"] == "CELLAR_ITEM"
+    assert "cellar body" in data["text"]
 
 
 def test_metadata_fields_remain_from_infocuria_even_when_cellar_supplements(monkeypatch):
@@ -250,9 +246,8 @@ def test_metadata_fields_remain_from_infocuria_even_when_cellar_supplements(monk
     assert "Environment" in data["keywords"]
 
 
-def test_supplementation_is_noop_when_cellar_has_nothing_extra(monkeypatch):
-    """If CELLAR returns no new languages, the fulltexts list is unchanged
-    and the metadata is untouched. Belt-and-braces idempotency."""
+def test_cellar_replaces_all_overlapping_languages(monkeypatch):
+    """Even exact language overlap is replaced by the canonical work."""
     eurlex_scraping._get_case_data_cached.cache_clear()
     _patch_infocuria(monkeypatch, doc_langs=["EN", "FR", "DE"])
     _patch_cellar(monkeypatch, languages=["EN", "FR", "DE"])  # exact overlap
@@ -261,9 +256,8 @@ def test_supplementation_is_noop_when_cellar_has_nothing_extra(monkeypatch):
 
     langs = sorted(entry["text_language"] for entry in data["fulltexts"])
     assert langs == ["DE", "EN", "FR"]
-    # No CELLAR_ITEM entries — every language already had an InfoCuria source.
     sources = {entry["text_source"] for entry in data["fulltexts"]}
-    assert sources == {"INFOCURIA_BLOB_HTML"}
+    assert sources == {"CELLAR_ITEM"}
 
 
 def test_supplementation_skips_when_cellar_work_uri_unresolvable(monkeypatch):

cellar_extractor-2.0.2/cellar_extractor.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "2.0.2",
-  "distance": 0,
-  "node": "gf354af5941cc140dd6de68aa3495f73831261e45",
-  "dirty": false,
-  "branch": "HEAD",
-  "node_date": "2026-08-28"
-}