echr-extractor 0.0.1.dev1__py3-none-any.whl

echr_extractor/ECHR_nodes_edges_list_transform.py

@@ -0,0 +1,308 @@
+import logging
+import re
+
+import dateparser
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+from .clean_ref import clean_pattern
+
+
+def open_metadata(PATH_metadata):
+    """
+    Finds the ECHR metadata file and loads it into a dataframe
+
+    param filename_metadata: string with path to metadata
+    """
+    try:
+        df = pd.read_csv(PATH_metadata)  # change hard coded path
+        return df
+    except FileNotFoundError:
+        logging.warning("File not found. Please check the path to the metadata file.")
+        return False
+
+
+def concat_metadata(df):
+    agg_func = {
+        "itemid": "first",
+        "appno": "first",
+        "article": "first",
+        "conclusion": "first",
+        "docname": "first",
+        "doctype": "first",
+        "doctypebranch": "first",
+        "ecli": "first",
+        "importance": "first",
+        "judgementdate": "first",
+        "languageisocode": ", ".join,
+        "originatingbody": "first",
+        "violation": "first",
+        "nonviolation": "first",
+        "extractedappno": "first",
+        "scl": "first",
+    }
+    new_df = df.groupby("ecli").agg(agg_func)
+    # print(new_df)
+    return new_df
+
+
+def get_language_from_metadata(df):
+    df = concat_metadata(df)
+    df.to_json("langisocode-nodes.json", orient="records")
+
+
+def retrieve_edges_list(df):
+    """
+    Returns a dataframe consisting of 2 columns 'source' and 'target' which
+    indicate a reference link between cases.
+
+    params:
+    df -- the node list extracted from the metadata
+    df -- the complete dataframe from the metadata
+    """
+    edges = list()
+
+    count = 0
+    missing_cases = []
+    bar = tqdm(total=len(df.index), colour="GREEN", position=0, leave=True)
+    for index, item in df.iterrows():
+        bar.update(1)
+        eclis = []
+        extracted_appnos = []
+        if item.extractedappno is not np.nan:
+            extracted_appnos = item.extractedappno.split(";")
+        if item.scl is np.nan:
+            continue
+        """
+            Split the references from the scl column
+            into a list of references.
+
+            Example:
+            references in string: "Ali v. Switzerland, 5 August 1998, § 32,
+            Reports of Judgments and
+            Decisions 1998-V;Sevgi Erdogan v. Turkey (striking out),
+            no. 28492/95, 29 April 2003"
+
+            ["Ali v. Switzerland, 5 August 1998, § 32, Reports of Judgments and
+            Decisions 1998-V", "Sevgi Erdogan v. Turkey (striking out), no.
+            28492/95, 29 April 2003"]
+            """
+        ref_list = item.scl.split(";")
+        new_ref_list = [i.replace("\n", "") for i in ref_list]
+
+        for ref in new_ref_list:
+            app_number = re.findall(r"\d{3,5}/\d{2}", ref)
+
+            app_number.extend(extracted_appnos)
+
+            app_number = set(app_number)
+
+            if len(app_number) > 0:
+                # get dataframe with all possible cases by application number
+                app_number = [";".join(app_number)]
+                case = lookup_app_number(app_number, df)
+                if len(case) == 0:  # if failed try name?
+                    case = lookup_casename(ref, df)
+            else:  # if no application number in reference
+                # get dataframe with all possible cases by casename
+                case = lookup_casename(ref, df)
+
+            components = ref.split(",")
+            # get the year of case
+            year_from_ref = get_year_from_ref(components)
+
+            # remove cases in different language than reference
+            case = remove_cases_based_on_language(case, components)
+            case = remove_cases_based_on_year(case, year_from_ref)
+
+            if len(case) > 0:
+                for _, row in case.iterrows():
+                    eclis.append(row.ecli)
+            else:
+                count = count + 1
+                missing_cases.append(ref)
+
+        eclis = set(eclis)
+        eclis = [i for i in eclis if (i and i != item.ecli)]
+        # add ecli to edges list
+        if len(eclis) == 0:  # This should not have to happen at every iteration,
+            # concat might be slow
+            continue
+        for target in eclis:
+            edges.append({"source": item.ecli, "target": target})
+
+    edges = pd.DataFrame.from_records(edges)
+    return edges
+
+
+def remove_cases_based_on_year(case, year_from_ref):
+    for id, i in case.iterrows():
+        if i.judgementdate is np.nan:
+            continue
+        try:
+            date = dateparser.parse(i.judgementdate)
+        except Exception:
+            date = False
+        if date:
+            year_from_case = date.year
+            if year_from_case - year_from_ref == 0:
+                case = case[
+                    case["judgementdate"].str.contains(
+                        str(year_from_ref), regex=False, flags=re.IGNORECASE, na=False
+                    )
+                ]
+    return case
+
+
+def remove_cases_based_on_language(cases, components):
+    for id, it in cases.iterrows():
+        if "v." in components[0]:
+            lang = "ENG"
+        else:
+            lang = "FRE"
+
+        if lang not in it.languageisocode:
+            cases = cases[
+                cases["languageisocode"].str.contains(
+                    lang, regex=False, flags=re.IGNORECASE, na=False
+                )
+            ]
+    return cases
+
+
+def lookup_app_number(pattern, df):
+    """
+    Returns a list with rows containing the cases linked
+    to the found app numbers.
+    """
+    row = df.loc[df["appno"].isin(pattern)]
+
+    if row.empty:
+        return pd.DataFrame()
+    elif row.shape[0] > 1:
+        return row
+    else:
+        return row
+
+
+def lookup_casename(ref, df):
+    """
+    Process the reference for lookup in metadata.
+    Returns the rows corresponding to the cases.
+
+    - Example of the processing (2 variants) -
+
+    Original reference from scl:
+    - Hentrich v. France, 22 September 1994, § 42, Series A no. 296-A
+    - Eur. Court H.R. James and Others judgment of 21 February 1986,
+    Series A no. 98, p. 46, para. 81
+
+    Split on ',' and take first item:
+    Hentrich v. France
+    Eur. Court H.R. James and Others judgment of 21 February 1986
+
+    If certain pattern from CLEAN_REF in case name, then remove:
+    Eur. Court H.R. James and Others judgment of 21 February 1986 -->
+        James and Others
+
+    Change name to upper case and add additional text to match metadata:
+    Hentrich v. France --> CASE OF HENTRICH V. FRANCE
+    James and Others --> CASE OF JAMES AND OTHERS
+    """
+    name = get_casename(ref)
+
+    # DEV note: In case, add more patterns to clean_ref.py in future
+    patterns = clean_pattern
+
+    uptext = name.upper()
+
+    if "NO." in uptext:
+        uptext = uptext.replace("NO.", "No.")
+
+    if "BV" in uptext:
+        uptext = uptext.replace("BV", "B.V.")
+
+    if "V." in name:
+        uptext = uptext.replace("V.", "v.")
+        lang = "ENG"
+    else:
+        uptext = uptext.replace("C.", "c.")
+        lang = "FRE"
+
+    for pattern in patterns:
+        uptext = re.sub(pattern, "", uptext)
+
+    uptext = re.sub(r"\[.*", "", uptext)
+    uptext = uptext.strip()
+    row = df[df["docname"].str.contains(uptext, regex=False, flags=re.IGNORECASE)]
+
+    # if len(row) == 0:
+    #     print("no cases matched: ", name)
+
+    return row
+
+
+def get_casename(ref):
+    count = 0
+    if "v." in ref:
+        slice_at_versus = ref.split("v.")  # skip if typo (count how many)
+    elif "c." in ref:
+        slice_at_versus = ref.split("c.")
+    else:
+        count = count + 1
+        name = ref.split(",")
+        return name[0]
+
+    num_commas = slice_at_versus[0].count(",")
+
+    if num_commas > 0:
+        num_commas = num_commas + 1
+        name = ",".join(ref.split(",", num_commas)[:num_commas])
+    else:
+        name = ref.split(",")
+        return name[0]
+    return name
+
+
+def get_year_from_ref(ref):
+    for component in ref:
+        if "§" in component:
+            continue
+        component = re.sub("judgment of ", "", component)
+        if dateparser.parse(component) is not None:
+            date = dateparser.parse(component)
+        elif "ECHR" in component or "CEDH" in component:
+            if "ECHR" in component or "CEDH" in component:
+                date = re.sub("ECHR ", "", component)
+                date = re.sub("CEDH ", "", date)
+                date = date.strip()
+                date = re.sub("-.*", "", date)
+                date = re.sub(r"\s.*", "", date)
+                date = dateparser.parse(date)
+
+    try:
+        return date.year
+    except AttributeError:
+        return 0
+
+
+def echr_nodes_edges(metadata_path=None, data=None):
+    """
+    Create nodes and edges list for the ECHR data.
+    """
+    logging.info("\n--- COLLECTING METADATA ---\n")
+    if metadata_path:
+        data = open_metadata(metadata_path)
+    elif data is None:
+        logging.warning("No dataframe data provided. Returning...")
+        return "", ""
+    logging.info("\n--- EXTRACTING NODES LIST ---\n")
+    # get_language_from_metadata(nodes)
+
+    logging.info("\n--- EXTRACTING EDGES LIST ---\n")
+    edges = retrieve_edges_list(data)
+
+    # nodes.to_json(JSON_ECHR_NODES, orient="records")
+    # edges.to_json(JSON_ECHR_EDGES, orient="records")
+    return data, edges

echr_extractor/__init__.py

@@ -0,0 +1,18 @@
+"""ECHR Extractor - Python library for extracting ECHR case data."""
+
+import logging
+
+from .echr import get_echr, get_echr_extra, get_nodes_edges
+
+__version__ = "1.0.44"
+__author__ = "LawTech Lab, Maastricht University"
+__email__ = "lawtech@maastrichtuniversity.nl"
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+
+__all__ = [
+    "get_echr",
+    "get_echr_extra",
+    "get_nodes_edges",
+]

echr_extractor/_version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '0.0.1.dev1'
+__version_tuple__ = version_tuple = (0, 0, 1, 'dev1')
+
+__commit_id__ = commit_id = None

echr_extractor/clean_ref.py

@@ -0,0 +1,5 @@
+"""
+This module contains the list of patterns for reference lookup in metadata
+"""
+
+clean_pattern = ["EUR. COURT H.R.", "JUDGMENT OF.*", " DU.*"]

echr_extractor/cli.py

@@ -0,0 +1,125 @@
+"""Command-line interface for ECHR Extractor."""
+
+import argparse
+import sys
+
+from . import get_echr, get_echr_extra, get_nodes_edges
+
+
+def main() -> None:
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description="Extract case law data from ECHR HUDOC database"
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # Basic extraction command
+    extract_parser = subparsers.add_parser("extract", help="Extract ECHR metadata")
+    add_common_args(extract_parser)
+
+    # Full extraction command
+    extract_full_parser = subparsers.add_parser(
+        "extract-full", help="Extract ECHR metadata and full text"
+    )
+    add_common_args(extract_full_parser)
+    extract_full_parser.add_argument(
+        "--threads",
+        type=int,
+        default=10,
+        help="Number of threads for parallel download (default: 10)",
+    )
+
+    # Network analysis command
+    network_parser = subparsers.add_parser(
+        "network", help="Generate nodes and edges for network analysis"
+    )
+    network_parser.add_argument(
+        "--metadata-path", type=str, help="Path to metadata CSV file"
+    )
+    network_parser.add_argument(
+        "--no-save", action="store_true", help="Don't save files, return objects only"
+    )
+
+    args = parser.parse_args()
+
+    if args.command is None:
+        parser.print_help()
+        sys.exit(1)
+
+    try:
+        if args.command == "extract":
+            result = get_echr(
+                start_id=args.start_id,
+                end_id=args.end_id,
+                count=args.count,
+                start_date=args.start_date,
+                end_date=args.end_date,
+                verbose=args.verbose,
+                save_file="n" if args.no_save else "y",
+                fields=args.fields,
+                language=args.language,
+            )
+            print(f"Extracted {len(result)} cases")
+
+        elif args.command == "extract-full":
+            df, texts = get_echr_extra(
+                start_id=args.start_id,
+                end_id=args.end_id,
+                count=args.count,
+                start_date=args.start_date,
+                end_date=args.end_date,
+                verbose=args.verbose,
+                save_file="n" if args.no_save else "y",
+                threads=args.threads,
+                fields=args.fields,
+                language=args.language,
+            )
+            print(f"Extracted {len(df)} cases with full text")
+
+        elif args.command == "network":
+            nodes, edges = get_nodes_edges(
+                metadata_path=args.metadata_path, save_file="n" if args.no_save else "y"
+            )
+            print(f"Generated {len(nodes)} nodes and {len(edges)} edges")
+
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def add_common_args(parser: argparse.ArgumentParser) -> None:
+    """Add common arguments to a parser."""
+    parser.add_argument(
+        "--start-id",
+        type=int,
+        default=0,
+        help="ID of first case to download (default: 0)",
+    )
+    parser.add_argument("--end-id", type=int, help="ID of last case to download")
+    parser.add_argument(
+        "--count", type=int, help="Number of cases per language to download"
+    )
+    parser.add_argument(
+        "--start-date", type=str, help="Start publication date (yyyy-mm-dd)"
+    )
+    parser.add_argument(
+        "--end-date", type=str, help="End publication date (yyyy-mm-dd)"
+    )
+    parser.add_argument(
+        "--verbose", action="store_true", help="Show progress information"
+    )
+    parser.add_argument(
+        "--no-save", action="store_true", help="Don't save files, return objects only"
+    )
+    parser.add_argument("--fields", nargs="+", help="Limit metadata fields to download")
+    parser.add_argument(
+        "--language",
+        nargs="+",
+        default=["ENG"],
+        help="Languages to download (default: ENG)",
+    )
+
+
+if __name__ == "__main__":
+    main()