echr-extractor 0.0.1.dev1__py3-none-any.whl

echr_extractor/echr.py

@@ -0,0 +1,266 @@
+import json
+import logging
+import os
+from pathlib import Path
+
+from .ECHR_html_downloader import download_full_text_main
+from .ECHR_metadata_harvester import get_echr_metadata
+from .ECHR_nodes_edges_list_transform import echr_nodes_edges
+
+"""
+Enhanced ECHR data extraction with improved batching, error handling, and memory management.
+
+Key improvements:
+- Date range batching for large datasets to prevent timeouts
+- Enhanced error handling with exponential backoff
+- Progress tracking with tqdm progress bars
+- Memory-efficient processing for large datasets
+- Configurable batch sizes and retry parameters
+- Better logging and status reporting
+
+The original functionality is preserved for backward compatibility.
+"""
+
+
+def get_echr(
+    start_id=0,
+    end_id=None,
+    start_date=None,
+    count=None,
+    end_date=None,
+    verbose=False,
+    save_file="y",
+    fields=None,
+    link=None,
+    language=None,
+    query_payload=None,
+    # New configuration parameters
+    batch_size=500,
+    timeout=60,
+    retry_attempts=3,
+    max_attempts=20,
+    days_per_batch=365,
+    progress_bar=True,
+    memory_efficient=True,
+):
+    """
+    Enhanced ECHR metadata extraction with improved reliability and performance.
+
+    This function provides a high-level interface for extracting ECHR metadata with
+    advanced features like date batching, progress tracking, and memory management.
+
+    :param int start_id: The index to start the search from (default: 0).
+    :param int end_id: The index to end search at, where None fetches all results.
+    :param str start_date: The point from which to save cases (YYYY-MM-DD format).
+    :param int count: Number of records to fetch (alternative to end_id).
+    :param str end_date: The point before which to save cases (YYYY-MM-DD format).
+    :param bool verbose: Whether or not to print extra information (default: False).
+    :param str save_file: Whether to save results to file ("y" or "n", default: "y").
+    :param list fields: List of fields to extract (default: None, uses all fields).
+    :param str link: Custom HUDOC link for advanced queries.
+    :param list language: List of language codes (default: ["ENG"]).
+    :param str query_payload: Custom query payload for advanced searches.
+    :param int batch_size: Number of records to fetch per batch, max 500 (default: 500).
+    :param float timeout: Request timeout in seconds (default: 60).
+    :param int retry_attempts: Number of retry attempts for failed requests (default: 3).
+    :param int max_attempts: Maximum total attempts before giving up (default: 20).
+    :param int days_per_batch: Number of days per date batch for large date ranges (default: 365).
+    :param bool progress_bar: Whether to show progress bar (default: True).
+    :param bool memory_efficient: Whether to use memory-efficient processing (default: True).
+
+    :return: pandas.DataFrame containing the extracted metadata, or False if extraction failed.
+
+    Example:
+        # Basic usage (backward compatible)
+        df = get_echr(start_id=0, end_id=1000, verbose=True)
+
+        # Advanced usage with date batching
+        df = get_echr(
+            start_date='2020-01-01',
+            end_date='2023-12-31',
+            batch_size=250,
+            days_per_batch=180,
+            progress_bar=True
+        )
+
+        # Memory-efficient processing for large datasets
+        df = get_echr(
+            start_id=0,
+            end_id=50000,
+            memory_efficient=True,
+            batch_size=200
+        )
+    """
+    if language is None:
+        language = ["ENG"]
+    if count:
+        end_id = int(start_id) + count
+        if verbose:
+            logging.info(f"--- STARTING ECHR DOWNLOAD FOR {count} RECORDS ---")
+    else:
+        if verbose:
+            logging.info("--- STARTING ECHR DOWNLOAD ---")
+
+    df = get_echr_metadata(
+        start_id=start_id,
+        end_id=end_id,
+        start_date=start_date,
+        end_date=end_date,
+        verbose=verbose,
+        fields=fields,
+        link=link,
+        language=language,
+        query_payload=query_payload,
+        batch_size=batch_size,
+        timeout=timeout,
+        retry_attempts=retry_attempts,
+        max_attempts=max_attempts,
+        days_per_batch=days_per_batch,
+        progress_bar=progress_bar,
+        memory_efficient=memory_efficient,
+    )
+    if df is False:
+        return False
+    if save_file == "y":
+        filename = determine_filename(start_id, end_id, start_date, end_date)
+        Path("data").mkdir(parents=True, exist_ok=True)
+        file_path = os.path.join("data", filename + ".csv")
+        df.to_csv(file_path, index=False)
+        logging.info("\n--- DONE ---")
+        return df
+    else:
+        logging.info("\n--- DONE ---")
+        return df
+
+
+def determine_filename(start_id, end_id, start_date, end_date):
+    if end_id:
+        if start_date and end_date:
+            filename = (
+                f"echr_metadata_index_{start_id}-{end_id}_dates_{start_date}-{end_date}"
+            )
+        elif start_date:
+            filename = f"echr_metadata_{start_id}-{end_id}_dates_{start_date}-END"
+        elif end_date:
+            filename = f"echr_metadata_{start_id}-{end_id}_datesSTART-{end_date}"
+        else:
+            filename = f"echr_metadata_{start_id}-{end_id}_dates_START-END"
+    else:
+        if start_date and end_date:
+            filename = (
+                f"echr_metadata_index_{start_id}-ALL_dates_{start_date}-{end_date}"
+            )
+        elif start_date:
+            filename = f"echr_metadata_{start_id}-ALL_dates_{start_date}-END"
+        elif end_date:
+            filename = f"echr_metadata_{start_id}-ALL_dates_START-{end_date}"
+        else:
+            filename = f"echr_metadata_{start_id}-ALL_dates_START-END"
+    return filename
+
+
+def get_echr_extra(
+    start_id=0,
+    end_id=None,
+    start_date=None,
+    count=None,
+    end_date=None,
+    verbose=False,
+    save_file="y",
+    threads=10,
+    fields=None,
+    link=None,
+    language=None,
+    query_payload=None,
+    # New configuration parameters
+    batch_size=500,
+    timeout=60,
+    retry_attempts=3,
+    max_attempts=20,
+    days_per_batch=365,
+    progress_bar=True,
+    memory_efficient=True,
+):
+    """
+    Enhanced ECHR metadata and full-text extraction with improved reliability and performance.
+
+    This function extracts both metadata and full-text content from ECHR cases with
+    advanced features like date batching, progress tracking, and memory management.
+
+    :param int start_id: The index to start the search from (default: 0).
+    :param int end_id: The index to end search at, where None fetches all results.
+    :param str start_date: The point from which to save cases (YYYY-MM-DD format).
+    :param int count: Number of records to fetch (alternative to end_id).
+    :param str end_date: The point before which to save cases (YYYY-MM-DD format).
+    :param bool verbose: Whether or not to print extra information (default: False).
+    :param str save_file: Whether to save results to file ("y" or "n", default: "y").
+    :param int threads: Number of threads for full-text download (default: 10).
+    :param list fields: List of fields to extract (default: None, uses all fields).
+    :param str link: Custom HUDOC link for advanced queries.
+    :param list language: List of language codes (default: ["ENG"]).
+    :param str query_payload: Custom query payload for advanced searches.
+    :param int batch_size: Number of records to fetch per batch, max 500 (default: 500).
+    :param float timeout: Request timeout in seconds (default: 60).
+    :param int retry_attempts: Number of retry attempts for failed requests (default: 3).
+    :param int max_attempts: Maximum total attempts before giving up (default: 20).
+    :param int days_per_batch: Number of days per date batch for large date ranges (default: 365).
+    :param bool progress_bar: Whether to show progress bar (default: True).
+    :param bool memory_efficient: Whether to use memory-efficient processing (default: True).
+
+    :return: tuple of (pandas.DataFrame, list) containing metadata and full-text data,
+             or (False, False) if extraction failed.
+    """
+    df = get_echr(
+        start_id=start_id,
+        end_id=end_id,
+        start_date=start_date,
+        end_date=end_date,
+        verbose=verbose,
+        count=count,
+        save_file="n",
+        fields=fields,
+        link=link,
+        language=language,
+        query_payload=query_payload,
+        batch_size=batch_size,
+        timeout=timeout,
+        retry_attempts=retry_attempts,
+        max_attempts=max_attempts,
+        days_per_batch=days_per_batch,
+        progress_bar=progress_bar,
+        memory_efficient=memory_efficient,
+    )
+    logging.info("Full-text download will now begin")
+    if df is False:
+        return False, False
+    json_list = download_full_text_main(df, threads)
+    logging.info("Full-text download finished")
+    if save_file == "y":
+        filename = determine_filename(start_id, end_id, start_date, end_date)
+        filename_json = filename.replace("metadata", "full_text")
+        Path("data").mkdir(parents=True, exist_ok=True)
+        file_path = os.path.join("data", filename + ".csv")
+        df.to_csv(file_path, index=False)
+        file_path_json = os.path.join("data", filename_json + ".json")
+        with open(file_path_json, "w") as f:
+            json.dump(json_list, f)
+        return df, json_list
+    else:
+        return df, json_list
+
+
+def get_nodes_edges(metadata_path=None, df=None, save_file="y"):
+    nodes, edges = echr_nodes_edges(metadata_path=metadata_path, data=df)
+    if save_file == "y":
+        Path("data").mkdir(parents=True, exist_ok=True)
+        edges.to_csv(
+            os.path.join("data", "ECHR_edges.csv"), index=False, encoding="utf-8"
+        )
+        nodes.to_csv(
+            os.path.join("data", "ECHR_nodes.csv"), index=False, encoding="utf-8"
+        )
+        nodes.to_json(os.path.join("data", "ECHR_nodes.json"), orient="records")
+        edges.to_json(os.path.join("data", "ECHR_edges.json"), orient="records")
+        return nodes, edges
+
+    return nodes, edges

echr_extractor/testing_file.py

@@ -0,0 +1,20 @@
+# import logging
+import sys
+from os.path import abspath
+
+import echr_extractor
+
+current_dir = abspath(__file__)
+correct_dir = "\\".join(current_dir.replace("\\", "/").split("/")[:-2])
+sys.path.append(correct_dir)
+# print(sys.path)
+
+
+if __name__ == "__main__":
+    payload = (
+        "contentsitename:ECHR AND (NOT (doctype=PR OR doctype=HFCOMOLD OR doctype=HECOMOLD)) "
+        'AND ((NOT "has been a violation of Article 6") AND ("has been no violation of Article 6")) '
+        'AND ((languageisocode="ENG")) AND ((documentcollectionid="GRANDCHAMBER") OR (documentcollectionid="CHAMBER"))'
+    )
+    df = echr_extractor.get_echr(query_payload=payload)
+    b = 2

echr_extractor-0.0.1.dev1.dist-info/METADATA

@@ -0,0 +1,189 @@
+Metadata-Version: 2.4
+Name: echr-extractor
+Version: 0.0.1.dev1
+Summary: Python library for extracting case law data from the European Court of Human Rights (ECHR) HUDOC database
+Author-email: LawTech Lab <lawtech@maastrichtuniversity.nl>
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/maastrichtlawtech/echr-extractor
+Project-URL: Repository, https://github.com/maastrichtlawtech/echr-extractor
+Project-URL: Bug Reports, https://github.com/maastrichtlawtech/echr-extractor/issues
+Project-URL: Documentation, https://github.com/maastrichtlawtech/echr-extractor
+Keywords: echr,extractor,european,convention,human,rights,court,case-law,legal,hudoc,data-extraction
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Legal Industry
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.26.0
+Requires-Dist: pandas>=1.3.0
+Requires-Dist: beautifulsoup4>=4.9.3
+Requires-Dist: dateparser>=1.0.0
+Requires-Dist: tqdm>=4.60.0
+Provides-Extra: dev
+Requires-Dist: pytest>=6.0; extra == "dev"
+Requires-Dist: pytest-cov>=2.10; extra == "dev"
+Requires-Dist: black>=21.0.0; extra == "dev"
+Requires-Dist: isort>=5.0.0; extra == "dev"
+Requires-Dist: flake8>=3.8.0; extra == "dev"
+Requires-Dist: mypy>=0.910; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: sphinx>=4.0.0; extra == "docs"
+Requires-Dist: sphinx-rtd-theme>=0.5.0; extra == "docs"
+Dynamic: license-file
+
+# ECHR Extractor
+
+Python library for extracting case law data from the European Court of Human Rights (ECHR) HUDOC database.
+
+## Features
+
+- Extract metadata for ECHR cases from the HUDOC database
+- Download full text content for cases
+- Support for custom date ranges and case ID ranges
+- Multiple language support
+- Generate nodes and edges for network analysis
+- Flexible output formats (CSV, JSON, in-memory DataFrames)
+
+## Installation
+
+```bash
+pip install echr-extractor
+```
+
+## Quick Start
+
+```python
+from echr_extractor import get_echr, get_echr_extra, get_nodes_edges
+
+# Get basic metadata for cases
+df = get_echr(start_id=0, count=100, language=['ENG'])
+
+# Get metadata + full text
+df, full_texts = get_echr_extra(start_id=0, count=100, language=['ENG'])
+
+# Generate network data
+nodes, edges = get_nodes_edges(df=df)
+```
+
+## Functions
+
+### `get_echr`
+
+Gets all available metadata for ECHR cases from the HUDOC database.
+
+**Parameters:**
+- `start_id` (int, optional): The ID of the first case to download (default: 0)
+- `end_id` (int, optional): The ID of the last case to download (default: maximum available)
+- `count` (int, optional): Number of cases per language to download (default: None)
+- `start_date` (str, optional): Start publication date (yyyy-mm-dd) (default: None)
+- `end_date` (str, optional): End publication date (yyyy-mm-dd) (default: current date)
+- `verbose` (bool, optional): Show progress information (default: False)
+- `fields` (list, optional): Limit metadata fields to download (default: all fields)
+- `save_file` (str, optional): Save as CSV file ('y') or return DataFrame ('n') (default: 'y')
+- `language` (list, optional): Languages to download (default: ['ENG'])
+- `link` (str, optional): Direct HUDOC search URL (default: None)
+- `query_payload` (str, optional): Direct API query payload (default: None)
+
+### `get_echr_extra`
+
+Gets metadata and downloads full text for each case.
+
+**Parameters:** Same as `get_echr` plus:
+- `threads` (int, optional): Number of threads for parallel download (default: 10)
+
+### `get_nodes_edges`
+
+Generates nodes and edges for network analysis from case metadata.
+
+**Parameters:**
+- `metadata_path` (str, optional): Path to metadata CSV file (default: None)
+- `df` (DataFrame, optional): Metadata DataFrame (default: None)
+- `save_file` (str, optional): Save as files ('y') or return objects ('n') (default: 'y')
+
+## Advanced Usage
+
+### Using Custom Search URLs
+
+You can use direct HUDOC search URLs:
+
+```python
+url = "https://hudoc.echr.coe.int/eng#{%22itemid%22:[%22001-57574%22]}"
+df = get_echr(link=url)
+```
+
+### Using Query Payloads
+
+For more robust searching, use simple field:value queries:
+
+```python
+payload = 'article:8'
+df = get_echr(query_payload=payload)
+```
+
+### Date Range Filtering
+
+```python
+df = get_echr(
+    start_date="2020-01-01",
+    end_date="2023-12-31",
+    language=['ENG', 'FRE']
+)
+```
+
+### Specific Fields Only
+
+```python
+fields = ['itemid', 'doctypebranch', 'title', 'kpdate']
+df = get_echr(count=100, fields=fields)
+```
+
+## Requirements
+
+- Python 3.8+
+- requests
+- pandas
+- beautifulsoup4
+- dateparser
+- tqdm
+
+## License
+
+This project is licensed under the Apache License 2.0 - see the LICENSE file for details.
+
+## Contributors
+
+- Benjamin Rodrigues de Miranda
+- Chloe Crombach
+- Piotr Lewandowski
+- Pranav Bapat
+- Shashank MC
+- Gijs van Dijck
+
+## Citation
+
+If you use this library in your research, please cite:
+
+```bibtex
+@software{echr_extractor,
+  title={ECHR Extractor: Python Library for European Court of Human Rights Data},
+  author={LawTech Lab, Maastricht University},
+  url={https://github.com/maastrichtlawtech/echr-extractor},
+  year={2024}
+}
+```
+
+## Support
+
+For bug reports and feature requests, please open an issue on GitHub.

echr_extractor-0.0.1.dev1.dist-info/RECORD

@@ -0,0 +1,15 @@
+echr_extractor/ECHR_html_downloader.py,sha256=dj2C0XZMMiV4PuFETgL6L4vY7acW0Wm6qDUScd2tybE,2689
+echr_extractor/ECHR_metadata_harvester.py,sha256=cXXejidQ0cRFQB6ryjWnzcYKqH4U9uS6K6nM7xyriuk,20106
+echr_extractor/ECHR_nodes_edges_list_transform.py,sha256=L2VjFQA1qXTqLgD9ug-9HCcqvjBlhjhJCTjOATgeTLM,9248
+echr_extractor/__init__.py,sha256=Ni6iuNF27KAfpLv6e7H9N66mL4nN0jEuW6soTtrAhkM,403
+echr_extractor/_version.py,sha256=QeQ7AWx2KoSWKCEjOGq3ydCSILGUGbDRIZahOZvyHSQ,717
+echr_extractor/clean_ref.py,sha256=GF6jx3tTALa_ztl2Y2gNRU2oztgZIq8XHnV8D6vj-d8,146
+echr_extractor/cli.py,sha256=WTjbjPanDa2zz76UmgVI4db_xMFJ4VIsiLZHXfaGkAw,3993
+echr_extractor/echr.py,sha256=j4E53I7u4Z7Cwx6HURPNxpjb24Ten3W-NFUjb5Tb35g,10136
+echr_extractor/testing_file.py,sha256=K6P8uTzeosaihXe-6bd3srxxqbD1J9UjuU-5DXbwooI,665
+echr_extractor-0.0.1.dev1.dist-info/licenses/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+echr_extractor-0.0.1.dev1.dist-info/METADATA,sha256=_JDtKtCIFEMa1oL8GxzqQN3-oZhIvLAcdXwvRU-_HAg,5842
+echr_extractor-0.0.1.dev1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+echr_extractor-0.0.1.dev1.dist-info/entry_points.txt,sha256=TmNI9MFujDX0hsVO9kCCUX4fy6HfLSHZQjAQXlJNtrw,59
+echr_extractor-0.0.1.dev1.dist-info/top_level.txt,sha256=_3q8klfvemMhS7KOxqjrEIJROrwhIM0bWKDAvldqfQY,15
+echr_extractor-0.0.1.dev1.dist-info/RECORD,,

echr_extractor-0.0.1.dev1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

echr_extractor-0.0.1.dev1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+echr-extractor = echr_extractor.cli:main