impc-api 1.0.7__tar.gz → 2.1.0__tar.gz

{impc_api-1.0.7/impc_api.egg-info → impc_api-2.1.0}/PKG-INFO

@@ -1,21 +1,19 @@
 Metadata-Version: 2.4
 Name: impc_api
-Version: 1.0.7
+Version: 2.1.0
 Summary: A package to facilitate making API requests to the IMPC Solr API
 Author: MPI2, Marina Kan, Diego Pava
-Project-URL: Homepage, https://github.com/mpi2/impc-api
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
+License-Expression: Apache-2.0
 License-File: LICENSE
 Requires-Dist: ipykernel>=6.29.5
 Requires-Dist: notebook>=7.2.2
 Requires-Dist: pandas>=2.2.0
-Requires-Dist: requests>=2.31.0
+Requires-Dist: requests>=2.33.0
 Requires-Dist: tqdm>=4.66.4
 Requires-Dist: pydantic>=2.9
-Provides-Extra: dev
-Requires-Dist: pytest>=8.2.2; extra == "dev"
-Dynamic: license-file
+Requires-Python: >=3.12
+Project-URL: Homepage, https://github.com/mpi2/impc-api
+Description-Content-Type: text/markdown
 
 # IMPC_API
 
@@ -23,7 +21,8 @@ Dynamic: license-file
 
 ## Installation Instructions
 
-0. Ensure that Python is installed on your system. The minimum required version is 3.10.
+0. Ensure that Python is installed on your system. The minimum required version is 3.12.
+   Python 3.10 will not be supported after version 2.0.0.
 
 1. **Create a virtual environment (optional but recommended)**:
 On Mac or Linux:
@@ -46,7 +45,7 @@ Create a [Jupyter Notebook](https://jupyter-notebook.readthedocs.io/en/latest/)
 The available functions can be imported as:
 
 ```python
-from impc_api import solr_request, batch_solr_request
+from impc_api import batch_solr_request, get_core_fields, solr_request
 ```
 
 # 1. Solr request
@@ -64,6 +63,31 @@ num_found, df = solr_request(
 )
 ```
 
+By default, `solr_request` uses a 10 second request timeout. To use a different timeout, pass the optional `timeout` argument:
+
+```python
+num_found, df = solr_request(
+    core='genotype-phenotype',
+    params={
+        'q': '*:*',
+        'rows': 10,
+        'fl': 'marker_symbol,allele_symbol,parameter_stable_id'
+    },
+    timeout=30
+)
+```
+
+To use another compatible Solr deployment, override the base URL. The same
+argument is available on `batch_solr_request`:
+
+```python
+num_found, df = solr_request(
+    core='genotype-phenotype',
+    params={'q': '*:*', 'rows': 10},
+    base_url='http://solr-host:8983/solr',
+)
+```
+
 ## a. Facet request
 
 `solr_request` allows facet requests
@@ -144,7 +168,7 @@ print(url)
 
 Use `batch_solr_request` for:
 
-- Large queries (>100,000 rows)
+- Large queries (>1,000,000 rows)
 - Querying multiple items in a list
 - Downloading data in `json` or `csv` format.
 
@@ -156,7 +180,7 @@ For large queries you can choose between seeing them in a DataFrame or downloadi
 
 This will fetch your data using the API responsibly and return a Pandas DataFrame
 
-When your request is larger than recommended and you have not opted for downloading the data, a warning will be presented and you should follow the instructions to proceed.
+When your request is larger than recommended and you have not opted to download the data, a warning will be presented suggesting that you retry your request and download the data.
 
 ```python
 df = batch_solr_request(
@@ -230,3 +254,20 @@ df = batch_solr_request(
 )
 print(df.head())
 ```
+# 3.Get Core Fields
+
+## Retrieve valid fields for a core
+
+Use `get_core_fields` before constructing a request to discover the documented
+fields available for a public IMPC Solr core. The function uses the same source
+of truth as request validation and does not make a network request.
+
+```python
+from impc_api import get_core_fields
+
+get_core_fields("genotype-phenotype")
+```
+
+The complete list is returned. An unknown core returns an empty list. Skills and
+other integrations should call this function instead of maintaining their own
+copies of core field names.

impc_api-1.0.7/PKG-INFO → impc_api-2.1.0/README.md

@@ -1,29 +1,11 @@
-Metadata-Version: 2.4
-Name: impc_api
-Version: 1.0.7
-Summary: A package to facilitate making API requests to the IMPC Solr API
-Author: MPI2, Marina Kan, Diego Pava
-Project-URL: Homepage, https://github.com/mpi2/impc-api
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: ipykernel>=6.29.5
-Requires-Dist: notebook>=7.2.2
-Requires-Dist: pandas>=2.2.0
-Requires-Dist: requests>=2.31.0
-Requires-Dist: tqdm>=4.66.4
-Requires-Dist: pydantic>=2.9
-Provides-Extra: dev
-Requires-Dist: pytest>=8.2.2; extra == "dev"
-Dynamic: license-file
-
 # IMPC_API
 
 `impc_api` is a Python package which provides several helper functions that wrap around the IMPC SOLR API. The functions in this package are intended for use in a Jupyter Notebook.
 
 ## Installation Instructions
 
-0. Ensure that Python is installed on your system. The minimum required version is 3.10.
+0. Ensure that Python is installed on your system. The minimum required version is 3.12.
+   Python 3.10 will not be supported after version 2.0.0.
 
 1. **Create a virtual environment (optional but recommended)**:
 On Mac or Linux:
@@ -46,7 +28,7 @@ Create a [Jupyter Notebook](https://jupyter-notebook.readthedocs.io/en/latest/)
 The available functions can be imported as:
 
 ```python
-from impc_api import solr_request, batch_solr_request
+from impc_api import batch_solr_request, get_core_fields, solr_request
 ```
 
 # 1. Solr request
@@ -64,6 +46,31 @@ num_found, df = solr_request(
 )
 ```
 
+By default, `solr_request` uses a 10 second request timeout. To use a different timeout, pass the optional `timeout` argument:
+
+```python
+num_found, df = solr_request(
+    core='genotype-phenotype',
+    params={
+        'q': '*:*',
+        'rows': 10,
+        'fl': 'marker_symbol,allele_symbol,parameter_stable_id'
+    },
+    timeout=30
+)
+```
+
+To use another compatible Solr deployment, override the base URL. The same
+argument is available on `batch_solr_request`:
+
+```python
+num_found, df = solr_request(
+    core='genotype-phenotype',
+    params={'q': '*:*', 'rows': 10},
+    base_url='http://solr-host:8983/solr',
+)
+```
+
 ## a. Facet request
 
 `solr_request` allows facet requests
@@ -144,7 +151,7 @@ print(url)
 
 Use `batch_solr_request` for:
 
-- Large queries (>100,000 rows)
+- Large queries (>1,000,000 rows)
 - Querying multiple items in a list
 - Downloading data in `json` or `csv` format.
 
@@ -156,7 +163,7 @@ For large queries you can choose between seeing them in a DataFrame or downloadi
 
 This will fetch your data using the API responsibly and return a Pandas DataFrame
 
-When your request is larger than recommended and you have not opted for downloading the data, a warning will be presented and you should follow the instructions to proceed.
+When your request is larger than recommended and you have not opted to download the data, a warning will be presented suggesting that you retry your request and download the data.
 
 ```python
 df = batch_solr_request(
@@ -230,3 +237,20 @@ df = batch_solr_request(
 )
 print(df.head())
 ```
+# 3.Get Core Fields
+
+## Retrieve valid fields for a core
+
+Use `get_core_fields` before constructing a request to discover the documented
+fields available for a public IMPC Solr core. The function uses the same source
+of truth as request validation and does not make a network request.
+
+```python
+from impc_api import get_core_fields
+
+get_core_fields("genotype-phenotype")
+```
+
+The complete list is returned. An unknown core returns an empty list. Skills and
+other integrations should call this function instead of maintaining their own
+copies of core field names.

{impc_api-1.0.7 → impc_api-2.1.0}/impc_api/__init__.py

@@ -1,6 +1,7 @@
 from .solr_request import solr_request
 from .batch_solr_request import batch_solr_request
+from .get_core_fields import get_core_fields
 from .utils import validators, warnings
 
 # Control what gets imported by client
-__all__ = ["solr_request", "batch_solr_request"]
+__all__ = ["solr_request", "batch_solr_request", "get_core_fields"]

{impc_api-1.0.7 → impc_api-2.1.0}/impc_api/batch_solr_request.py

@@ -1,6 +1,8 @@
 import json
 import warnings
+from collections.abc import Iterator
 from pathlib import Path
+from typing import Any, Dict, Iterable, List, Union
 
 import pandas as pd
 import requests
@@ -12,15 +14,24 @@ from impc_api.utils.warnings import (
     warning_config,
     RowsParamIgnored,
     UnsupportedDownloadFormatError,
+    LargeRequestMemoryWarning
 )
-from .solr_request import solr_request
+from .solr_request import DEFAULT_SOLR_BASE_URL, solr_request
 
 # Initialise warning config
 warning_config()
 
+BatchSolrChunk = Union[List[Dict[str, Any]], str]
+
+
 def batch_solr_request(
-    core, params, download=False, batch_size=5000, filename="batch_request"
-):
+    core: str,
+    params: Dict[str, Any],
+    download: bool = False,
+    batch_size: int = 5000,
+    filename: str = "batch_request",
+    base_url: str = DEFAULT_SOLR_BASE_URL,
+) -> pd.DataFrame:
     """Function for large API requests (>1,000,000 results). Fetches the data in batches and
     produces a Pandas DataFrame or downloads a file in json or csv formats.
 
@@ -32,11 +43,12 @@ def batch_solr_request(
         download (bool, optional): True for download a local file, False to display results as a DataFrame. Defaults to False.
         batch_size (int, optional): Size of batches to fetch the data. Defaults to 5000.
         filename (str, optional): When download=True, select the name of the file. Defaults to 'batch_request'.
+        base_url (str, optional): default public IMPC Solr endpoint.
+            Override this to use another compatible Solr deployment.
 
 
     Returns:
         pd.DataFrame: if download=False, displays a DataFrame with the results.
-        None: if download=True, displays a statement on the console and returns None.
     """
 
     # If params["rows"] is passed, the user is warned about no effect
@@ -72,7 +84,10 @@ def batch_solr_request(
 
     # Determine the total number of rows. Note that we do not request any data (rows = 0).
     num_results, _ = solr_request(
-        core=core, params={**params, "start": 0, "rows": 0, "wt": "json"}, silent=True
+        core=core,
+        params={**params, "start": 0, "rows": 0, "wt": "json"},
+        silent=True,
+        base_url=base_url,
     )
     print(f"Number of found documents: {num_results}")
 
@@ -86,44 +101,37 @@ def batch_solr_request(
             # Implement loop behaviour
             print("Downloading file...")
             filename_path = Path(f"{filename}.{params['wt']}")
-            gen = _batch_solr_generator(core, params, num_results)
+            gen = _batch_solr_generator(core, params, num_results, base_url=base_url)
             _solr_downloader(params, filename_path, gen)
             print(f"File saved as: {filename_path}")
         except UnsupportedDownloadFormatError as e:
             raise e
         except Exception as e:
-            raise (f"An error ocurred while downloading the data:{e}")
+            raise Exception(f"An error occurred while downloading the data:{e}")
 
             # Try to read the downloaded file
         try:
             print("Reading downloaded file...")
             return _read_downloaded_file(filename_path, params["wt"])
         except Exception as e:
-            raise Exception(f"An unexpected error occured:{e}")
-
-    # If the number of results is small enough and download is off, it's okay to show as df
-    if num_results < 1000000 and not download:
-        return _batch_to_df(core, params, num_results)
+            raise Exception(f"An unexpected error occurred:{e}")
 
-    # If it's too big, warn the user and ask if they want to proceed.
-    else:
-        print(
-            "Your request might exceed the available memory. We suggest setting 'download=True' and reading the file in batches"
-        )
-        prompt = input(
-            "Do you wish to proceed anyway? press ('y' or enter to proceed) / type('n' or 'exit' to cancel)"
+    
+    # If a request is too large, warn the user to use download=true.
+    if num_results > 1000000 and not download:
+        warnings.warn(
+            message="This request may exceed available memory. If the download fails, set 'download=True' and try again.",
+            category=LargeRequestMemoryWarning,
         )
-        match prompt:
-            case "n" | "exit":
-                print("Exiting gracefully")
-                exit()
-            case "y" | "":
-                print("Fetching data...")
-                return _batch_to_df(core, params, num_results)
-
+    return _batch_to_df(core, params, num_results, base_url=base_url)
 
 # Helper batch_to_df
-def _batch_to_df(core, params, num_results):
+def _batch_to_df(
+    core: str,
+    params: Dict[str, Any],
+    num_results: int,
+    base_url: str = DEFAULT_SOLR_BASE_URL,
+) -> pd.DataFrame:
     """Helper function to fetch data in batches and display them in a DataFrame
 
     Args:
@@ -148,6 +156,7 @@ def _batch_to_df(core, params, num_results):
                 core=core,
                 params={**params, "start": start, "rows": batch_size},
                 silent=True,
+                base_url=base_url,
             )
 
             # Update progress bar with the number of rows requested.
@@ -161,7 +170,12 @@ def _batch_to_df(core, params, num_results):
         return pd.concat(chunks, ignore_index=True)
 
 
-def _batch_solr_generator(core, params, num_results):
+def _batch_solr_generator(
+    core: str,
+    params: Dict[str, Any],
+    num_results: int,
+    base_url: str = DEFAULT_SOLR_BASE_URL,
+) -> Iterator[BatchSolrChunk]:
     """Generator function to fetch results from the SOLR API in batches using pagination.
 
     Args:
@@ -175,8 +189,7 @@ def _batch_solr_generator(core, params, num_results):
     Yields:
         ([dict, str]): A JSON object or plain text with the results.
     """
-    base_url = "https://www.ebi.ac.uk/mi/impc/solr/"
-    solr_url = base_url + core + "/select"
+    solr_url = f"{base_url.rstrip('/')}/{core}/select"
     start = params["start"]
     batch_size = params["rows"]
 
@@ -205,7 +218,11 @@ def _batch_solr_generator(core, params, num_results):
 
 
 # File writer
-def _solr_downloader(params, filename, solr_generator):
+def _solr_downloader(
+    params: Dict[str, Any],
+    filename: Path,
+    solr_generator: Iterable[BatchSolrChunk],
+) -> None:
     """Function to write the data from the generator into the specified format.
     Supports json and csv only.
 
@@ -230,6 +247,7 @@ def _solr_downloader(params, filename, solr_generator):
         elif params.get("wt") == "csv":
             first_chunk = True
             for chunk in solr_generator:
+                assert isinstance(chunk, str)
                 lines = chunk.splitlines()
                 if first_chunk:
                     # Write all lines in the first chunk
@@ -241,7 +259,7 @@ def _solr_downloader(params, filename, solr_generator):
 
 
 # File reader
-def _read_downloaded_file(filename: Path, request_format):
+def _read_downloaded_file(filename: Path, request_format: str) -> pd.DataFrame:
     """Wrapper for reading files into Pandas DataFrames
 
     Args:

impc_api-2.1.0/impc_api/get_core_fields.py

@@ -0,0 +1,22 @@
+from .utils.validators import ValidationJson
+
+
+def get_core_fields(core: str) -> list[str]:
+    """Return the documented fields available for an IMPC Solr core.
+
+    Fields are read from the same source of truth used to validate requests.
+    This function does not make a request to the IMPC Solr API.
+
+    Args:
+        core: Name of a public IMPC Solr core.
+
+    Returns:
+        The documented field names, or an empty list when ``core`` is unknown.
+
+    Example:
+        >>> from impc_api import get_core_fields
+        >>> fields = get_core_fields("genotype-phenotype")
+        >>> fields[:3]
+        ['doc_id', 'ontology_db_id', 'assertion_type']
+    """
+    return ValidationJson().valid_fields(core)

{impc_api-1.0.7 → impc_api-2.1.0}/impc_api/solr_request.py

@@ -1,3 +1,5 @@
+from typing import Any, Dict, Optional, Tuple, Union
+
 import pandas as pd
 import requests
 from IPython.display import display
@@ -5,12 +7,28 @@ from tqdm import tqdm
 
 from impc_api.utils.validators import CoreParamsValidator
 
+RequestTimeout = Union[float, Tuple[float, float]]
+SolrRequestResult = Optional[
+    Union[Tuple[int, pd.DataFrame], Tuple[Optional[str], None]]
+]
+
+DEFAULT_REQUEST_TIMEOUT: RequestTimeout = 10.0
+DEFAULT_SOLR_BASE_URL = "https://www.ebi.ac.uk/mi/impc/solr/"
+
 # Display the whole dataframe <15
 pd.set_option("display.max_rows", 15)
 pd.set_option("display.max_columns", None)
 
 # Create helper function
-def solr_request(core, params, silent=False, validate=False, url_only=False):
+def solr_request(
+    core: str,
+    params: Dict[str, Any],
+    silent: bool = False,
+    validate: bool = False,
+    url_only: bool = False,
+    timeout: RequestTimeout = DEFAULT_REQUEST_TIMEOUT,
+    base_url: str = DEFAULT_SOLR_BASE_URL,
+) -> SolrRequestResult:
     """Performs a single Solr request to the IMPC Solr API.
 
     Args:
@@ -24,6 +42,10 @@ def solr_request(core, params, silent=False, validate=False, url_only=False):
             if any parameter seems invalid.
         url_only (bool, optional): default False
             If true, returns the request URL but no data. 
+        timeout (int or float, optional): default 10
+            Number of seconds to wait for the API request before timing out.
+        base_url (str, optional): default public IMPC Solr endpoint.
+            Override this to use another compatible Solr deployment.
 
 
     Returns:
@@ -77,10 +99,9 @@ def solr_request(core, params, silent=False, validate=False, url_only=False):
     if validate:
         CoreParamsValidator(core=core, params=params)
 
-    base_url = "https://www.ebi.ac.uk/mi/impc/solr/"
-    solr_url = base_url + core + "/select"
+    solr_url = f"{base_url.rstrip('/')}/{core}/select"
 
-    response = requests.get(solr_url, params=params)
+    response = requests.get(solr_url, params=params, timeout=timeout)
 
     if url_only:
         if response.status_code == 200:
@@ -124,7 +145,7 @@ def solr_request(core, params, silent=False, validate=False, url_only=False):
         print("Error:", response.status_code, response.text)
 
 
-def _process_faceting(data, params):
+def _process_faceting(data: Dict[str, Any], params: Dict[str, Any]) -> pd.DataFrame:
     """Processes the faceting data from an API response.
     Note: This function should not be used alone but only as a helper function for solr_request().
 
@@ -157,7 +178,12 @@ def _process_faceting(data, params):
 
 
 # Batch request based on solr_request.
-def batch_request(core, params, batch_size):
+def batch_request(
+    core: str,
+    params: Dict[str, Any],
+    batch_size: int,
+    base_url: str = DEFAULT_SOLR_BASE_URL,
+) -> pd.DataFrame:
     """Calls `solr_request` multiple times with `params`
      to retrieve results in chunk `batch_size` rows at a time.
 
@@ -166,7 +192,8 @@ def batch_request(core, params, batch_size):
     Args:
          core (str): name of IMPC solr core.
          params (dict): dictionary containing the API call parameters.
-         batch_size (int): Size of batches (number of docs) per request.
+        batch_size (int): Size of batches (number of docs) per request.
+        base_url (str, optional): default public IMPC Solr endpoint.
 
      Returns:
          pandas.DataFrame: Pandas.DataFrame object with the information requested.
@@ -188,7 +215,10 @@ def batch_request(core, params, batch_size):
         )
     # Determine the total number of rows. Note that we do not request any data (rows = 0).
     num_results, _ = solr_request(
-        core=core, params={**params, "start": 0, "rows": 0}, silent=True
+        core=core,
+        params={**params, "start": 0, "rows": 0},
+        silent=True,
+        base_url=base_url,
     )
     # Initialise everything for data retrieval.
     start = 0
@@ -203,6 +233,7 @@ def batch_request(core, params, batch_size):
                 core=core,
                 params={**params, "start": start, "rows": batch_size},
                 silent=True,
+                base_url=base_url,
             )
             # Record chunk.
             chunks.append(df_chunk)

{impc_api-1.0.7 → impc_api-2.1.0}/impc_api/utils/validators.py

@@ -19,7 +19,8 @@ import json
 import warnings
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import List, Dict
+from typing import Any, Collection, Dict, List
+
 from pydantic import BaseModel, model_validator, field_validator
 from impc_api.utils.warnings import (
     warning_config,
@@ -38,14 +39,14 @@ class ValidationJson:
     _validation_json: Dict[str, List[str]] = field(default_factory=dict, init=False)
 
     # Eager initialisation
-    def __post_init__(self):
+    def __post_init__(self) -> None:
         self._validation_json = self.load_core_fields(self.CORE_FILE)
 
     def load_core_fields(self, filename: Path) -> Dict[str, List[str]]:
         with open(filename, "r") as f:
             return json.load(f)
 
-    def valid_cores(self):
+    def valid_cores(self) -> Collection[str]:
         return self._validation_json.keys()
 
     def valid_fields(self, core: str) -> List[str]:
@@ -59,11 +60,13 @@ def get_fields(fields: str) -> List[str]:
 
 class CoreParamsValidator(BaseModel):
     core: str
-    params: Dict
+    params: Dict[str, Any]
 
     @model_validator(mode="before")
     @classmethod
-    def validate_core_and_fields(cls, values):
+    def validate_core_and_fields(
+        cls: type["CoreParamsValidator"], values: Dict[str, Any]
+    ) -> Dict[str, Any]:
         invalid_core: bool = False
         core = values.get("core")
         params = values.get("params")
@@ -108,7 +111,7 @@ class DownloadFormatValidator(BaseModel):
     wt: str
 
     @field_validator("wt")
-    def validate_wt(cls, value):
+    def validate_wt(cls: type["DownloadFormatValidator"], value: str) -> str:
         supported_formats = {"json", "csv"}
         if value not in supported_formats:
             raise UnsupportedDownloadFormatError(

{impc_api-1.0.7 → impc_api-2.1.0}/impc_api/utils/warnings.py

@@ -1,6 +1,7 @@
 """Module for warnings and excepton utils"""
 
 import warnings
+from typing import Optional, Type, Union
 
 
 # Custom warnings
@@ -16,16 +17,26 @@ class RowsParamIgnored(Warning):
     """Warning raised when the row param is ignored"""
 
 
+class LargeRequestMemoryWarning(Warning):
+    """Warning raised when user triggers a large request without downloading in batches"""
+
+
 # custom exceptions
 class UnsupportedDownloadFormatError(Exception):
     """Exception raised when the format is not supported for download"""
 
 
 # Custom warning function
-def warning_config():
+def warning_config() -> None:
     """Customises formatting and filters for warnings"""
 
-    def custom_warning(message, category, filename, lineno, line=None):
+    def custom_warning(
+        message: Union[str, Warning],
+        category: Type[Warning],
+        filename: str,
+        lineno: int,
+        line: Optional[str] = None,
+    ) -> str:
         return f"{category.__name__}: {message}\n"
 
     warnings.formatwarning = custom_warning

{impc_api-1.0.7 → impc_api-2.1.0}/pyproject.toml

@@ -1,11 +1,13 @@
 [build-system]
-requires = ["setuptools"]
-build-backend = "setuptools.build_meta"
+requires = ["uv_build>=0.11.7,<0.12.0"]
+build-backend = "uv_build"
 
 [project]
 name = "impc_api"
-version = "1.0.7"
+version = "2.1.0"
 description = "A package to facilitate making API requests to the IMPC Solr API"
+license = "Apache-2.0"
+license-files = ["LICENSE"]
 authors = [
     { name = "MPI2" },
     { name = "Marina Kan" },
@@ -15,22 +17,21 @@ dependencies = [
     "ipykernel>=6.29.5",
     "notebook>=7.2.2",
     "pandas>=2.2.0",
-    "requests>=2.31.0",
+    "requests>=2.33.0",
     "tqdm>=4.66.4",
     "pydantic>=2.9"
 ]
 
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.12"
 
-[project.optional-dependencies]
+[dependency-groups]
 dev = [
     "pytest>=8.2.2"
 ]
 
-[tool.setuptools.packages.find]
-include = ["impc_api", "impc_api.*"]
+[tool.uv.build-backend]
+module-root = ""
 
 [project.urls]
 "Homepage" = "https://github.com/mpi2/impc-api"
-