onspy 0.1.0__py3-none-any.whl

onspy/__init__.py

@@ -0,0 +1,56 @@
+"""
+onspy: Python client for the Office of National Statistics (ONS) API
+
+This package provides client functions for accessing the Office of National Statistics API
+at https://api.beta.ons.gov.uk/v1.
+"""
+
+import logging
+import os
+
+# Configure logging
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+# Add console handler if not already added
+if not logger.handlers:
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.INFO)
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    )
+    console_handler.setFormatter(formatter)
+    logger.addHandler(console_handler)
+
+# Enable debug logging if ONS_DEBUG is set
+if os.environ.get("ONS_DEBUG", "").lower() in ("1", "true", "yes"):
+    logger.setLevel(logging.DEBUG)
+    for handler in logger.handlers:
+        handler.setLevel(logging.DEBUG)
+    logger.debug("Debug logging enabled for onspy")
+
+from .datasets import (
+    ons_datasets,
+    ons_ids,
+    ons_desc,
+    ons_editions,
+    ons_latest_href,
+    ons_latest_version,
+    ons_latest_edition,
+    ons_find_latest_version_across_editions,
+)
+from .get import ons_get, ons_get_obs, ons_dim, ons_dim_opts, ons_meta
+from .code_lists import (
+    ons_codelists,
+    ons_codelist,
+    ons_codelist_editions,
+    ons_codelist_edition,
+    ons_codes,
+    ons_code,
+    ons_code_dataset,
+)
+from .search import ons_search
+from .browse import ons_browse, ons_browse_qmi
+
+__version__ = "0.1.0"
+__author__ = "Joe Wait"

onspy/browse.py

@@ -0,0 +1,85 @@
+"""
+Browser functionality for ONS.
+
+This module provides functions to quickly open ONS webpages in a browser.
+"""
+
+import webbrowser
+from typing import Optional
+
+from .datasets import ons_datasets, assert_valid_id, id_number
+
+
+def ons_browse() -> str:
+    """Quickly browse to ONS' developer webpage.
+
+    This function opens the ONS developer webpage in a browser.
+
+    Returns:
+        The URL of the webpage
+
+    Examples:
+        >>> import onspy
+        >>> onspy.ons_browse()
+    """
+    url = "https://developer.ons.gov.uk/"
+    _open_url(url)
+    return url
+
+
+def ons_browse_qmi(id: str = None) -> Optional[str]:
+    """Quickly browse to dataset's Quality and Methodology Information (QMI).
+
+    This function opens the QMI webpage for a dataset in a browser.
+
+    Args:
+        id: Dataset ID
+
+    Returns:
+        The URL of the webpage, or None if the dataset is not found
+
+    Examples:
+        >>> import onspy
+        >>> onspy.ons_browse_qmi("cpih01")
+    """
+    datasets = ons_datasets()
+    if datasets is None:
+        return None
+
+    if not assert_valid_id(id, datasets):
+        return None
+
+    idx = id_number(id, datasets)
+
+    # Handle nested dictionary
+    try:
+        if hasattr(datasets.iloc[idx], "qmi") and hasattr(
+            datasets.iloc[idx].qmi, "href"
+        ):
+            url = datasets.iloc[idx].qmi.href
+        elif isinstance(datasets.iloc[idx].get("qmi", {}), dict):
+            url = datasets.iloc[idx]["qmi"].get("href", None)
+        else:
+            return None
+
+        if url:
+            _open_url(url)
+            return url
+        return None
+    except (AttributeError, KeyError, IndexError):
+        return None
+
+
+def _open_url(url: str, open_browser: bool = True) -> str:
+    """Open a URL in the default browser.
+
+    Args:
+        url: The URL to open
+        open_browser: Whether to actually open the browser (default: True)
+
+    Returns:
+        The URL
+    """
+    if open_browser:
+        webbrowser.open(url)
+    return url

onspy/client.py

@@ -0,0 +1,209 @@
+"""
+ONS API Client module.
+
+This module provides a centralized client class for making requests to the ONS API.
+"""
+
+import logging
+import requests
+import json
+from typing import Optional, Dict, Any
+
+logger = logging.getLogger(__name__)
+
+
+class ONSClient:
+    """Client for interacting with the Office of National Statistics API."""
+
+    # API Constants
+    EMPTY = ""
+    ENDPOINT = "https://api.beta.ons.gov.uk/v1"
+    # Identify your bot
+    USER_AGENT = (
+        "onspy/0.1.0 (MyOrganisation contact@myorg.com +http://www.myorg.com/bot.html)"
+    )
+
+    def __init__(self, endpoint: Optional[str] = None):
+        """Initialize the ONS client.
+
+        Args:
+            endpoint: Custom API endpoint URL (optional)
+        """
+        self.endpoint = endpoint or self.ENDPOINT
+        self._session = requests.Session()
+        self._session.headers.update(self._get_browser_headers())
+
+    def _get_browser_headers(self) -> Dict[str, str]:
+        """Get browser-like headers to help with HTTP requests.
+
+        Returns:
+            Dictionary of headers that mimic a browser
+        """
+        return {
+            "User-Agent": self.USER_AGENT,
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+            "Pragma": "no-cache",
+            "Cache-Control": "no-cache",
+        }
+
+    def has_internet(self) -> bool:
+        """Check if internet connection is available.
+
+        Returns:
+            bool: True if internet connection is available, False otherwise
+        """
+        try:
+            # Try to connect to a widely available service
+            self._session.get("https://www.google.com", timeout=5)
+            return True
+        except requests.ConnectionError:
+            return False
+
+    def build_url(self, path_segments: Dict[str, Optional[str]]) -> str:
+        """Build a request URL from path segments.
+
+        Args:
+            path_segments: Dictionary mapping path segment names to values
+
+        Returns:
+            Full request URL
+        """
+        # Build path from segments
+        path_parts = []
+        for key, value in path_segments.items():
+            if value is None:
+                continue
+            elif value == self.EMPTY:
+                path_parts.append(key)
+            else:
+                path_parts.append(f"{key}/{value}")
+
+        path = "/".join(path_parts)
+        url = f"{self.endpoint}/{path}"
+
+        logger.debug(f"Built URL: {url}")
+        return url
+
+    def make_request(
+        self,
+        url: str,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        **kwargs,
+    ) -> Optional[requests.Response]:
+        """Make HTTP request to the ONS API.
+
+        Args:
+            url: Request URL
+            limit: Number of records to return (optional)
+            offset: Position in the dataset to start from (optional)
+            **kwargs: Additional arguments to pass to requests.get
+
+        Returns:
+            Response object if successful, None otherwise
+        """
+        logger.debug(f"Making request to URL: {url}")
+        if limit is not None:
+            logger.debug(f"With limit: {limit}")
+        if offset is not None:
+            logger.debug(f"With offset: {offset}")
+
+        if not self.has_internet():
+            logger.error("Unable to connect: No internet connection available")
+            print(
+                "Unable to connect: Please ensure you have an active internet connection or access through a secure connection."
+            )
+            return None
+
+        try:
+            # Prepare parameters
+            params = {}
+            if limit is not None:
+                params["limit"] = limit
+            if offset is not None:
+                params["offset"] = offset
+
+            logger.debug(f"Request params: {params}")
+
+            # Try up to 3 times with exponential backoff
+            for attempt in range(3):
+                try:
+                    logger.debug(f"Attempt {attempt+1}/3")
+
+                    response = self._session.get(
+                        url, params=params, timeout=30, **kwargs
+                    )
+
+                    logger.debug(f"Response status code: {response.status_code}")
+
+                    response.raise_for_status()  # Raise exception for HTTP errors
+
+                    if response.status_code == 200:
+                        logger.debug(
+                            f"Request successful. Content length: {len(response.content)}"
+                        )
+
+                    return response
+
+                except (
+                    requests.exceptions.RequestException,
+                    requests.exceptions.HTTPError,
+                ) as e:
+                    logger.warning(f"Request failed: {e}")
+
+                    if attempt == 2:  # Last attempt
+                        logger.error(f"Request failed after 3 attempts: {e}")
+                        print(f"Request failed: {e}")
+                        return None
+
+                    logger.debug(f"Retrying...")
+                    continue  # Try again
+
+        except Exception as err:
+            logger.error(f"Unexpected error during request: {err}", exc_info=True)
+            print(f"Error: {err}")
+            return None
+
+    def process_response(self, response: requests.Response) -> Dict[str, Any]:
+        """Process HTTP response and convert to JSON.
+
+        Args:
+            response: HTTP response object
+
+        Returns:
+            JSON content as dictionary
+        """
+        try:
+            json_data = response.json()
+            if logger.isEnabledFor(logging.DEBUG):
+                logger.debug(f"Response JSON keys: {list(json_data.keys())}")
+                if "items" in json_data:
+                    logger.debug(f"Number of items: {len(json_data['items'])}")
+                    if len(json_data["items"]) > 0:
+                        logger.debug(
+                            f"First item keys: {list(json_data['items'][0].keys())}"
+                        )
+            return json_data
+        except json.JSONDecodeError:
+            logger.error("Error decoding JSON response", exc_info=True)
+            print("Error decoding JSON response")
+            return {}
+
+    @classmethod
+    def get_instance(cls) -> "ONSClient":
+        """Get a singleton instance of the ONS client.
+
+        Returns:
+            ONSClient instance
+        """
+        if not hasattr(cls, "_instance"):
+            cls._instance = cls()
+        return cls._instance
+
+
+# Create a default client instance
+default_client = ONSClient.get_instance()

onspy/code_lists.py

@@ -0,0 +1,300 @@
+"""
+Explore codes and lists.
+
+This module provides functions to get details about codes and code lists stored by ONS.
+Codes are used to provide a common definition when presenting statistics with related categories.
+"""
+
+from typing import Optional, List, Dict, Any
+
+from .utils import (
+    build_base_request,
+    make_request,
+    process_response,
+    EMPTY,
+)
+
+
+def ons_codelists() -> Optional[List[str]]:
+    """Get a list of all available code lists.
+
+    Returns:
+        List of code list IDs, or None if the request fails
+
+    Examples:
+        >>> import onspy
+        >>> onspy.ons_codelists()
+    """
+    req = build_base_request(**{"code-lists": EMPTY})
+    res = make_request(req, limit=80)
+    if res is None:
+        return None
+
+    raw = process_response(res)
+
+    # Extract IDs from items
+    try:
+        return [item["links"]["self"]["id"] for item in raw.get("items", [])]
+    except (KeyError, TypeError):
+        return []
+
+
+def assert_valid_codeid(id: str) -> bool:
+    """Check if a code list ID is valid.
+
+    Args:
+        id: Code list ID
+
+    Returns:
+        True if valid, raises ValueError otherwise
+    """
+    if id is None:
+        raise ValueError("You must specify a 'code_id', see ons_codelists()")
+
+    ids = ons_codelists()
+    if ids is None:
+        return False
+
+    if id not in ids:
+        raise ValueError(f"Invalid code_id '{id}'. See ons_codelists() for valid IDs.")
+
+    return True
+
+
+def ons_codelist(code_id: str = None) -> Optional[Dict[str, Any]]:
+    """Get details for a specific code list.
+
+    Args:
+        code_id: Code list ID
+
+    Returns:
+        Dictionary with code list details, or None if the request fails
+
+    Examples:
+        >>> import onspy
+        >>> onspy.ons_codelist(code_id="quarter")
+    """
+    if not assert_valid_codeid(code_id):
+        return None
+
+    req = build_base_request(**{"code-lists": code_id})
+    res = make_request(req)
+    if res is None:
+        return None
+
+    return process_response(res)
+
+
+def ons_codelist_editions(code_id: str = None) -> Optional[List[Dict[str, Any]]]:
+    """Get editions for a code list.
+
+    Args:
+        code_id: Code list ID
+
+    Returns:
+        List of editions, or None if the request fails
+
+    Examples:
+        >>> import onspy
+        >>> onspy.ons_codelist_editions(code_id="quarter")
+    """
+    if not assert_valid_codeid(code_id):
+        return None
+
+    req = build_base_request(**{"code-lists": code_id, "editions": EMPTY})
+    res = make_request(req)
+    if res is None:
+        return None
+
+    raw = process_response(res)
+    return raw.get("items", [])
+
+
+def assert_valid_edition(code_id: str, edition: str) -> bool:
+    """Check if an edition is valid for a code list.
+
+    Args:
+        code_id: Code list ID
+        edition: Edition name
+
+    Returns:
+        True if valid, raises ValueError otherwise
+    """
+    if edition is None:
+        raise ValueError("You must specify an 'edition', see ons_codelist_editions()")
+
+    editions = ons_codelist_editions(code_id)
+    if editions is None:
+        return False
+
+    edition_names = [e.get("edition", "") for e in editions]
+    if edition not in edition_names:
+        raise ValueError(
+            f"Invalid edition '{edition}'. Valid editions are: {', '.join(edition_names)}"
+        )
+
+    return True
+
+
+def ons_codelist_edition(
+    code_id: str = None, edition: str = None
+) -> Optional[Dict[str, Any]]:
+    """Get details for a specific edition of a code list.
+
+    Args:
+        code_id: Code list ID
+        edition: Edition name
+
+    Returns:
+        Dictionary with edition details, or None if the request fails
+
+    Examples:
+        >>> import onspy
+        >>> onspy.ons_codelist_edition(code_id="quarter", edition="one-off")
+    """
+    if not assert_valid_codeid(code_id):
+        return None
+
+    if not assert_valid_edition(code_id, edition):
+        return None
+
+    req = build_base_request(**{"code-lists": code_id, "editions": edition})
+    res = make_request(req)
+    if res is None:
+        return None
+
+    return process_response(res)
+
+
+def ons_codes(
+    code_id: str = None, edition: str = None
+) -> Optional[List[Dict[str, Any]]]:
+    """Get codes for a specific edition of a code list.
+
+    Args:
+        code_id: Code list ID
+        edition: Edition name
+
+    Returns:
+        List of codes, or None if the request fails
+
+    Examples:
+        >>> import onspy
+        >>> onspy.ons_codes(code_id="quarter", edition="one-off")
+    """
+    if not assert_valid_codeid(code_id):
+        return None
+
+    if not assert_valid_edition(code_id, edition):
+        return None
+
+    req = build_base_request(
+        **{"code-lists": code_id, "editions": edition, "codes": EMPTY}
+    )
+    res = make_request(req)
+    if res is None:
+        return None
+
+    raw = process_response(res)
+    return raw.get("items", [])
+
+
+def assert_valid_code(code_id: str, edition: str, code: str) -> bool:
+    """Check if a code is valid for an edition of a code list.
+
+    Args:
+        code_id: Code list ID
+        edition: Edition name
+        code: Code value
+
+    Returns:
+        True if valid, raises ValueError otherwise
+    """
+    if code is None:
+        raise ValueError("You must specify a 'code', see ons_codes()")
+
+    codes = ons_codes(code_id, edition)
+    if codes is None:
+        return False
+
+    code_values = [c.get("code", "") for c in codes]
+    if code not in code_values:
+        raise ValueError(
+            f"Invalid code '{code}'. Valid codes are: {', '.join(code_values)}"
+        )
+
+    return True
+
+
+def ons_code(
+    code_id: str = None, edition: str = None, code: str = None
+) -> Optional[Dict[str, Any]]:
+    """Get details for a specific code.
+
+    Args:
+        code_id: Code list ID
+        edition: Edition name
+        code: Code value
+
+    Returns:
+        Dictionary with code details, or None if the request fails
+
+    Examples:
+        >>> import onspy
+        >>> onspy.ons_code(code_id="quarter", edition="one-off", code="q2")
+    """
+    if not assert_valid_codeid(code_id):
+        return None
+
+    if not assert_valid_edition(code_id, edition):
+        return None
+
+    if not assert_valid_code(code_id, edition, code):
+        return None
+
+    req = build_base_request(
+        **{"code-lists": code_id, "editions": edition, "codes": code}
+    )
+    res = make_request(req)
+    if res is None:
+        return None
+
+    return process_response(res)
+
+
+def ons_code_dataset(
+    code_id: str = None, edition: str = None, code: str = None
+) -> Optional[List[Dict[str, Any]]]:
+    """Get datasets that use a specific code.
+
+    Args:
+        code_id: Code list ID
+        edition: Edition name
+        code: Code value
+
+    Returns:
+        List of datasets, or None if the request fails
+
+    Examples:
+        >>> import onspy
+        >>> onspy.ons_code_dataset(code_id="quarter", edition="one-off", code="q2")
+    """
+    if not assert_valid_codeid(code_id):
+        return None
+
+    if not assert_valid_edition(code_id, edition):
+        return None
+
+    if not assert_valid_code(code_id, edition, code):
+        return None
+
+    req = build_base_request(
+        **{"code-lists": code_id, "editions": edition, "codes": code, "datasets": EMPTY}
+    )
+
+    res = make_request(req)
+    if res is None:
+        return None
+
+    raw = process_response(res)
+    return raw.get("items", [])