comexpy 0.1.0__py3-none-any.whl

comexpy/__init__.py

@@ -0,0 +1,102 @@
+"""comexpy — access the Brazilian Foreign Trade Statistics API (ComexStat).
+
+A pandas-friendly Python port of the R package ``comexr``. Query Brazilian
+export and import data — general trade statistics (1997-present), city-level
+data, and historical records (1989-1996) — plus auxiliary tables for product
+codes (NCM, NBM, HS), countries, economic blocs and classifications (CGCE,
+SITC, ISIC). The public function names mirror the R API so knowledge
+transfers directly between the two.
+
+Every query/table function returns a :class:`pandas.DataFrame`; detail
+functions return a ``dict``.
+
+Quick start
+-----------
+>>> import comexpy
+>>> comexpy.comex_export("2024-01", "2024-12", details="country")   # doctest: +SKIP
+>>> comexpy.comex_countries(search="china")                          # doctest: +SKIP
+>>> comexpy.comex_details("general")                                 # doctest: +SKIP
+"""
+from __future__ import annotations
+
+from ._client import BASE_URL, ComexError, get_options, set_options
+from ._format import DETAILS_MAP
+from ._msg import set_verbose
+from .historical import comex_historical
+from .query import comex_export, comex_import, comex_query
+from .query_city import comex_query_city
+from .tables import (
+    comex_available_years,
+    comex_blocs,
+    comex_cities,
+    comex_city_detail,
+    comex_countries,
+    comex_country_detail,
+    comex_customs_unit_detail,
+    comex_customs_units,
+    comex_details,
+    comex_filter_values,
+    comex_filters,
+    comex_last_update,
+    comex_metrics,
+    comex_state_detail,
+    comex_states,
+    comex_transport_mode_detail,
+    comex_transport_modes,
+)
+from .tables_classifications import comex_cgce, comex_isic, comex_sitc
+from .tables_products import (
+    comex_hs,
+    comex_nbm,
+    comex_nbm_detail,
+    comex_ncm,
+    comex_ncm_detail,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # Query functions (POST)
+    "comex_query",
+    "comex_export",
+    "comex_import",
+    "comex_query_city",
+    "comex_historical",
+    # API metadata (GET)
+    "comex_last_update",
+    "comex_available_years",
+    "comex_filters",
+    "comex_filter_values",
+    "comex_details",
+    "comex_metrics",
+    # Auxiliary tables - geography
+    "comex_countries",
+    "comex_country_detail",
+    "comex_blocs",
+    "comex_states",
+    "comex_state_detail",
+    "comex_cities",
+    "comex_city_detail",
+    "comex_transport_modes",
+    "comex_transport_mode_detail",
+    "comex_customs_units",
+    "comex_customs_unit_detail",
+    # Auxiliary tables - products
+    "comex_ncm",
+    "comex_ncm_detail",
+    "comex_nbm",
+    "comex_nbm_detail",
+    "comex_hs",
+    # Auxiliary tables - classifications
+    "comex_cgce",
+    "comex_sitc",
+    "comex_isic",
+    # Configuration / helpers
+    "set_options",
+    "get_options",
+    "set_verbose",
+    "ComexError",
+    "DETAILS_MAP",
+    "BASE_URL",
+    "__version__",
+]

comexpy/_client.py

@@ -0,0 +1,202 @@
+"""HTTP layer for the ComexStat API.
+
+Mirrors ``comex_get`` / ``comex_post`` from the R package's ``utils.R``:
+
+* GET and POST helpers against :data:`BASE_URL`.
+* User-configurable retry/timeout behaviour (the ComexStat API rate-limits
+  aggressively with HTTP 429 and a 10-second recommended back-off).
+* Automatic retry on SSL certificate-verification failures — the ComexStat
+  servers use ICP-Brasil certificates that some systems do not trust.
+
+On failure a :class:`ComexError` is raised with a friendly message.
+"""
+from __future__ import annotations
+
+import time
+from typing import Any, Mapping, Optional
+
+import requests
+
+from . import _msg
+
+BASE_URL = "https://api-comexstat.mdic.gov.br"
+
+_USER_AGENT = "comexpy (Python package)"
+
+# Defaults mirror the R package options (comexr.*). The API recommends a
+# 10-second wait after a 429, so retry_time defaults to 10.
+_CONFIG = {
+    "timeout_get": 60,
+    "timeout_post": 120,
+    "max_tries": 3,
+    "retry_time": 10,
+    "ssl_verify": True,
+}
+
+
+class ComexError(RuntimeError):
+    """Raised when a ComexStat API request fails."""
+
+
+def set_options(
+    *,
+    timeout_get: Optional[int] = None,
+    timeout_post: Optional[int] = None,
+    max_tries: Optional[int] = None,
+    retry_time: Optional[int] = None,
+    ssl_verify: Optional[bool] = None,
+) -> None:
+    """Configure HTTP retry/timeout behaviour (equivalent to the R options).
+
+    The ComexStat API frequently returns rate-limit errors (HTTP 429,
+    *"Você excedeu o limite de solicitações..."*) or times out. Adjust these
+    settings to work around such errors without overloading the servers.
+
+    Parameters
+    ----------
+    timeout_get : int, optional
+        Seconds to wait for a response on GET requests (default 60).
+    timeout_post : int, optional
+        Seconds to wait for a response on POST requests (default 120).
+    max_tries : int, optional
+        Maximum number of attempts for a failing request (default 3).
+        Adjusting ``retry_time`` is generally a better way to avoid errors.
+    retry_time : int, optional
+        Seconds to wait between retries after a transient failure
+        (default 10, matching the API's recommended back-off).
+    ssl_verify : bool, optional
+        Whether to verify SSL certificates. Set to ``False`` to skip
+        verification when the ICP-Brasil certificate chain is not trusted.
+    """
+    if timeout_get is not None:
+        _CONFIG["timeout_get"] = int(timeout_get)
+    if timeout_post is not None:
+        _CONFIG["timeout_post"] = int(timeout_post)
+    if max_tries is not None:
+        _CONFIG["max_tries"] = int(max_tries)
+    if retry_time is not None:
+        _CONFIG["retry_time"] = int(retry_time)
+    if ssl_verify is not None:
+        _CONFIG["ssl_verify"] = bool(ssl_verify)
+
+
+def get_options() -> dict:
+    """Return a copy of the current HTTP configuration."""
+    return dict(_CONFIG)
+
+
+def _is_ssl_error(exc: Exception) -> bool:
+    text = f"{exc} {getattr(exc, '__cause__', '')}".lower()
+    return any(t in text for t in ("ssl", "certificate", "peer"))
+
+
+def _perform(method: str, url: str, **kwargs: Any) -> requests.Response:
+    """Perform a request with retries and SSL auto-fallback."""
+    max_tries = max(1, int(_CONFIG["max_tries"]))
+    retry_time = int(_CONFIG["retry_time"])
+    verify = bool(_CONFIG["ssl_verify"])
+
+    last_exc: Optional[Exception] = None
+    for attempt in range(1, max_tries + 1):
+        try:
+            resp = requests.request(method, url, verify=verify, **kwargs)
+        except requests.exceptions.SSLError as exc:
+            # SSL verification failed: retry once without verification and
+            # remember the choice for the rest of the session.
+            if verify:
+                _msg.warn(
+                    "SSL certificate verification failed. Retrying without "
+                    "SSL verification. To suppress this, call "
+                    "comexpy.set_options(ssl_verify=False)."
+                )
+                _CONFIG["ssl_verify"] = False
+                verify = False
+                last_exc = exc
+                continue
+            last_exc = exc
+        except requests.RequestException as exc:
+            last_exc = exc
+        else:
+            # Retry on rate-limit / transient server errors.
+            if resp.status_code in (429, 500, 502, 503, 504) and attempt < max_tries:
+                time.sleep(retry_time)
+                continue
+            return resp
+
+        if attempt < max_tries:
+            time.sleep(retry_time)
+
+    raise ComexError(
+        f"Failed to perform HTTP request to the ComexStat API.\n"
+        f"  x {last_exc}\n  i URL: {url}"
+    )
+
+
+def _check(resp: requests.Response, endpoint: str) -> Any:
+    if resp.status_code >= 400:
+        try:
+            body = resp.json()
+            msg = body.get("message") or (
+                body.get("error", {}).get("message")
+                if isinstance(body.get("error"), dict)
+                else None
+            )
+        except ValueError:
+            msg = None
+        msg = msg or f"HTTP {resp.status_code}"
+        raise ComexError(
+            f"API request failed (HTTP {resp.status_code})\n"
+            f"  i Endpoint: {endpoint}\n  i Message: {msg}"
+        )
+    try:
+        return resp.json()
+    except ValueError as exc:  # pragma: no cover
+        raise ComexError(
+            f"Could not parse API response as JSON.\n  i Endpoint: {endpoint}"
+        ) from exc
+
+
+def comex_get(
+    endpoint: str,
+    query: Optional[Mapping[str, Any]] = None,
+    verbose: bool = True,
+) -> Any:
+    """Perform a GET request to the ComexStat API and return parsed JSON."""
+    url = BASE_URL + endpoint
+    if verbose:
+        _msg.step(f"GET {endpoint}")
+    clean = {k: v for k, v in (query or {}).items() if v is not None}
+    resp = _perform(
+        "GET",
+        url,
+        params=clean or None,
+        headers={"Accept": "application/json", "User-Agent": _USER_AGENT},
+        timeout=_CONFIG["timeout_get"],
+    )
+    return _check(resp, endpoint)
+
+
+def comex_post(
+    endpoint: str,
+    body: Any,
+    query: Optional[Mapping[str, Any]] = None,
+    verbose: bool = True,
+) -> Any:
+    """Perform a POST request to the ComexStat API and return parsed JSON."""
+    url = BASE_URL + endpoint
+    if verbose:
+        _msg.step(f"POST {endpoint}")
+    clean = {k: v for k, v in (query or {}).items() if v is not None}
+    resp = _perform(
+        "POST",
+        url,
+        params=clean or None,
+        json=body,
+        headers={
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+            "User-Agent": _USER_AGENT,
+        },
+        timeout=_CONFIG["timeout_post"],
+    )
+    return _check(resp, endpoint)

comexpy/_format.py

@@ -0,0 +1,275 @@
+"""Response conversion, validation and name mappings.
+
+Ported from the R package's ``utils.R``:
+
+* :func:`response_to_df` / :func:`extract_single` handle the ComexStat
+  response shapes empirically observed across endpoints.
+* :func:`validate_period` / :func:`convert_flow` validate user arguments.
+* :data:`DETAILS_MAP` translates user-friendly aliases (``hs4``,
+  ``transport_mode``, ``cgce_n1``, …) to the API's internal names
+  (``heading``, ``via``, ``BECLevel1``, …).
+"""
+from __future__ import annotations
+
+import re
+from typing import Any, Mapping, Optional
+
+import pandas as pd
+
+# -------------------------------------------------------------------------
+# Response conversion
+# -------------------------------------------------------------------------
+
+
+def _flatten_value(val: Any) -> Any:
+    """Collapse nested lists to a comma-joined string; keep scalars as-is."""
+    if val is None:
+        return None
+    if isinstance(val, (list, tuple)):
+        return ", ".join(str(v) for v in val)
+    if isinstance(val, dict):
+        return ", ".join(str(v) for v in val.values())
+    return val
+
+
+def response_to_df(response: Any, path: str = "data") -> pd.DataFrame:
+    """Convert a ComexStat API response into a :class:`pandas.DataFrame`.
+
+    Handles every known response pattern:
+
+    * ``{"data": {"list": [...rows...], "count": N}}`` — most endpoints,
+      including POST ``/general`` and ``/cities``.
+    * ``{"data": [...rows...]}`` — ``/tables/uf``, ``/tables/cities``,
+      ``/tables/ways``, ``/tables/urf``, POST ``/historical-data/``.
+    * ``{"data": [[...rows...]]}`` — ``/general/filters/{filter}``.
+    """
+    data: Any = response
+    if isinstance(response, Mapping) and path in response:
+        data = response[path]
+
+    if data is None:
+        return pd.DataFrame()
+
+    # Pattern 1: {"list": [...], "count": N}
+    if isinstance(data, Mapping) and "list" in data:
+        data = data["list"]
+
+    if data is None:
+        return pd.DataFrame()
+
+    # Pattern 3: unwrap single-element unnamed lists wrapping the rows.
+    while (
+        isinstance(data, (list, tuple))
+        and len(data) == 1
+        and isinstance(data[0], (list, tuple))
+    ):
+        data = data[0]
+
+    if isinstance(data, (list, tuple)):
+        if len(data) == 0:
+            return pd.DataFrame()
+        rows = [
+            {k: _flatten_value(v) for k, v in row.items()}
+            if isinstance(row, Mapping)
+            else {"value": _flatten_value(row)}
+            for row in data
+        ]
+        return pd.DataFrame(rows)
+
+    # Named object that is not a list of rows -> single-row frame.
+    if isinstance(data, Mapping):
+        return pd.DataFrame([{k: _flatten_value(v) for k, v in data.items()}])
+
+    return pd.DataFrame()
+
+
+def extract_single(response: Any) -> Any:
+    """Extract a single record (dict/scalar) from a detail-endpoint response.
+
+    Handles ``{"data": {...}}``, ``{"data": [{...}]}``,
+    ``{"data": {"list": [{...}]}}`` and ``{"data": null}``.
+    """
+    if not isinstance(response, Mapping):
+        return None
+    data = response.get("data")
+    if data is None:
+        return None
+
+    if isinstance(data, Mapping) and "list" in data:
+        lst = data["list"]
+        if not lst:
+            return None
+        return lst[0]
+
+    if isinstance(data, (list, tuple)):
+        if len(data) == 0:
+            return None
+        return data[0]
+
+    return data
+
+
+# -------------------------------------------------------------------------
+# Validation
+# -------------------------------------------------------------------------
+
+_PERIOD_RE = re.compile(r"^\d{4}-\d{2}$")
+
+
+def validate_period(start_period: str, end_period: str) -> None:
+    """Validate ``YYYY-MM`` period strings and their ordering."""
+    if not _PERIOD_RE.match(str(start_period)):
+        raise ValueError(
+            f"Invalid start period: {start_period}. "
+            "Use format 'YYYY-MM' (e.g. '2023-01')."
+        )
+    if not _PERIOD_RE.match(str(end_period)):
+        raise ValueError(
+            f"Invalid end period: {end_period}. "
+            "Use format 'YYYY-MM' (e.g. '2023-12')."
+        )
+    if start_period > end_period:
+        raise ValueError("Start period must be before or equal to end period.")
+
+
+def convert_flow(flow: str) -> str:
+    """Normalise a trade-flow argument to ``"export"`` or ``"import"``."""
+    fl = str(flow).lower()
+    if fl in ("exp", "export", "exports"):
+        return "export"
+    if fl in ("imp", "import", "imports"):
+        return "import"
+    raise ValueError(f"Invalid flow: {flow}. Use 'export' or 'import'.")
+
+
+# -------------------------------------------------------------------------
+# Name mappings (user-friendly -> API names)
+# -------------------------------------------------------------------------
+
+#: Verified against the live endpoints (/general/filters, /general/details,
+#: /cities/filters, /historical-data/filters). The map also maps every API
+#: name to itself, so callers who already know the API name may pass it
+#: verbatim.
+DETAILS_MAP = {
+    # Geographic
+    "country": "country",
+    "bloc": "economicBlock",
+    "economic_block": "economicBlock",
+    "economicBlock": "economicBlock",
+    "state": "state",
+    "city": "city",
+    "transport_mode": "via",
+    "via": "via",
+    "customs_unit": "urf",
+    "urf": "urf",
+    # Products - NCM and Harmonized System (HS2/HS4/HS6)
+    "ncm": "ncm",
+    "hs6": "subHeading",
+    "sh6": "subHeading",
+    "subheading": "subHeading",
+    "subHeading": "subHeading",
+    "hs4": "heading",
+    "sh4": "heading",
+    "heading": "heading",
+    "hs2": "chapter",
+    "sh2": "chapter",
+    "chapter": "chapter",
+    "section": "section",
+    # CGCE (a.k.a. BEC - Broad Economic Categories)
+    "cgce_n1": "BECLevel1",
+    "cgce_n2": "BECLevel2",
+    "cgce_n3": "BECLevel3",
+    "BECLevel1": "BECLevel1",
+    "BECLevel2": "BECLevel2",
+    "BECLevel3": "BECLevel3",
+    # SITC / CUCI
+    "sitc_section": "SITCSection",
+    "sitc_division": "SITCDivision",
+    "sitc_chapter": "SITCDivision",
+    "sitc_group": "SITCGroup",
+    "sitc_position": "SITCGroup",
+    "sitc_subgroup": "SITCSubGroup",
+    "sitc_subposition": "SITCSubGroup",
+    "sitc_basic_heading": "SITCBasicHeading",
+    "sitc_item": "SITCBasicHeading",
+    "SITCSection": "SITCSection",
+    "SITCDivision": "SITCDivision",
+    "SITCGroup": "SITCGroup",
+    "SITCSubGroup": "SITCSubGroup",
+    "SITCBasicHeading": "SITCBasicHeading",
+    # ISIC
+    "isic_section": "ISICSection",
+    "isic_division": "ISICDivision",
+    "isic_group": "ISICGroup",
+    "isic_class": "ISICClass",
+    "ISICSection": "ISICSection",
+    "ISICDivision": "ISICDivision",
+    "ISICGroup": "ISICGroup",
+    "ISICClass": "ISICClass",
+    # NBM (historical)
+    "nbm": "nbm",
+}
+
+_API_NAMES = set(DETAILS_MAP.values())
+
+
+def get_api_name(name: str) -> str:
+    """Translate a user-friendly detail/filter alias to its API name."""
+    if name in DETAILS_MAP:
+        return DETAILS_MAP[name]
+    if name in _API_NAMES:
+        return name
+    from . import _msg
+
+    _msg.warn(f"Unknown detail/filter: {name}. Will be sent as-is.")
+    return name
+
+
+def _as_list(x: Any) -> list:
+    if x is None:
+        return []
+    if isinstance(x, (list, tuple, set)):
+        return list(x)
+    return [x]
+
+
+def build_details(details: Any) -> list:
+    """Build the API ``details`` list from user aliases."""
+    return [get_api_name(d) for d in _as_list(details)]
+
+
+def build_filters(filters: Optional[Mapping[str, Any]]) -> list:
+    """Build the API ``filters`` list from a name -> values mapping."""
+    if not filters:
+        return []
+    return [
+        {"filter": get_api_name(name), "values": _as_list(values)}
+        for name, values in filters.items()
+    ]
+
+
+def build_metrics(
+    metric_fob: bool = True,
+    metric_kg: bool = True,
+    metric_statistic: bool = False,
+    metric_freight: bool = False,
+    metric_insurance: bool = False,
+    metric_cif: bool = False,
+) -> list:
+    """Build the API ``metrics`` list from the metric flags."""
+    metrics = []
+    if metric_fob:
+        metrics.append("metricFOB")
+    if metric_kg:
+        metrics.append("metricKG")
+    if metric_statistic:
+        metrics.append("metricStatistic")
+    if metric_freight:
+        metrics.append("metricFreight")
+    if metric_insurance:
+        metrics.append("metricInsurance")
+    if metric_cif:
+        metrics.append("metricCIF")
+    if not metrics:
+        raise ValueError("At least one metric must be selected.")
+    return metrics

comexpy/_msg.py

@@ -0,0 +1,47 @@
+"""Lightweight console messages, mirroring the R package's cli alerts.
+
+Messages go to stderr and can be silenced with :func:`set_verbose`.
+"""
+from __future__ import annotations
+
+import sys
+
+_VERBOSE = True
+
+
+def set_verbose(verbose: bool) -> None:
+    """Enable or disable informational messages (success/step/info).
+
+    Warnings are always shown. Errors are raised as exceptions.
+
+    Parameters
+    ----------
+    verbose : bool
+        If ``False``, suppress progress and success messages.
+    """
+    global _VERBOSE
+    _VERBOSE = bool(verbose)
+
+
+def is_verbose() -> bool:
+    return _VERBOSE
+
+
+def info(message: str) -> None:
+    if _VERBOSE:
+        print(f"ℹ {message}", file=sys.stderr)
+
+
+def step(message: str) -> None:
+    if _VERBOSE:
+        print(f"→ {message}", file=sys.stderr)
+
+
+def success(message: str) -> None:
+    if _VERBOSE:
+        print(f"✔ {message}", file=sys.stderr)
+
+
+def warn(message: str) -> None:
+    # Warnings are always shown, regardless of verbosity.
+    print(f"! {message}", file=sys.stderr)