notoecd 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

notoecd/__init__.py

@@ -0,0 +1,7 @@
+__version__ = "0.1.2"
+
+from .calls import get_df
+from .structure import get_structure
+from .datasets import search_keywords
+
+__all__ = ["get_df", "get_structure", "search_keywords"]

notoecd/calls.py

@@ -0,0 +1,57 @@
+import pandas as pd
+from typing import Union
+from functools import lru_cache
+from .structure import get_structure
+
+@lru_cache(maxsize=256)
+def _fetch_df(url: str) -> pd.DataFrame:
+    return pd.read_csv(url, storage_options={"User-Agent": "Mozilla/5.0"})
+
+
+def _clean(s: str) -> str: 
+    return str(s).strip().lower()
+
+
+def _build_filter_expression(
+    agencyID: str,
+    dataflowID: str,
+    filters: dict,
+) -> str:
+    
+    s = get_structure(agencyID, dataflowID)
+    filters = {_clean(k): v for k, v in filters.items()}
+
+    parts = []
+    for dim in s.toc.title:
+        dim_key = _clean(dim)
+        if dim_key in filters:
+            val = filters[dim_key]
+            if isinstance(val, str):
+                val = [val]
+            parts.append("+".join(_clean(v) for v in val))
+        else:
+            parts.append("")
+
+    return ".".join(parts).upper()
+
+
+def get_df(
+    agencyID: str,
+    dataflowID: str,
+    filters: Union[str, dict],
+    version: str = "",
+) -> pd.DataFrame:
+    
+    if isinstance(filters, dict):
+        filter_expression = _build_filter_expression(agencyID, dataflowID, filters)
+    else:
+        filter_expression = _clean(filters).upper()
+
+    url = (
+        f"https://sdmx.oecd.org/public/rest/data/"
+        f"{agencyID},{dataflowID},{version}/{filter_expression}"
+        "?dimensionAtObservation=AllDimensions&format=csvfile"
+    )
+
+    base_df = _fetch_df(url)
+    return base_df.copy()

notoecd/datasets.py

@@ -0,0 +1,122 @@
+import re
+import html
+import requests
+import unicodedata
+import pandas as pd
+from typing import Union, List
+import xml.etree.ElementTree as ET
+
+url = "https://sdmx.oecd.org/public/rest/dataflow/all"
+
+NS = {
+    "message": "http://www.sdmx.org/resources/sdmxml/schemas/v2_1/message",
+    "structure": "http://www.sdmx.org/resources/sdmxml/schemas/v2_1/structure",
+    "common": "http://www.sdmx.org/resources/sdmxml/schemas/v2_1/common",
+    "xml": "http://www.w3.org/XML/1998/namespace",
+}
+
+_ws_re = re.compile(r"\s+")
+_tag_re = re.compile(r"<[^>]+>")
+
+def _clean(s: str | None) -> str | None:
+    if s is None: return None
+    s = html.unescape(s)
+    s = _tag_re.sub("", s)
+    s = _ws_re.sub(" ", s).strip()
+    return s or None
+
+# Cache
+_datasets: pd.DataFrame | None = None
+
+def _load_datasets() -> pd.DataFrame:
+    """
+    Loads OECD datasets and keeps them in memory.
+    """
+    global _datasets
+    if _datasets is not None: return _datasets
+
+    headers = {"Accept": "application/vnd.sdmx.structure+xml;version=2.1"}
+    r = requests.get(url, headers=headers, timeout=30)
+    r.raise_for_status()
+    root = ET.fromstring(r.content)
+
+    rows = []
+    for df in root.findall(".//structure:Dataflow", NS):
+        dataflow_id = df.attrib.get("id")
+        agency_id = df.attrib.get("agencyID")
+
+        name_elem = df.find("common:Name[@xml:lang='en']", NS)
+        desc_elem = df.find("common:Description[@xml:lang='en']", NS)
+
+        name = _clean("".join(name_elem.itertext())) if name_elem is not None else None
+        desc_raw = "".join(desc_elem.itertext()) if desc_elem is not None else None
+        desc = _clean(desc_raw)
+
+        rows.append(
+            {
+                "dataflowID": dataflow_id,
+                "agencyID": agency_id,
+                "name": name,
+                "description": desc,
+            }
+        )
+
+    _datasets = pd.DataFrame(rows)
+    return _datasets
+
+def search_keywords(keywords: Union[str, List[str]]) -> pd.DataFrame:
+    """
+    Searches OECD datasets for a set of keywords.
+
+    Args:
+        keywords (str | list[str]): Single keyword or list of keywords. Acts as OR.
+
+    Returns:
+        pd.DataFrame: Matching rows.
+    """
+    datasets = _load_datasets()
+
+    # Normalize keywords input
+    if isinstance(keywords, str): keywords = [keywords]
+    elif not isinstance(keywords, list): raise TypeError("keywords must be a string or list of strings")
+
+    # Clean and drop empty keywords
+    keywords = [k for k in keywords if isinstance(k, str) and k.strip()]
+    if not keywords: raise ValueError("No valid keywords provided.")
+
+    def _normalize_series(s: pd.Series) -> pd.Series:
+        s = s.fillna("").astype(str).str.lower()
+        return s.map(
+            lambda x: "".join(ch for ch in unicodedata.normalize("NFKD", x)
+                if not unicodedata.combining(ch))
+        )
+
+    # Combined normalized text for each row
+    text = datasets["name"].fillna("").astype(str) + " " + datasets["description"].fillna("").astype(str)
+    text_norm = _normalize_series(text)
+
+    # Normalize keywords similarly
+    def _normalize_kw(kw: str) -> str:
+        kw = kw.lower()
+        kw = unicodedata.normalize("NFKD", kw)
+        return "".join(ch for ch in kw if not unicodedata.combining(ch))
+
+    norm_keywords = [_normalize_kw(k) for k in keywords]
+
+    # Vectorized OR search + simple score = count of matching keywords
+    overall_mask = pd.Series(False, index=datasets.index)
+    score = pd.Series(0, index=datasets.index, dtype="int64")
+
+    for kw in norm_keywords:
+        if not kw: continue
+        # plain substring search, no regex
+        m = text_norm.str.contains(kw, na=False, regex=False)
+        overall_mask |= m
+        score = score.add(m.astype("int8"), fill_value=0)
+
+    # Filter and sort by relevance
+    result = datasets.loc[overall_mask].copy()
+    result["_match_score"] = score.loc[overall_mask]
+    result = result.sort_values("_match_score", ascending=False)
+
+    return result[['agencyID', 'dataflowID', 'name', 'description']]

notoecd/structure.py

@@ -0,0 +1,144 @@
+import requests
+import pandas as pd
+from functools import lru_cache
+import xml.etree.ElementTree as ET
+
+class Structure():
+    def __init__(self, agencyID:str, dataflowID:str):
+        self.ns = {
+            'message': 'http://www.sdmx.org/resources/sdmxml/schemas/v2_1/message',
+            'common': 'http://www.sdmx.org/resources/sdmxml/schemas/v2_1/common',
+            'structure': 'http://www.sdmx.org/resources/sdmxml/schemas/v2_1/structure',
+            'xml': 'http://www.w3.org/XML/1998/namespace'
+        }
+
+        url = f"https://sdmx.oecd.org/public/rest/dataflow/{agencyID}/{dataflowID}/?references=all"
+        response = requests.get(url)
+        tree = ET.ElementTree(ET.fromstring(response.content))
+        self.root = tree.getroot()
+
+        self.concepts = self.build_concepts_dict()
+        self.values = self.build_values_dict()
+        self.toc = self.build_toc()
+        self.params = list(self.values.keys())
+
+    def build_concepts_dict(self) -> dict:
+        # Extract dimensions from the Concepts section
+        concepts = {'DIMENSIONS':{}, 'CODELISTS':{}}
+        conceptScheme_section = self.root.findall('.//message:Structures/structure:Concepts/structure:ConceptScheme', self.ns)
+        if conceptScheme_section:
+            for concept_scheme in conceptScheme_section:
+                for concept in concept_scheme.findall('structure:Concept', self.ns):
+                    concept_id = concept.get('id')
+                    name_elem = concept.find('common:Name[@{http://www.w3.org/XML/1998/namespace}lang="en"]', self.ns)
+                    if concept_id and name_elem is not None:
+                        concepts['DIMENSIONS'][concept_id] = name_elem.text
+                    for core_representation in concept.findall('structure:CoreRepresentation', self.ns):
+                        for enumeration in core_representation.findall('structure:Enumeration', self.ns):
+                            ref_elem = enumeration.find('Ref')
+                            if ref_elem is not None:
+                                codelist_id = ref_elem.get('id')
+                                if concept_id and codelist_id:
+                                    concepts['CODELISTS'][concept_id] = (
+                                        codelist_id.split("CL_")[1] if codelist_id.startswith("CL_") 
+                                        else codelist_id
+                                    )
+        
+        # Extract codes from the Codelists section
+        codes_section = self.root.findall('.//message:Structures/structure:Codelists/structure:Codelist', self.ns)
+        if codes_section:
+            for code_scheme in codes_section:
+                dimension = code_scheme.get('id')[3:]
+                concepts[dimension] = {}
+                for code in code_scheme.findall('structure:Code', self.ns):
+                    code_id = code.get('id')
+                    name_elem = code.find('common:Name[@{http://www.w3.org/XML/1998/namespace}lang="en"]', self.ns)
+                    if code_id and name_elem is not None:
+                        concepts[dimension][code_id] = name_elem.text
+        return concepts
+
+    def build_values_dict(self) -> dict:
+        # Extract values from Constraints section
+        values = {}
+        constraints_section = self.root.findall('.//message:Structures/structure:Constraints/structure:ContentConstraint', self.ns)
+        if constraints_section:
+            constraint = constraints_section[0]
+            cube_region = constraint.find('structure:CubeRegion', self.ns)
+            if cube_region is not None:
+                for key_value in cube_region.findall('common:KeyValue', self.ns):
+                    key = key_value.get('id')
+                    value_elems = key_value.findall('common:Value', self.ns)
+                    if key is not None and value_elems:
+                        values[key] = [value_elem.text for value_elem in value_elems if value_elem.text]
+        return values
+
+    def build_toc(self) -> pd.DataFrame:
+        # Builds a DataFrame containing the call dimensions and options (table of contents)
+        data_structure = self.root.find('.//message:Structures/structure:DataStructures/structure:DataStructure', self.ns)
+        dimension_list = data_structure.find('.//structure:DataStructureComponents/structure:DimensionList', self.ns)
+        rows = []
+        
+        if dimension_list is not None:
+            dimensions = dimension_list.findall('.//structure:Dimension', self.ns)
+            
+            # Sort dimensions by their 'position' to ensure correct order
+            sorted_dimensions = sorted(dimensions, key=lambda dim: int(dim.get('position')))
+            
+            for dimension in sorted_dimensions:
+                dimension_id = dimension.get('id')
+                position = dimension.get('position')
+
+                # Get the corresponding values for the dimension (if present in the 'values' dictionary)
+                dimension_values = self.values.get(dimension_id, [])
+                
+                rows.append({
+                    'id': int(position),
+                    'title': dimension_id,
+                    'values': dimension_values
+                })
+
+        df = pd.DataFrame(rows)
+        df = df.sort_values(by='id').reset_index(drop=True)
+        return df
+
+    def explain_vals(self, dimension:str) -> dict:
+        # Explains the parameters of a given dimension
+        dimension = dimension.strip().upper()
+        concepts = self.concepts
+        values = self.values[dimension]
+        codelists = concepts['CODELISTS']
+
+        def fallback(dimension, concepts):
+            # Look for subset of keys
+            toc = self.toc
+            concept_mapping = {}
+            for title in toc.title:
+                vals = set(toc[toc['title'] == title].values[0][2])
+                concept_mapping[title] = [i for i in concepts.keys() if vals.issubset(set(concepts[i].keys()))][0]
+            concept_mapping['FREQ'] = 'FREQ'
+            if (dimension not in concepts) and (dimension in concept_mapping): 
+                    dimension = concept_mapping[dimension]
+            concepts = concepts[dimension]
+            return {i: concepts[i] for i in values}
+
+        if dimension not in codelists:
+            if dimension in concepts: 
+                return concepts[dimension]
+            return fallback(dimension, concepts)
+        
+        dim_codelist = codelists[dimension]
+        explanation = concepts[dim_codelist]
+        clean_values = [i for i in values if i in explanation]
+
+        if len(clean_values) == 0:
+            return fallback(dimension, concepts)
+
+        unclean_values = [i for i in values if i not in explanation]
+        if len(unclean_values) > 0:
+            print(f"Could not find explanation for {unclean_values}")
+
+        return {i:explanation[i] for i in clean_values}
+
+@lru_cache(maxsize=128)
+def get_structure(agencyID: str, dataflowID: str) -> Structure:
+    return Structure(agencyID, dataflowID)

{notoecd-0.1.0.dist-info → notoecd-0.1.2.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: notoecd
-Version: 0.1.0
-Summary: Library for interacting with the OECD Data Explorer through Python
+Version: 0.1.2
+Summary: Unofficial library for interacting with the OECD Data Explorer through Python.
 Author-email: Daniel Vegara Balsa <daniel.vegarabalsa@oecd.org>
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/dani-37/notoecd
@@ -14,7 +14,8 @@ Requires-Dist: requests>=2.31
 
 ⚠️ **Unofficial package, not endorsed by the OECD.**
 
-A lightweight Python interface for exploring OECD SDMX structures and downloading OECD regional datasets.  
+A lightweight Python interface for exploring OECD SDMX structures and downloading OECD datasets.  
+
 The package provides utilities for:
 
 - Discovering dataset metadata  
@@ -110,5 +111,3 @@ The returned object is a pandas DataFrame containing the requested subset of OEC
 ## Examples
 
 You can see this full example as a notebook called example.ipynb.
-
-

notoecd-0.1.2.dist-info/RECORD

@@ -0,0 +1,8 @@
+notoecd/__init__.py,sha256=J3-DA__Z9hHzp6eOszRMFh4VE6tk1YtSNigW7el4qDM,189
+notoecd/calls.py,sha256=CDx-1wJ4myXtoihIfTvjHoXBvIwylvv7AdN_UL5gnF4,1492
+notoecd/datasets.py,sha256=a1L45vie6q4cjNXAued2gQ4oE4Fbpk25kdhDTexMuAI,4158
+notoecd/structure.py,sha256=sq6HrjNLfK-UWr9Cuqxun_DhHLPdegX7j7pKYcEYcJM,7169
+notoecd-0.1.2.dist-info/METADATA,sha256=kt6OjNXdlGpn5K-_WQm-z-GLKbbPLgF3mq0C7YSGNho,3192
+notoecd-0.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+notoecd-0.1.2.dist-info/top_level.txt,sha256=GrcbH10OAguGh5dkpzst216N_C-NtZ-QF1nlXiUpeLs,8
+notoecd-0.1.2.dist-info/RECORD,,

notoecd-0.1.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+notoecd

notoecd-0.1.0.dist-info/RECORD

@@ -1,4 +0,0 @@
-notoecd-0.1.0.dist-info/METADATA,sha256=UcQeYIOzhpdpZE5HcFVm_7DCjxdJmLkcCdbSDsrPTHY,3191
-notoecd-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-notoecd-0.1.0.dist-info/top_level.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-notoecd-0.1.0.dist-info/RECORD,,

notoecd-0.1.0.dist-info/top_level.txt

@@ -1 +0,0 @@
-