mrio-toolbox 1.1.1__py3-none-any.whl → 1.1.3__py3-none-any.whl

mrio_toolbox/extractors/extractors.py

@@ -1,79 +0,0 @@
-"""
-Module for extracting and converting data from various sources.
-"""
-
-import logging as log
-import os
-log = log.getLogger(__name__)
-
-def extract_MRIO(table,year,source,destination,
-                 preprocessing = False,
-                 saving_kwargs = dict(),
-                 extraction_kwargs = dict()):
-    """
-    Extract MRIO data and save it to a NetCDF file.
-
-    Specific extractors are called based on the table name.
-    Refer to the individual extractor functions for more details.
-    
-    Parameters
-    ----------
-    table : str
-        Name of the MRIO table to extract. Currently supported:
-        
-        - 'eora26': Extracts Eora26 data.
-        - 'gloria': Extracts GLORIA data.
-        - 'wiod': Extracts WIOD data.
-        - 'icio': Extracts ICIO data.
-        - 'exiobase3': Extracts EXIOBASE3 data.
-        - 'figaro': Extracts FIGARO data.
-        - 'emerging': Extracts EMERGING data.
-        - 'gtap11': Extracts GTAP 11 data.
-    
-    year : str
-        Year of the data to extract.
-    source : path-like
-        Path to the source directory containing the raw data files.
-    destination : path-like
-        Path to the destination directory where the NetCDF file will be saved.
-    preprocessing : dict
-        Parameters for preprocessing the table
-        If left empty, no preprocessing is done
-    extraction_kwargs : dict
-        Additional keyword arguments specific to the extractor function.
-    saving_kwargs : dict
-        Additional keyword arguments for saving the MRIO data
-    """
-    log.info(f"Extracting MRIO data for table '{table}' for year {year} from {source} to {destination}")
-    if table == 'eora26':
-        from mrio_toolbox.extractors.eora.eora_extractor import extract_eora26
-        mrio = extract_eora26(year, source, **extraction_kwargs)
-    elif table == 'gloria':
-        from mrio_toolbox.extractors.gloria.gloria_extractor import extract_gloria
-        mrio =extract_gloria(year, source, **extraction_kwargs)
-    elif table == 'wiod16':
-        from mrio_toolbox.extractors.wiod.wiod_extractor import extract_wiod
-        mrio = extract_wiod(year, source, **extraction_kwargs)
-    elif table == 'icio':
-        from mrio_toolbox.extractors.icio.icio_extractor import extract_icio
-        mrio =extract_icio(year, source, **extraction_kwargs)
-    elif table == 'exiobase3':
-        from mrio_toolbox.extractors.exiobase.exiobase_extractor import extract_exiobase3
-        mrio = extract_exiobase3(year, source, **extraction_kwargs)
-    elif table == 'figaro':
-        from mrio_toolbox.extractors.figaro.figaro_extractor import extract_figaro
-        mrio = extract_figaro(year, source, **extraction_kwargs)
-    elif table == 'emerging':
-        from mrio_toolbox.extractors.emerging.emerging_extractor import extract_emerging
-        mrio = extract_emerging(year, source, **extraction_kwargs)
-    elif table == 'gtap11':
-        from mrio_toolbox.extractors.gtap11 import extract_gtap11
-        mrio = extract_gtap11(year, source, destination, **extraction_kwargs)
-    else:
-        raise ValueError(f"Unsupported MRIO table: {table}")
-    if preprocessing:
-        mrio.preprocess(**preprocessing)
-    if not saving_kwargs and not destination:
-        return mrio
-    filepath = os.path.join(destination, f"{table}_year{year}.nc")
-    mrio.save(file = filepath, **saving_kwargs)

mrio_toolbox/extractors/figaro/__init__.py

@@ -1,3 +0,0 @@
-"""
-This module contains the figaro extractor and the figaro downloader.
-"""

mrio_toolbox/extractors/figaro/figaro_downloader.py

@@ -1,280 +0,0 @@
-"""
-Download Figaro 25ed from the CIRCABC website.
-
-@author: wirth
-"""
-
-from selenium import webdriver
-from selenium.webdriver.common.by import By
-from selenium.webdriver.firefox.options import Options as FirefoxOptions
-from selenium.webdriver.chrome.options import Options as ChromeOptions
-from selenium.common.exceptions import NoSuchElementException, ElementClickInterceptedException, WebDriverException
-import time
-import os
-import logging 
-
-log = logging.getLogger(__name__)
-
-
-def wait_for_download(file_path, timeout=60):
-    """
-    Wait until the given file has been fully downloaded.
-    
-    Parameters
-    ----------
-    file_path : str
-        The path to the file that is being downloaded.
-    timeout : int
-        Maximum time to wait for the download to complete, in seconds.
-    """
-    folder = os.path.dirname(file_path)
-    end_time = time.time() + timeout
-
-    while time.time() < end_time:
-        # Firefox: file exists, size stops changing
-        if os.path.exists(file_path):
-            size_old = os.path.getsize(file_path)
-            time.sleep(1)
-            size_new = os.path.getsize(file_path)
-            if size_new == size_old:
-                time.sleep(1)  # wait a bit more to ensure download is completed
-                return True
-
-        # Chrome: check for any .crdownload temp file
-        if not any(name.endswith(".crdownload") for name in os.listdir(folder)):
-            if os.path.exists(file_path):
-                time.sleep(1)  # wait a bit more to ensure download is completed
-                return True
-            
-        time.sleep(0.5)
-    raise TimeoutError(f"Download not completed within {timeout} seconds: {file_path}")
-
-def get_driver(destination, headless=True, prefer="chrome"):
-    """
-    Try to get a Selenium driver. Falls back to Chrome if Firefox is not available.
-    
-    Parameters
-    ----------
-    destination : str
-        Download folder for browser.
-    headless : bool
-        Run browser in headless mode.
-    prefer : str
-        Preferred browser: "firefox" or "chrome".
-    """
-
-    def make_firefox():
-        options = FirefoxOptions()
-        if headless: options.add_argument("--headless")
-        options.add_argument("--window-size=1920,1080")
-        options.set_preference("browser.download.folderList", 2)
-        options.set_preference("browser.download.dir", destination)
-        return webdriver.Firefox(options=options)
-
-    def make_chrome():
-        options = ChromeOptions()
-        if headless: options.add_argument("--headless=new")
-        options.add_argument("--window-size=1920,1080")
-        prefs = {"download.default_directory": destination,
-                 "download.prompt_for_download": False,
-                 "download.directory_upgrade": True,
-                 "safebrowsing.enabled": True,
-                 "profile.default_content_setting_values.automatic_downloads": 1}
-        options.add_experimental_option("prefs", prefs)
-        return webdriver.Chrome(options=options)
-
-    tried = []
-    for choice in ([prefer, "chrome", "firefox"] if prefer == "firefox" else [prefer, "firefox", "chrome"]):
-        try:
-            if choice == "firefox":
-                log.info("Trying Firefox driver...")
-                return make_firefox()
-            elif choice == "chrome":
-                log.info("Trying Chrome driver...")
-                return make_chrome()
-        except WebDriverException as e:
-            log.warning(f"{choice.capitalize()} driver failed: {e}")
-            tried.append(choice)
-    raise RuntimeError(f"Could not start any browser driver (tried {tried}). Please install Firefox or Chrome.")
-
-
-def safe_click(driver, by, value, description="element"):
-    """
-    Safely clicks an element on the page, handling exceptions and logging errors.
-    
-    Parameters:
-    -----------
-    driver: WebDriver
-        The Selenium WebDriver instance.
-    by: By
-        The method to locate the element (e.g., By.XPATH, By.CSS_SELECTOR).
-    value: str
-        The value to locate the element.
-    description: str
-        A description of the element for logging purposes.
-        
-    Notes:
-    ------
-    If you want to debug the click, you can set the headless mode to False in the download_figaro function.
-    This will open the browser window and allow you to see what is happening. 
-    In the browser window, you can right-click on the element and select "Inspect" to see the HTML structure.
-    """
-    try:
-        elem = driver.find_element(by, value)
-        driver.execute_script("arguments[0].click();", elem)  # Click with JavaScript to avoid issues with overlays or pop-ups
-        log.info(f"Clicked {description}")
-    except NoSuchElementException:
-        log.error(f"{description} not found: {value}")
-        raise RuntimeError(f"{description} not found: {value}. The download was aborted. Likely the page structure of the CIRCABC website has changed. "
-                           "If you are a developer, try to debug without headless mode. If you are a user, you may want to download the figaro tables manually.")
-    except ElementClickInterceptedException:
-        log.error(f"{description} was obstructed: {value}")
-        raise RuntimeError(f"{description} was obstructed: {value}. The download was aborted. Likely the page structure of the CIRCABC website has changed."
-                           "If you are a developer, try to debug without headless mode. If you are a user, you may want to download the figaro tables manually.")
-    except Exception as e:
-        log.error(f"Error clicking {description}: {e}.")
-        raise RuntimeError("The download was aborted. Likely the page structure of the CIRCABC website has changed."
-                           "If you are a developer, try to debug without headless mode. If you are a user, you may want to download the figaro tables manually.")
-
-
-
-def download_figaro(year, destination, format = 'industry by industry', sut = False, headless = True): 
-    """
-    Downloads the specified format of the EU input-output matrix from Figaro.
-    
-    Parameters:
-    -----------
-    destination: str
-        A path to the folder where the downloaded file will be saved.
-    year: int
-        The year of the data to download.
-    format: str, optional
-        Either 'industry by industry' or 'product by product'.
-    sut: Boolean, optional 
-        If True, also downloads the supply and use tables, otherwise only the input-output matrix.
-    headless: Boolean, optional
-        If True, runs the browser in headless mode (no GUI). Default is True.
-    """
-    
-    # Check if year is valid
-    if not isinstance(year, int) or year < 2010 or year > 2023:
-        raise ValueError("As of August 2025, the Figaro database contains IO tables for the years 2010 to 2023. Please provide a valid year within this range."
-                         "If you are sure that the year 2024 is already available, please update this check accordingly.")
-    
-    # Check if destination exists
-    if not os.path.exists(destination):
-        raise FileNotFoundError(f"The destination folder '{destination}' does not exist. Please create it before downloading.")
-
-    if format == 'industry by industry':
-        format_abbr = "ind-by-ind"
-    elif format == 'product by product':
-        format_abbr = "prod-by-prod"
-    else:
-        raise ValueError("The 'format' parameter must be either 'industry by industry' or 'product by product'.")
-    
-    # Check if files already exist
-    paths = {
-        "io_path" : os.path.join(destination, f"matrix_eu-ic-io_{format_abbr}_25ed_{year}.csv"),
-        "sup_path" : os.path.join(destination, f"matrix_eu-ic-supply_25ed_{year}.csv"),
-        "use_path" : os.path.join(destination, f"matrix_eu-ic-use_25ed_{year}.csv"), 
-        "excel_path" : os.path.join(destination, f"Description_FIGARO_Tables(25ed).xlsx")
-    }
-    url = "https://circabc.europa.eu/ui/group/cec66924-a924-4f91-a0ef-600a0531e3ba/library/0d8bab1e-d159-40b9-9aff-ef8e6d58e24e?p=1&n=10&sort=name_ASC"
-
-    if any(not os.path.exists(p) for p in paths.values()):
-        
-        driver = get_driver(destination, headless=headless, prefer="chrome")
-        driver.get(url)
-        driver.implicitly_wait(5) 
-        
-        if not os.path.exists(paths["excel_path"]):
-            log.info("Downloading the description of the Figaro tables")
-            # Find and click the Excel file
-            safe_click(driver, By.XPATH, "//a[contains(text(), 'Description_FIGARO_Tables(25ed).xlsx')]", "Excel file link")
-            
-            # Find and click the download button
-            safe_click(driver, By.CSS_SELECTOR, ".download", "Download button for Excel file")
-            
-            # Wait for the download to complete
-            wait_for_download(paths["excel_path"])
-            
-            # Go back to the main page
-            driver.get(url)
-        else:
-            log.info(f"The description of the Figaro tables is already in the folder '{destination}', skipping download")
-        
-        if not os.path.exists(paths["io_path"]):
-            log.info(f"Downloading IO table for format '{format_abbr}' and year '{year}'")
-            # Find and click the desired format (ixi or pxp)
-            safe_click(driver, By.XPATH, f"//a[contains(text(), '{format}')]", f"format '{format}' link")
-
-            # Find and click the CSV matrix format
-            safe_click(driver, By.XPATH, "//a[contains(text(), 'CSV matrix format')]", "CSV matrix format link")
-
-            # Click for the second page if year > 2019
-            if year > 2019:
-                time.sleep(0.5) # we need an explicit wait here, because the element is found before it is clickable
-                safe_click(driver, By.CLASS_NAME, "next-page", "Next page button")
-
-            # Find and click the desired year
-            safe_click(driver, By.XPATH, f"//a[contains(text(), '_{year}.csv')]", f"Year '{year}' link")
-
-            # Find and click the download button
-            safe_click(driver, By.CSS_SELECTOR, ".download", "Download button for IO table")
-
-            # Wait for the download to complete
-            wait_for_download(paths["io_path"])
-            driver.get(url)
-        else:
-            log.info(f"The IO tables for format '{format}' and year '{year}' are already in the folder '{destination}', skipping download")
-
-        if sut == True: 
-            if not os.path.exists(paths["sup_path"]):
-                log.info(f"Downloading supply table for year '{year}'")
-                # Find and click the supply table
-                safe_click(driver, By.XPATH, f"//a[contains(text(), 'Supply tables')]", "Supply tables link")
-
-                # Find and click the CSV matrix format
-                safe_click(driver, By.XPATH, "//a[contains(text(), 'CSV matrix format')]", "CSV matrix format link")
-
-                # Click for the second page if year > 2019
-                if year > 2019:
-                    time.sleep(0.5) 
-                    safe_click(driver, By.CLASS_NAME, "next-page", "Next page button")
-
-                # Find and click the desired year
-                safe_click(driver, By.XPATH, f"//a[contains(text(), '_{year}.csv')]", f"Year '{year}' link")
-
-                # Find and click the download button
-                safe_click(driver, By.CSS_SELECTOR, ".download", "Download button for supply table")
-
-                wait_for_download(paths["sup_path"])
-                driver.get(url)
-            else: 
-                log.info(f"The use tables for year '{year}' are already in the folder '{destination}', skipping download")
-
-            if not os.path.exists(paths["use_path"]):
-                log.info(f"Downloading supply table for year '{year}'")
-
-                # Find and click the supply table
-                safe_click(driver, By.XPATH, f"//a[contains(text(), 'Use tables')]", "Use tables link")
-
-                # Find and click the CSV matrix format
-                safe_click(driver, By.XPATH, "//a[contains(text(), 'CSV matrix format')]", "CSV matrix format link")
-
-                # Click for the second page if year > 2019
-                if year > 2019:
-                    time.sleep(0.5)
-                    safe_click(driver, By.CLASS_NAME, "next-page", "Next page button")
-
-                # Find and click the desired year
-                safe_click(driver, By.XPATH, f"//a[contains(text(), '_{year}.csv')]", f"Year '{year}' link")
-
-                # Find and click the download button
-                safe_click(driver, By.CSS_SELECTOR, ".download", "Download button for use table")
-                wait_for_download(paths["use_path"])
-            else:
-                log.info(f"The use tables for year '{year}' are already in the folder '{destination}', skipping download")
-        driver.quit()
-    else:
-        log.info(f"The files for format '{format_abbr}' and year '{year}' are already in the folder '{destination}', skipping download")

mrio_toolbox/extractors/figaro/figaro_extractor.py

@@ -1,187 +0,0 @@
-"""
-Load and convert Figaro MRIO files. 
-
-Supports Figaro inter industry IO, supply and use tables in csv matrix format
-https://ec.europa.eu/eurostat/web/esa-supply-use-input-tables/database#Input-output%20tables%20industry%20by%20industry
-
-The extractor loads the IO table and if available the supply and use tables.
-
-@author: wirth
-"""
-
-import os
-import logging
-import pandas as pd
-
-from mrio_toolbox import MRIO
-from mrio_toolbox.utils.savers._to_nc import save_to_nc
-
-log = logging.getLogger(__name__)
-
-def extract_figaro(year, source, format = 'industry by industry', sut = "none", edition=25):
-    """
-    Extract FIGARO data. 
-
-    Loads FIGARO tables and labels and store them as NetCDF for further use with 
-    the mrio_toolbox library. Currently the extractor does not support emission 
-    satellite accounts (I couldn't find them on the figaro website).
-    
-    Put all tables  as well as the 'Description_FIGARO_Tables({edition}ed).xlsx' file 
-    in the same source folder. 
-
-    Parameters
-    ----------
-    year : str
-        Data year to load.
-    source : path-like
-        Path to folder where raw data is stored
-    format : str, optional
-       Either 'industry by industry' or 'product by product'.
-    sut : str, optional
-        Supply and use tables to load, by default "none".
-        Available options are "none", "supply", "use" or "both".
-    edition : int, optional
-        Edition of the FIGARO tables, by default 25. The alternative is 24.
-    """
-    
-    if format == 'industry by industry':
-        format_abbr = "ind-by-ind"
-    elif format == 'product by product':
-        format_abbr = "prod-by-prod"
-    else:
-        raise ValueError("The 'format' parameter must be either 'industry by industry' or 'product by product'.")
-        
-    log.info(f"Extracting FIGARO IO table for year {year}, load IO table...")
-    raw = pd.read_csv(os.path.join(source, f"matrix_eu-ic-io_{format_abbr}_{edition}ed_{year}.csv"), dtype = str)
-    log.info("Loaded IO table")
-
-    if sut in ["supply", "both"]:
-        log.info(f"Check if supply table is available for year {year}...")
-        if os.path.isfile(os.path.join(source, f"matrix_eu-ic-supply_{edition}ed_{year}.csv")):
-            log.info("Supply table found, loading...")
-            raw_supply = pd.read_csv(os.path.join(source, f"matrix_eu-ic-supply_{edition}ed_{year}.csv"), dtype = str)
-            log.info("Loaded supply table")
-    
-    if sut in ["use", "both"]:    
-        log.info(f"Check if use table is available for year {year}...")
-        if os.path.isfile(os.path.join(source, f"matrix_eu-ic-use_{edition}ed_{year}.csv")):
-            log.info("Use table found, loading...")
-            raw_use = pd.read_csv(os.path.join(source, f"matrix_eu-ic-use_{edition}ed_{year}.csv"), dtype = str)
-            log.info("Loaded use table")
-        
-    log.info("Extracting labels...")
-    if edition == 24:
-        df = pd.read_excel(os.path.join(source, f"Description_FIGARO_Tables({edition}ed).xlsx"), header=5, sheet_name = "Prod, Ind & Accounting items").dropna(axis=1, how='all')
-    elif edition == 25:
-        df = pd.read_excel(os.path.join(source, f"Description_FIGARO_Tables({edition}ed).xlsx"), header=3, sheet_name = "Prod, Ind & Accounting items").dropna(axis=1, how='all')
-    else: 
-        ValueError(f"Edition {edition} not yet supported. Please use edition 24 or 25.")
-        
-    # Countries are not in the correct order in the excel sheet, so get countries from raw data
-    column_labs = raw.columns[1:]
-    countries = column_labs.str.split("_").str[0]
-    countries = list(dict.fromkeys(countries))
-
-    # Get other labels from excel sheet
-    sectors = df["Label.1"].tolist()
-    cpa_labs = df["Label"].tolist()
-    va_labs = df["Label.2"].dropna().tolist()
-    y_labs = df["Label.3"].dropna().tolist()
-
-    labels = {
-        "countries": countries,
-        "sectors": sectors,
-        "y_labs": y_labs,
-        "va_labs": va_labs 
-    }
-    c, s, y, va  = len(countries), len(sectors), len(y_labs), len(va_labs)
-    if 'raw_supply' in locals() or 'raw_use' in locals():
-        labels["cpa_labs"] = cpa_labs
-        cpa =  len(cpa_labs)
-    log.info("Labels extracted")
-    
-    log.info("Extracting parts from raw data...")
-    raw = raw.iloc[:, 1:].astype(float).to_numpy()
-
-    tables = {}
-    tables["t"] = raw[:c*s, :c*s]
-    tables["y"] = raw[:c*s, c*s:(c*s+c*y)]
-    tables["va"] = raw[c*s:(c*s+c*va), :c*s]
-    tables["vay"] = raw[c*s:(c*s+c*va), c*s:(c*s+c*y)]
-    log.info("Extracted parts from raw data")
-
-    # Treat supply table if available
-    if 'raw_supply' in locals():
-        log.info("Extracting supply table...")
-        raw_supply = raw_supply.iloc[:, 1:].astype(float).to_numpy()
-        tables["sup"] = raw_supply[:c*cpa, :c*s]
-        log.info("Extracted supply table")
-    else:
-        log.info("No supply table found, skipping...")
-        
-    # Treat use table if available
-    if 'raw_use' in locals():
-        log.info("Extracting use table...")
-        raw_use = raw_use.iloc[:, 1:].astype(float).to_numpy()
-        tables["use_t"] = raw_use[:c*cpa, :c*s]
-        tables["use_y"] = raw_use[:c*cpa, c*s:c*s + c*y]
-        tables["use_va"] = raw_use[c*cpa:c*cpa+c*va, :c*s]
-        tables["use_vay"] = raw_use[c*cpa:(c*cpa+c*va), c*s:(c*s+c*y)]
-        log.info("Extracted use table")
-    else:
-        log.info("No use table found, skipping...")
-
-    # Assemble mrio object
-    log.info("Building MRIO object...")
-    m = MRIO()
-    m.add_dimensions(labels)
-    log.info("Building MRIO objects from parts containing labels and tables...")
-    m.parts["t"] = m.new_part(name="t",
-        data= tables["t"],
-        dimensions = [["countries","sectors"],["countries", "sectors"]])
-    log.info("t part added")
-    m.parts["y"] = m.new_part(name="y",
-        data= tables["y"],
-        dimensions = [["countries","sectors"],["countries", "y_labs"]])
-    log.info("y part added")
-    m.parts["va"] = m.new_part(name="va",
-        data= tables["va"],
-        dimensions = ["va_labs",["countries", "sectors"]])
-    log.info("va part added")
-    m.parts["vay"] = m.new_part(name="vay",
-        data= tables["vay"],
-        dimensions = ["va_labs",["countries", "y_labs"]])
-    log.info("vay part added")
-    if 'sup' in tables:
-        m.parts["sup"] = m.new_part(name="sup",
-            data= tables["sup"],
-            dimensions = [["countries","cpa_labs"],["countries", "sectors"]])
-        log.info("sup part added")
-    if 'use_t' in tables:
-        m.parts["use_t"] = m.new_part(name="use_t",
-            data= tables["use_t"],
-            dimensions = [["countries","cpa_labs"],["countries", "sectors"]])
-        log.info("use_t part added")
-        m.parts["use_y"] = m.new_part(name="use_y",
-            data= tables["use_y"],
-            dimensions = [["countries","cpa_labs"],["countries", "y_labs"]])
-        log.info("use_y part added")
-        m.parts["use_va"] = m.new_part(name="use_va",
-            data= tables["use_va"],
-            dimensions = ["va_labs",["countries", "sectors"]])
-        log.info("use_va part added")
-        m.parts["use_vay"] = m.new_part(name="use_vay",
-            data= tables["use_vay"],
-            dimensions = ["va_labs",["countries", "y_labs"]])
-        log.info("use_vay part added")
-    log.info("MRIO object built")
-    
-    # Add metadata
-    log.info("Adding metadata to MRIO object...")
-    m.metadata["table"] = "figaro"
-    m.metadata["edition"] = edition
-    m.metadata["year"] = year
-    m.metadata["format"] = format
-    m.metadata["sut"] = sut
-    m.name = f"figaro_{year}_{format}"
-    return m

mrio_toolbox/extractors/gloria/__init__.py

@@ -1,3 +0,0 @@
-"""
-This module contains the gloria extractor
-"""