pr-imports 0.2.0__tar.gz

pr_imports-0.2.0/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.3
+Name: pr-imports
+Version: 0.2.0
+Summary: Add your description here
+Requires-Dist: censusforge>=1.2.0
+Requires-Dist: duckdb>=1.4.5
+Requires-Dist: jp-tools>=0.5.8
+Requires-Dist: polars>=1.36.1
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+

pr_imports-0.2.0/pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "pr-imports"
+version = "0.2.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "censusforge>=1.2.0",
+    "duckdb>=1.4.5",
+    "jp-tools>=0.5.8",
+    "polars>=1.36.1",
+]
+
+[project.scripts]
+pr-imports = "pr_imports:main"
+
+[build-system]
+requires = ["uv_build>=0.11.30,<0.12.0"]
+build-backend = "uv_build"

pr_imports-0.2.0/src/pr_imports/__init__.py

@@ -0,0 +1,7 @@
+from .utils import TradeUtils
+#from .jp_imports import JPTrade
+
+from importlib.metadata import version
+
+__version__ = version("pr_imports")
+__all__ = ["TradeUtils", "JPTrade"]

pr_imports-0.2.0/src/pr_imports/jp_imports.py

@@ -0,0 +1,359 @@
+import importlib.resources as resources
+from datetime import datetime as dt
+
+import polars as pl
+
+from .utils import TradeUtils
+
+LEVEL_GROUPS = {
+    "total": [],
+    "naics": ["naics"],
+    "hts": ["hts_code"],
+    "country": ["country"],
+}
+
+TIME_GROUPS = {
+    "yearly": ["year"],
+    "fiscal": ["fiscal_year"],
+    "qtr": ["year", "qtr"],
+    "monthly": ["year", "month"],
+}
+
+
+class JPTrade(TradeUtils):
+    """
+    Data processing class for the various data sources in DataPull.
+    Optimized to dynamically aggregate metrics using a configuration-driven design.
+    """
+
+    def __init__(
+        self,
+        saving_dir: str = "data/",
+        log_file: str = "data.log",
+    ):
+        """
+        Initialize the DataProcess class.
+        """
+        super().__init__(saving_dir, log_file)
+        self.agr_file = str(
+            resources.files("jp_imports").joinpath("resources/code_agr.json")
+        )
+
+    def process_int_jp(
+        self,
+        level: str,
+        time_frame: str,
+        datetime: str = "",
+        agriculture_filter: bool = False,
+        level_filter: str = "",
+    ) -> pl.DataFrame:
+        """
+        Process the data for Puerto Rico Statistics Institute provided to JP.
+        """
+        df = self.pull_int_jp()
+
+        if agriculture_filter:
+            df = df.filter(pl.col("agri_prod") == 1)
+
+        # Unified taxonomy filtering
+        if level in ["hts", "naics", "country"]:
+            level_map = {"hts": "hts_code", "naics": "naics", "country": "country"}
+            filter_col = level_map[level]
+
+            df = df.filter(pl.col(filter_col).str.starts_with(level_filter))
+            if df.is_empty():
+                raise ValueError(f"Invalid {level.upper()} code: {level_filter}")
+
+        # Streamlined date routing
+        if datetime:
+            times = datetime.split("+")
+            if len(times) == 2:
+                start_date = dt.strptime(times[0], "%Y-%m-%d")
+                end_date = dt.strptime(times[1], "%Y-%m-%d")
+                df = df.filter(pl.col("date").is_between(start_date, end_date))
+            elif len(times) == 1:
+                df = df.filter(pl.col("date").dt.year() == int(datetime))
+            else:
+                raise ValueError(
+                    'Invalid time format. Use "date" or "start_date+end_date"'
+                )
+
+        df = self.conversion(df)
+
+        return self.process_data(time_frame=time_frame, level=level, base=df)
+
+    def process_data(
+        self, time_frame: str, level: str, base: pl.DataFrame
+    ) -> pl.DataFrame:
+        """
+        Process the data based on dynamic groupings without logic duplication.
+        """
+        if time_frame not in TIME_GROUPS or level not in LEVEL_GROUPS:
+            raise ValueError(
+                f"Invalid combination layout requested: {time_frame=}, {level=}"
+            )
+
+        group_by_keys = TIME_GROUPS[time_frame] + LEVEL_GROUPS[level]
+
+        df = self.filter_data(base, group_by_keys)
+
+        coalesce_exprs = []
+        for col in group_by_keys:
+            right_col = f"{col}_right"
+            if right_col in df.columns:
+                coalesce_exprs.append(
+                    pl.when(pl.col(col).is_null())
+                    .then(pl.col(right_col))
+                    .otherwise(pl.col(col))
+                    .alias(col)
+                )
+
+        if coalesce_exprs:
+            df = df.with_columns(coalesce_exprs)
+
+        df = df.drop([c for c in df.columns if c.endswith("_right")])
+
+        target_metrics = ["imports", "exports", "imports_qty", "exports_qty"]
+        df = (
+            df.with_columns(pl.col(target_metrics).fill_null(0))
+            .sort(group_by_keys)
+            .with_columns(
+                net_exports=pl.col("exports") - pl.col("imports"),
+                net_qty=pl.col("exports_qty") - pl.col("imports_qty"),
+            )
+        )
+
+        return df
+
+    def process_price(self, agriculture_filter: bool = False) -> pl.DataFrame:
+        """
+        Calculate rolling price statistics on top of monthly item structures.
+        """
+        df = self.process_int_jp(
+            time_frame="monthly", level="hts", agriculture_filter=agriculture_filter
+        )
+        df = df.with_columns(pl.col("imports_qty", "exports_qty").replace(0, 1))
+        df = df.with_columns(hs4=pl.col("hts_code").str.slice(0, 4))
+
+        df = df.group_by(pl.col("hs4", "month", "year")).agg(
+            pl.col("imports").sum().alias("imports"),
+            pl.col("exports").sum().alias("exports"),
+            pl.col("imports_qty").sum().alias("imports_qty"),
+            pl.col("exports_qty").sum().alias("exports_qty"),
+        )
+
+        df = df.with_columns(
+            price_imports=pl.col("imports") / pl.col("imports_qty"),
+            price_exports=pl.col("exports") / pl.col("exports_qty"),
+        )
+
+        df = df.with_columns(date=pl.datetime(pl.col("year"), pl.col("month"), 1)).sort(
+            "date"
+        )
+
+        # Rolling Statistical Engine
+        results = df.with_columns(
+            pl.col("price_imports")
+            .rolling_mean(window_size=3, min_samples=1)
+            .over("hs4")
+            .alias("moving_price_imports"),
+            pl.col("price_exports")
+            .rolling_mean(window_size=3, min_samples=1)
+            .over("hs4")
+            .alias("moving_price_exports"),
+            pl.col("price_imports")
+            .rolling_std(window_size=3, min_samples=1)
+            .over("hs4")
+            .alias("moving_price_imports_std"),
+            pl.col("price_exports")
+            .rolling_std(window_size=3, min_samples=1)
+            .over("hs4")
+            .alias("moving_price_exports_std"),
+        ).with_columns(
+            pl.col("moving_price_imports")
+            .rank("ordinal")
+            .over("date")
+            .alias("rank_imports")
+            .cast(pl.Int64),
+            pl.col("moving_price_exports")
+            .rank("ordinal")
+            .over("date")
+            .alias("rank_exports")
+            .cast(pl.Int64),
+            upper_band_imports=pl.col("moving_price_imports")
+            + 2 * pl.col("moving_price_imports_std"),
+            lower_band_imports=pl.col("moving_price_imports")
+            - 2 * pl.col("moving_price_imports_std"),
+            upper_band_exports=pl.col("moving_price_exports")
+            + 2 * pl.col("moving_price_exports_std"),
+            lower_band_exports=pl.col("moving_price_exports")
+            - 2 * pl.col("moving_price_exports_std"),
+        )
+
+        results = df.join(results, on=["date", "hs4"], how="left", validate="1:1")
+
+        # Clean up overlap column names after explicit join validation
+        results = results.with_columns(
+            year=pl.when(pl.col("year").is_null())
+            .then(pl.col("year_right"))
+            .otherwise(pl.col("year")),
+            month=pl.when(pl.col("month").is_null())
+            .then(pl.col("month_right"))
+            .otherwise(pl.col("month")),
+            imports=pl.when(pl.col("imports").is_null())
+            .then(pl.col("imports_right"))
+            .otherwise(pl.col("imports")),
+            exports=pl.when(pl.col("exports").is_null())
+            .then(pl.col("exports_right"))
+            .otherwise(pl.col("exports")),
+            price_imports=pl.when(pl.col("price_imports").is_null())
+            .then(pl.col("price_imports_right"))
+            .otherwise(pl.col("price_imports")),
+            price_exports=pl.when(pl.col("price_exports").is_null())
+            .then(pl.col("price_exports_right"))
+            .otherwise(pl.col("price_exports")),
+            imports_qty=pl.when(pl.col("imports_qty").is_null())
+            .then(pl.col("exports_qty_right"))
+            .otherwise(pl.col("imports_qty")),
+            exports_qty=pl.when(pl.col("exports_qty").is_null())
+            .then(pl.col("exports_qty"))
+            .otherwise(pl.col("exports_qty")),
+        ).drop([c for c in results.columns if c.endswith("_right")])
+
+        # Track month shifts for sequential year-over-year deltas
+        results = results.with_columns(
+            pl.col("moving_price_imports")
+            .pct_change()
+            .over("date", "hs4")
+            .alias("pct_change_imports")
+        ).sort(by=["date", "hs4"])
+
+        results = results.with_columns(
+            pl.when(pl.col("date").dt.year() > 1)
+            .then(pl.col("moving_price_imports").shift(12))
+            .otherwise(None)
+            .alias("prev_year_imports"),
+            pl.when(pl.col("date").dt.year() > 1)
+            .then(pl.col("moving_price_exports").shift(12))
+            .otherwise(None)
+            .alias("prev_year_exports"),
+            pl.when(pl.col("date").dt.year() > 1)
+            .then(pl.col("rank_imports").shift(12))
+            .otherwise(None)
+            .alias("prev_year_rank_imports"),
+            pl.when(pl.col("date").dt.year() > 1)
+            .then(pl.col("rank_exports").shift(12))
+            .otherwise(None)
+            .alias("prev_year_rank_exports"),
+        )
+
+        results = results.with_columns(
+            (
+                (pl.col("moving_price_imports") - pl.col("prev_year_imports"))
+                / pl.col("prev_year_imports")
+            ).alias("pct_change_imports_year_over_year"),
+            (
+                (pl.col("moving_price_exports") - pl.col("prev_year_exports"))
+                / pl.col("prev_year_exports")
+            ).alias("pct_change_exports_year_over_year"),
+            (pl.col("rank_imports") - pl.col("prev_year_rank_imports")).alias(
+                "rank_imports_change_year_over_year"
+            ),
+            (
+                pl.col("rank_exports").cast(pl.Int64)
+                - pl.col("prev_year_rank_exports").cast(pl.Int64)
+            ).alias("rank_exports_change_year_over_year"),
+        ).sort(by=["date", "hs4"])
+
+        return results
+
+    def conversion(self, df: pl.DataFrame) -> pl.DataFrame:
+        """
+        Convert the data to the correct units (kg).
+
+        Parameters
+        ----------
+        df: pl.LazyFrame
+            Data to convert.
+
+        Returns
+        -------
+        pl.LazyFrame
+            Converted data.
+        """
+
+        df = df.with_columns(pl.col("qty_1", "qty_2").fill_null(strategy="zero"))
+        df = df.with_columns(
+            conv_1=pl.when(pl.col("unit_1").str.to_lowercase() == "kg")
+            .then(pl.col("qty_1") * 1)
+            .when(pl.col("unit_1").str.to_lowercase() == "l")
+            .then(pl.col("qty_1") * 1)
+            .when(pl.col("unit_1").str.to_lowercase() == "doz")
+            .then(pl.col("qty_1") / 0.756)
+            .when(pl.col("unit_1").str.to_lowercase() == "m3")
+            .then(pl.col("qty_1") * 1560)
+            .when(pl.col("unit_1").str.to_lowercase() == "t")
+            .then(pl.col("qty_1") * 907.185)
+            .when(pl.col("unit_1").str.to_lowercase() == "kts")
+            .then(pl.col("qty_1") * 1)
+            .when(pl.col("unit_1").str.to_lowercase() == "pfl")
+            .then(pl.col("qty_1") * 0.789)
+            .when(pl.col("unit_1").str.to_lowercase() == "gm")
+            .then(pl.col("qty_1") * 1000)
+            .otherwise(pl.col("qty_1")),
+            conv_2=pl.when(pl.col("unit_2").str.to_lowercase() == "kg")
+            .then(pl.col("qty_2") * 1)
+            .when(pl.col("unit_2").str.to_lowercase() == "l")
+            .then(pl.col("qty_2") * 1)
+            .when(pl.col("unit_2").str.to_lowercase() == "doz")
+            .then(pl.col("qty_2") / 0.756)
+            .when(pl.col("unit_2").str.to_lowercase() == "m3")
+            .then(pl.col("qty_2") * 1560)
+            .when(pl.col("unit_2").str.to_lowercase() == "t")
+            .then(pl.col("qty_2") * 907.185)
+            .when(pl.col("unit_2").str.to_lowercase() == "kts")
+            .then(pl.col("qty_2") * 1)
+            .when(pl.col("unit_2").str.to_lowercase() == "pfl")
+            .then(pl.col("qty_2") * 0.789)
+            .when(pl.col("unit_2").str.to_lowercase() == "gm")
+            .then(pl.col("qty_2") * 1000)
+            .otherwise(pl.col("qty_2")),
+            qtr=pl.when(
+                (pl.col("date").dt.month() >= 1) & (pl.col("date").dt.month() <= 3)
+            )
+            .then(1)
+            .when((pl.col("date").dt.month() >= 4) & (pl.col("date").dt.month() <= 6))
+            .then(2)
+            .when((pl.col("date").dt.month() >= 7) & (pl.col("date").dt.month() <= 9))
+            .then(3)
+            .when((pl.col("date").dt.month() >= 10) & (pl.col("date").dt.month() <= 12))
+            .then(4),
+            fiscal_year=pl.when(pl.col("date").dt.month() > 6)
+            .then(pl.col("date").dt.year() + 1)
+            .otherwise(pl.col("date").dt.year())
+            .alias("fiscal_year"),
+            month=pl.col("date").dt.month(),
+            year=pl.col("date").dt.year(),
+        ).with_columns(qty=pl.col("conv_1") + pl.col("conv_2"))
+        return df
+
+    def filter_data(self, df: pl.DataFrame, filter: list) -> pl.DataFrame:
+        """
+        Filter the data based on the filter list.
+        """
+        df = df.filter(pl.col("hts_code").is_not_null())
+        imports = (
+            df.filter(pl.col("trade_id") == 1)
+            .group_by(filter)
+            .agg(pl.sum("data", "qty"))
+            .sort(filter)
+            .rename({"data": "imports", "qty": "imports_qty"})
+        )
+        exports = (
+            df.filter(pl.col("trade_id") == 2)
+            .group_by(filter)
+            .agg(pl.sum("data", "qty"))
+            .sort(filter)
+            .rename({"data": "exports", "qty": "exports_qty"})
+        )
+        return imports.join(exports, on=filter, how="full")

pr_imports-0.2.0/src/pr_imports/resources/code_agr.json

@@ -0,0 +1,131 @@
+{
+  "0": 405,
+  "1": 1514,
+  "2": 1206,
+  "3": 2105,
+  "4": 208,
+  "5": 302,
+  "6": 2104,
+  "7": 802,
+  "8": 301,
+  "9": 1504,
+  "10": 1508,
+  "11": 804,
+  "12": 307,
+  "13": 807,
+  "14": 708,
+  "15": 1105,
+  "16": 404,
+  "17": 1103,
+  "18": 106,
+  "19": 2102,
+  "20": 908,
+  "21": 906,
+  "22": 1511,
+  "23": 410,
+  "24": 2006,
+  "25": 1106,
+  "26": 703,
+  "27": 209,
+  "28": 704,
+  "29": 2009,
+  "30": 713,
+  "31": 201,
+  "32": 910,
+  "33": 1701,
+  "34": 1517,
+  "35": 710,
+  "36": 408,
+  "37": 210,
+  "38": 1104,
+  "39": 1209,
+  "40": 206,
+  "41": 403,
+  "42": 1516,
+  "43": 1101,
+  "44": 1605,
+  "45": 803,
+  "46": 706,
+  "47": 801,
+  "48": 1509,
+  "49": 701,
+  "50": 1902,
+  "51": 907,
+  "52": 1806,
+  "53": 714,
+  "54": 2103,
+  "55": 2008,
+  "56": 1005,
+  "57": 1302,
+  "58": 2007,
+  "59": 909,
+  "60": 1207,
+  "61": 810,
+  "62": 1602,
+  "63": 1202,
+  "64": 811,
+  "65": 1502,
+  "66": 2004,
+  "67": 1901,
+  "68": 306,
+  "69": 202,
+  "70": 1601,
+  "71": 1518,
+  "72": 1109,
+  "73": 709,
+  "74": 303,
+  "75": 712,
+  "76": 1805,
+  "77": 707,
+  "78": 2005,
+  "79": 1905,
+  "80": 406,
+  "81": 904,
+  "82": 1214,
+  "83": 806,
+  "84": 207,
+  "85": 808,
+  "86": 1507,
+  "87": 2001,
+  "88": 705,
+  "89": 2101,
+  "90": 1513,
+  "91": 304,
+  "92": 1704,
+  "93": 203,
+  "94": 1212,
+  "95": 407,
+  "96": 401,
+  "97": 1804,
+  "98": 813,
+  "99": 1102,
+  "100": 1211,
+  "101": 1702,
+  "102": 1904,
+  "103": 1008,
+  "104": 1604,
+  "105": 402,
+  "106": 204,
+  "107": 1107,
+  "108": 305,
+  "109": 1108,
+  "110": 2002,
+  "111": 409,
+  "112": 805,
+  "113": 809,
+  "114": 2106,
+  "115": 901,
+  "116": 1510,
+  "117": 1512,
+  "118": 2003,
+  "119": 1515,
+  "120": 902,
+  "121": 1301,
+  "122": 1006,
+  "123": 1501,
+  "124": 1004,
+  "125": 711,
+  "126": 702,
+  "127": 905,
+  "128": 1001
+}

pr_imports-0.2.0/src/pr_imports/utils.py

@@ -0,0 +1,411 @@
+import hashlib
+import importlib.resources as resources
+import logging
+import shutil
+import subprocess
+import tempfile
+import zipfile
+from pathlib import Path
+
+import duckdb
+import polars as pl
+from CensusForge import CensusAPI
+from jp_tools import download
+
+
+class TradeUtils:
+    """
+    This class pulls data from the CENSUS and the Puerto Rico Institute of Statistics
+
+    """
+
+    def __init__(
+        self,
+        saving_dir: str = "data/",
+        log_file: str = "data.log",
+    ):
+        self.saving_dir = Path(saving_dir)
+        self.conn = duckdb.connect()
+
+        logging.basicConfig(
+            level=logging.INFO,
+            format="%(asctime)s - %(levelname)s - %(message)s",
+            datefmt="%d-%b-%y %H:%M:%S",
+            filename=log_file,
+        )
+
+    def pull_int_jp(self, update: bool = False) -> pl.DataFrame:
+        """
+        Pulls data from the Puerto Rico Institute of Statistics used by the JP.
+            Saved them in the raw directory as parquet files.
+
+        Parameters
+        ----------
+        None
+
+        Returns
+        -------
+        None
+        """
+        output_dir = (
+            self.saving_dir
+            / "raw"       
+        )
+
+        output_dir.mkdir(parents=True, exist_ok=True)
+        file_path = output_dir / "jp_data"
+
+        
+        file_path = self.saving_dir / "raw" / "jp_data.parquet"
+        name_hash = hashlib.md5(str(file_path).encode()).hexdigest()
+        temp_csv = Path(tempfile.gettempdir()) / f"{name_hash}.csv"
+
+        if not file_path.exists() or update:
+
+            headers = {
+                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            }
+
+            download(
+                url="https://datos.estadisticas.pr/dataset/027ddbe1-c51c-46bf-aec3-a62d5d7e8539/resource/b8367825-a3de-41cf-8794-e42c10987b6f/download/ftrade_all_iepr.csv",
+                filename=str(temp_csv),
+                verify=False,
+                timeout=120,
+                headers=headers,
+            )
+            df = pl.read_csv(temp_csv, ignore_errors=True)
+
+            agri_prod = pl.read_json(
+                str(resources.files("pr_imports").joinpath("resources/code_agr.json"))
+            ).transpose()
+            agri_prod = (
+                agri_prod.with_columns(pl.nth(0).cast(pl.String).str.zfill(4))
+                .to_series()
+                .to_list()
+            )
+            df = df.rename({col: col.lower() for col in df.collect_schema().names()})
+            df = df.with_columns(
+                date=pl.col("year").cast(pl.String)
+                + "-"
+                + pl.col("month").cast(pl.String)
+                + "-01",
+                unit_1=pl.col("unit_1").str.to_lowercase(),
+                unit_2=pl.col("unit_2").str.to_lowercase(),
+                hts_code=pl.col("commodity_code")
+                .cast(pl.String)
+                .str.zfill(10)
+                .str.replace("'", ""),
+                trade_id=pl.when(pl.col("trade") == "i").then(1).otherwise(2),
+            )
+
+            df = df.with_columns(pl.col("date").cast(pl.Date))
+
+            df = df.with_columns(
+                agri_prod=pl.when(pl.col("hts_code").is_in(agri_prod))
+                .then(1)
+                .otherwise(0)
+            )
+            df = df.with_columns(
+                sitc=pl.when(pl.col("sitc_short_desc").str.starts_with("Civilian"))
+                .then(9998)
+                .when(pl.col("sitc_short_desc").str.starts_with("-"))
+                .then(9999)
+                .otherwise(pl.col("sitc"))
+            )
+            df = df.filter(pl.col("hts_code").is_not_null())
+            df = df.select(
+                pl.col(
+                    "date",
+                    "country",
+                    "trade_id",
+                    "agri_prod",
+                    "hts_code",
+                    "hts_desc",
+                    "data",
+                    "qty_1",
+                    "unit_1",
+                    "qty_2",
+                    "unit_2",
+                    "sitc",
+                    "naics",
+                )
+            )
+
+            df.write_parquet(file_path)
+
+        return pl.read_parquet(file_path)
+
+    def pull_int_org(self, update: bool = False, export: bool = True) -> pl.DataFrame:
+        """
+        Downloads, extracts, and unifies organizational data from the Puerto Rico
+        Institute of Statistics.
+
+        The process involves a nested extraction: a parent ZIP contains multiple
+        inner ZIPs, which in turn contain the target CSV files. Due to known
+        formatting issues with the source ZIP files, the method employs a fallback
+        to the '7z' system utility if the standard zipfile library fails.
+
+        Args:
+            update (bool): If True, ignores existing local parquet files and
+                re-downloads/processes the data. Defaults to False.
+
+        Returns:
+            pl.DataFrame: A Polars DataFrame containing the unified data from all
+                extracted CSVs.
+
+        Note:
+            Requires '7z' to be installed in the system PATH as a fallback for
+            malformed archive handling.
+        """
+
+        # Define Output file and hash
+        parquet_export_path = self.saving_dir / "raw" / "jp_org_exports.parquet"
+        parquet_import_path = self.saving_dir / "raw" / "jp_org_imports.parquet"
+        name_hash = hashlib.md5(str(parquet_export_path).encode()).hexdigest()
+
+        # Define Temporary directories
+        temp_zip = Path(tempfile.gettempdir()) / f"jp_org_zip_{name_hash}.zip"
+        stage1_dir = Path(tempfile.gettempdir()) / f"stage1_{name_hash}"
+        stage2_dir = Path(tempfile.gettempdir()) / f"stage2_{name_hash}"
+
+        if not parquet_export_path.exists() or update:
+
+            download(
+                url="http://apps.estadisticas.pr/iepr/LinkClick.aspx?fileticket=JVyYmIHqbqc%3d&tabid=284&mid=244930",
+                filename=(str(temp_zip)),
+            )
+
+            # Stage 1: Extract all Parent zip file
+            # WARNING: Added the use of subprocess due to mal formated zip from estadisticas.pr.gov
+
+            stage1_dir.mkdir(parents=True, exist_ok=True)
+
+            try:
+                with zipfile.ZipFile(temp_zip, "r") as zip_ref:
+                    zip_ref.extractall(stage1_dir)
+
+            except zipfile.BadZipFile:
+                print("Warning: Used 7z subprocess due to mall formated file")
+                subprocess.run(
+                    ["7z", "x", str(temp_zip), f"-o{stage1_dir}", "-y"],
+                    capture_output=True,
+                )
+
+            stage2_dir.mkdir(parents=True, exist_ok=True)
+            inner_zips = list(stage1_dir.rglob("*.zip"))
+
+            for izip in inner_zips:
+                try:
+                    with zipfile.ZipFile(izip, "r") as zip_ref:
+                        # Extracting to our second temp directory
+                        zip_ref.extractall(stage2_dir)
+                except zipfile.BadZipFile:
+                    print(
+                        f"Warning: {izip.name} was also malformed. Switching back to 7-zip for this file."
+                    )
+                    subprocess.run(
+                        ["7z", "x", str(izip), f"-o{stage2_dir}", "-y"],
+                        capture_output=True,
+                    )
+
+            # Stage 3: unify data into a single file
+            csv_files = list(stage2_dir.rglob("*.csv"))
+            if not csv_files:
+                raise FileNotFoundError(
+                    "Extraction successful, but no CSV files were found to process."
+                )
+
+            df_list = [pl.read_csv(f, ignore_errors=True) for f in csv_files]
+            df = pl.concat(df_list)
+            df = df.with_columns(
+                country=pl.col("country").str.to_lowercase(),
+                hts=pl.col("HTS").str.replace("'", ""),
+                date=(
+                    pl.col("year").cast(pl.String)
+                    + "-"
+                    + pl.col("month").cast(pl.String).str.zfill(2)
+                    + "-01"
+                ).str.to_datetime("%Y-%m-%d"),
+            )
+            df = df.select(
+                pl.col(
+                    "date",
+                    "country",
+                    "hts",
+                    "unit_1",
+                    "qty_1",
+                    "unit_2",
+                    "qty_2",
+                    "value",
+                    "import_export",
+                )
+            )
+
+            # Save Export Data
+            df_exports = df.filter(pl.col("import_export") == "e").drop("import_export")
+            df_exports.write_parquet(parquet_export_path)
+
+            # Save import data
+            df_imports = df.filter(pl.col("import_export") == "i").drop("import_export")
+            df_imports.write_parquet(parquet_import_path)
+
+            # Stage 4: Cleanup
+            shutil.rmtree(stage1_dir)
+            shutil.rmtree(stage2_dir)
+            temp_zip.unlink(missing_ok=True)
+
+        if export:
+            return pl.read_parquet(parquet_export_path)
+        else:
+            return pl.read_parquet(parquet_import_path)
+
+    def pull_census_hts(self, exports: bool, state: str) -> pl.DataFrame:
+        """
+        Pulls HTS data from the Census and saves them in a parquet file.
+
+        Parameters
+        ----------
+        end_year: int
+            The last year to pull data from.
+        start_year: int
+            The first year to pull data from.
+        exports: bool
+            If True, pulls exports data. If False, pulls imports data.
+        state: str
+            The state to pull data from (e.g. "PR" for Puerto Rico).
+
+        Returns
+        -------
+        None
+        """
+
+        for _year in range(2010, 2023):
+            exports_path = Path(
+                f"{self.saving_dir}raw/census-hts-exports-{state}-{_year}.parquet"
+            )
+            imports_path = Path(
+                f"{self.saving_dir}raw/census-hts-imports-{state}-{_year}.parquet"
+            )
+
+            if exports_path.exists() and imports_path.exists():
+                continue
+
+            req_exports = CensusAPI().timeseries_query(
+                dataset="timeseries-intltrade-exports-statehs",
+                params_list=[
+                    "CTY_CODE",
+                    "CTY_NAME",
+                    "ALL_VAL_MO",
+                    "COMM_LVL",
+                    "E_COMMODITY",
+                ],
+                year=_year,
+                extra=f"STATE={state}",
+                skip_checks=True,
+            )
+
+            req_imports = CensusAPI().timeseries_query(
+                dataset="timeseries-intltrade-imports-statehs",
+                params_list=[
+                    "CTY_CODE",
+                    "CTY_NAME",
+                    "GEN_VAL_MO",
+                    "COMM_LVL",
+                    "I_COMMODITY",
+                ],
+                year=_year,
+                extra=f"STATE={state}",
+                skip_checks=True,
+            )
+
+            df_exports = pl.DataFrame(req_exports)
+            df_exports.write_parquet(exports_path)
+
+            df_imports = pl.DataFrame(req_imports)
+            df_imports.write_parquet(imports_path)
+        if exports:
+            return self.conn.execute(
+                f"SELECT * FROM '{self.saving_dir}raw/census-hts-exports-{state}-*.parquet';"
+            ).pl()
+        else:
+            return self.conn.execute(
+                f"SELECT * FROM '{self.saving_dir}raw/census-hts-imports-{state}-*.parquet';"
+            ).pl()
+
+    def pull_census_naics(self, exports: bool, state: str) -> pl.DataFrame:
+        """
+        Pulls NAICS data from the Census and saves them in a parquet file.
+
+        Parameters
+        ----------
+        end_year: int
+            The last year to pull data from.
+        start_year: int
+            The first year to pull data from.
+        exports: bool
+            If True, pulls exports data. If False, pulls imports data.
+        state: str
+            The state to pull data from (e.g. "PR" for Puerto Rico).
+
+        Returns
+        -------
+        None
+        """
+
+        for _year in range(2010, 2023):
+            exports_path = Path(
+                f"{self.saving_dir}raw/census-naics-exports-{state}-{_year}.parquet"
+            )
+            imports_path = Path(
+                f"{self.saving_dir}raw/census-naics-imports-{state}-{_year}.parquet"
+            )
+
+            if exports_path.exists() and imports_path.exists():
+                continue
+
+            req_exports = CensusAPI().timeseries_query(
+                dataset="timeseries-intltrade-exports-statenaics",
+                params_list=[
+                    "CTY_CODE",
+                    "CTY_NAME",
+                    "ALL_VAL_MO",
+                    "COMM_LVL",
+                    "NAICS",
+                ],
+                year=_year,
+                extra=f"STATE={state}",
+                skip_checks=True,
+            )
+
+            req_imports = CensusAPI().timeseries_query(
+                dataset="timeseries-intltrade-imports-statenaics",
+                params_list=[
+                    "CTY_CODE",
+                    "CTY_NAME",
+                    "GEN_VAL_MO",
+                    "COMM_LVL",
+                    "NAICS",
+                ],
+                year=_year,
+                extra=f"STATE={state}",
+                skip_checks=True,
+            )
+
+            df_exports = pl.DataFrame(req_exports)
+            df_exports = df_exports.rename(df_exports.row(0, named=True))
+            df_exports = df_exports.slice(1)
+            df_exports.write_parquet(exports_path)
+
+            df_imports = pl.DataFrame(req_imports)
+            df_imports = df_imports.rename(df_imports.row(0, named=True))
+            df_imports = df_imports.slice(1)
+            df_imports.write_parquet(imports_path)
+        if exports:
+            return self.conn.execute(
+                f"SELECT * FROM '{self.saving_dir}raw/census-naics-exports-{state}-*.parquet';"
+            ).pl()
+        else:
+            return self.conn.execute(
+                f"SELECT * FROM '{self.saving_dir}raw/census-naics-imports-{state}-*.parquet';"
+            ).pl()