cdfidata 0.1.0__py3-none-any.whl

cdfidata/__init__.py

@@ -0,0 +1,12 @@
+from cdfidata.sources.tlr import TLRLoader
+from cdfidata.sources.clr import CLRLoader
+from cdfidata.sources.awards import AwardsLoader
+from cdfidata.pipeline.downloader import download_tlr, list_cached, clear_cache
+from cdfidata.pipeline.exporter import to_csv, to_sqlite, to_parquet
+
+__version__ = "0.1.0"
+__all__ = [
+    "TLRLoader", "CLRLoader", "AwardsLoader",
+    "download_tlr", "list_cached", "clear_cache",
+    "to_csv", "to_sqlite", "to_parquet",
+]

cdfidata/pipeline/cleaner.py

@@ -0,0 +1,115 @@
+"""
+Standardize and clean raw CDFI Fund DataFrames.
+Handles column renaming, type casting, null handling, and boolean normalization.
+"""
+import pandas as pd
+from cdfidata.utils.schema import BOOL_TRUE_VALUES, BOOL_FALSE_VALUES
+
+
+def normalize_columns(df: pd.DataFrame, column_map: dict) -> pd.DataFrame:
+    """
+    Rename columns using a mapping dict.
+    Only renames columns that exist in the DataFrame.
+
+    Args:
+        df:         Raw DataFrame
+        column_map: Dict of {RAW_NAME: clean_name}
+
+    Returns:
+        DataFrame with renamed columns
+    """
+    df.columns = df.columns.str.strip().str.upper()
+    available = {k: v for k, v in column_map.items() if k in df.columns}
+    return df.rename(columns=available)
+
+
+def cast_types(df: pd.DataFrame, dtype_map: dict) -> pd.DataFrame:
+    """
+    Cast columns to specified types.
+    Silently skips columns that don't exist or can't be cast.
+
+    Args:
+        df:        DataFrame with clean column names
+        dtype_map: Dict of {column_name: type_string}
+
+    Returns:
+        DataFrame with cast columns
+    """
+    for col, dtype in dtype_map.items():
+        if col not in df.columns:
+            continue
+        try:
+            if dtype == "bool":
+                df[col] = df[col].apply(_parse_bool)
+            elif dtype == "int":
+                df[col] = pd.to_numeric(df[col], errors="coerce").astype("Int64")
+            elif dtype == "float":
+                df[col] = pd.to_numeric(df[col], errors="coerce")
+            else:
+                df[col] = df[col].astype(dtype)
+        except Exception:
+            pass
+    return df
+
+
+def _parse_bool(val) -> bool:
+    """Parse a value to boolean using CDFI Fund conventions."""
+    if pd.isna(val):
+        return None
+    s = str(val).strip()
+    if s in BOOL_TRUE_VALUES:
+        return True
+    if s in BOOL_FALSE_VALUES:
+        return False
+    return None
+
+
+def clean_strings(df: pd.DataFrame, cols: list = None) -> pd.DataFrame:
+    """
+    Strip whitespace and normalize string columns to uppercase.
+
+    Args:
+        df:   DataFrame
+        cols: List of columns to clean (default: all object columns)
+
+    Returns:
+        Cleaned DataFrame
+    """
+    if cols is None:
+        cols = df.select_dtypes(include="object").columns.tolist()
+    for col in cols:
+        if col in df.columns:
+            df[col] = df[col].str.strip()
+    return df
+
+
+def drop_empty_rows(df: pd.DataFrame, required_cols: list) -> pd.DataFrame:
+    """Drop rows where all required columns are null."""
+    return df.dropna(subset=required_cols, how="all")
+
+
+def standardize(
+    df: pd.DataFrame,
+    column_map: dict,
+    dtype_map: dict,
+    required_cols: list = None,
+) -> pd.DataFrame:
+    """
+    Full standardization pipeline: rename → cast → clean → drop empty.
+
+    Args:
+        df:            Raw DataFrame
+        column_map:    Column rename mapping
+        dtype_map:     Type casting mapping
+        required_cols: Columns required to be non-null
+
+    Returns:
+        Clean, standardized DataFrame
+    """
+    df = normalize_columns(df, column_map)
+    df = cast_types(df, dtype_map)
+    df = clean_strings(df)
+    if required_cols:
+        df = drop_empty_rows(df, required_cols)
+    df = df.reset_index(drop=True)
+    return df

cdfidata/pipeline/downloader.py

@@ -0,0 +1,134 @@
+"""
+Download and cache CDFI Fund public datasets locally.
+Files are cached to avoid repeated downloads.
+"""
+import os
+import zipfile
+import requests
+from pathlib import Path
+
+from cdfidata.utils.schema import CACHE_DIR, TLR_URLS
+
+
+def get_cache_dir() -> Path:
+    """Return and create the local cache directory."""
+    path = Path(CACHE_DIR)
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def cache_path(filename: str) -> Path:
+    """Return the full cache path for a given filename."""
+    return get_cache_dir() / filename
+
+
+def is_cached(filename: str) -> bool:
+    """Check if a file is already cached locally."""
+    return cache_path(filename).exists()
+
+
+def download_file(url: str, filename: str, force: bool = False) -> Path:
+    """
+    Download a file from a URL and cache it locally.
+
+    Args:
+        url:      Full URL to download from
+        filename: Local filename to save as
+        force:    Re-download even if cached
+
+    Returns:
+        Path to the cached file
+    """
+    path = cache_path(filename)
+
+    if path.exists() and not force:
+        print(f"Using cached file: {path}")
+        return path
+
+    print(f"Downloading {filename}...")
+    print(f"URL: {url}")
+
+    response = requests.get(url, stream=True, timeout=120)
+    response.raise_for_status()
+
+    total = int(response.headers.get("content-length", 0))
+    downloaded = 0
+
+    with open(path, "wb") as f:
+        for chunk in response.iter_content(chunk_size=8192):
+            f.write(chunk)
+            downloaded += len(chunk)
+            if total:
+                pct = downloaded / total * 100
+                print(f"\r  {pct:.1f}% ({downloaded/1e6:.1f}MB)", end="")
+
+    print(f"\nSaved to {path}")
+    return path
+
+
+def extract_zip(zip_path: Path, extract_to: Path = None) -> list:
+    """
+    Extract a zip file to the cache directory.
+
+    Args:
+        zip_path:   Path to the zip file
+        extract_to: Directory to extract to (default: cache dir)
+
+    Returns:
+        List of extracted file paths
+    """
+    extract_to = extract_to or get_cache_dir()
+
+    with zipfile.ZipFile(zip_path, "r") as zf:
+        names = zf.namelist()
+        zf.extractall(extract_to)
+        print(f"Extracted {len(names)} files to {extract_to}")
+
+    return [extract_to / name for name in names]
+
+
+def download_tlr(year: int, force: bool = False) -> Path:
+    """
+    Download TLR/CLR zip file for a given fiscal year.
+
+    Args:
+        year:  Fiscal year e.g. 2022
+        force: Re-download even if cached
+
+    Returns:
+        Path to the downloaded zip file
+    """
+    if year not in TLR_URLS:
+        available = list(TLR_URLS.keys())
+        raise ValueError(
+            f"No TLR URL available for FY{year}. "
+            f"Available years: {available}"
+        )
+
+    url = TLR_URLS[year]
+    filename = f"TLR_CLR_FY{year}.zip"
+    return download_file(url, filename, force=force)
+
+
+def list_cached() -> list:
+    """List all files currently in the local cache."""
+    cache = get_cache_dir()
+    files = list(cache.iterdir())
+    if not files:
+        print("Cache is empty.")
+    else:
+        print(f"Cached files in {cache}:")
+        for f in sorted(files):
+            size_mb = f.stat().st_size / 1e6
+            print(f"  {f.name} ({size_mb:.1f}MB)")
+    return files
+
+
+def clear_cache() -> None:
+    """Delete all cached files."""
+    cache = get_cache_dir()
+    count = 0
+    for f in cache.iterdir():
+        f.unlink()
+        count += 1
+    print(f"Cleared {count} cached files from {cache}")

cdfidata/pipeline/exporter.py

@@ -0,0 +1,65 @@
+"""
+Export cleaned CDFI Fund DataFrames to CSV, SQLite, and Parquet formats.
+"""
+import pandas as pd
+from pathlib import Path
+
+
+def to_csv(df: pd.DataFrame, path: str, **kwargs) -> None:
+    """Export DataFrame to CSV."""
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    df.to_csv(path, index=False, **kwargs)
+    print(f"Exported {len(df):,} rows to {path}")
+
+
+def to_sqlite(df: pd.DataFrame, db_path: str, table_name: str,
+              if_exists: str = "replace") -> None:
+    """
+    Export DataFrame to a SQLite database table.
+
+    Args:
+        df:         DataFrame to export
+        db_path:    Path to SQLite database file
+        table_name: Table name to write to
+        if_exists:  'replace', 'append', or 'fail'
+    """
+    import sqlite3
+    path = Path(db_path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    with sqlite3.connect(path) as conn:
+        df.to_sql(table_name, conn, if_exists=if_exists, index=False)
+    print(f"Exported {len(df):,} rows to {path} (table: {table_name})")
+
+
+def to_parquet(df: pd.DataFrame, path: str, **kwargs) -> None:
+    """Export DataFrame to Parquet format."""
+    try:
+        import pyarrow
+    except ImportError:
+        raise ImportError(
+            "pyarrow is required for parquet export. "
+            "Install it with: pip install pyarrow"
+        )
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    df.to_parquet(path, index=False, **kwargs)
+    print(f"Exported {len(df):,} rows to {path}")
+
+
+def export_all(df: pd.DataFrame, base_path: str, table_name: str) -> None:
+    """
+    Export to all three formats at once.
+
+    Args:
+        df:         DataFrame to export
+        base_path:  Base path without extension
+        table_name: SQLite table name
+    """
+    to_csv(df, f"{base_path}.csv")
+    to_sqlite(df, f"{base_path}.db", table_name)
+    try:
+        to_parquet(df, f"{base_path}.parquet")
+    except ImportError:
+        print("Skipping parquet export — pyarrow not installed")

cdfidata/sources/awards.py

@@ -0,0 +1,99 @@
+"""
+CDFI Fund Awards Database loader.
+Covers all CDFI Fund program awardees across all years.
+"""
+import pandas as pd
+import numpy as np
+import random
+from typing import Optional
+
+from cdfidata.pipeline.cleaner import standardize
+from cdfidata.pipeline.exporter import to_csv, to_sqlite
+from cdfidata.utils.schema import AWARDS_COLUMNS, AWARDS_DTYPES
+
+
+class AwardsLoader:
+    """
+    Loader for CDFI Fund Awards Database.
+
+    Usage:
+        loader = AwardsLoader()
+        df = loader.load_sample()
+        df_il = loader.filter_state("IL")
+    """
+
+    def __init__(self):
+        self._df: Optional[pd.DataFrame] = None
+
+    def load_from_file(self, path: str) -> pd.DataFrame:
+        """Load awards data from a local CSV file."""
+        print(f"Loading awards data from {path}...")
+        df = pd.read_csv(path, dtype=str, low_memory=False)
+        df = standardize(df, AWARDS_COLUMNS, AWARDS_DTYPES,
+                         required_cols=["award_amount"])
+        self._df = df
+        return df
+
+    def load_sample(self, n: int = 500) -> pd.DataFrame:
+        """Generate synthetic awards sample data for testing."""
+        random.seed(42)
+        np.random.seed(42)
+
+        states = ["IL", "NY", "CA", "TX", "GA", "NC", "OH", "PA", "FL", "MI"]
+        programs = ["FA", "NACA", "NMTC", "BEA", "CMF", "BOND"]
+        inst_types = ["Loan Fund", "Bank", "Credit Union", "Venture Fund",
+                      "Depository Institution Holding Company"]
+
+        records = []
+        for i in range(n):
+            records.append({
+                "awardee_name": f"Community Development Fund {i+1}",
+                "state": random.choice(states),
+                "award_year": random.randint(2010, 2023),
+                "award_amount": round(np.random.lognormal(14, 1)),
+                "program": random.choice(programs),
+                "award_type": "Financial Assistance",
+                "institution_type": random.choice(inst_types),
+            })
+
+        self._df = pd.DataFrame(records)
+        return self._df
+
+    def filter_state(self, state: str) -> pd.DataFrame:
+        self._check_loaded()
+        return self._df[self._df["state"] == state.upper()].copy()
+
+    def filter_program(self, program: str) -> pd.DataFrame:
+        self._check_loaded()
+        return self._df[
+            self._df["program"].str.contains(program, case=False, na=False)
+        ].copy()
+
+    def filter_year(self, year: int) -> pd.DataFrame:
+        self._check_loaded()
+        return self._df[self._df["award_year"] == year].copy()
+
+    def summary(self) -> None:
+        self._check_loaded()
+        df = self._df
+        print(f"\nAwards Database Summary")
+        print(f"  Total awards:     {len(df):,}")
+        print(f"  Total amount:     ${df['award_amount'].sum()/1e9:.2f}B")
+        print(f"  States covered:   {df['state'].nunique()}")
+        print(f"  Programs:         {df['program'].nunique()}")
+        print(f"  Year range:       {df['award_year'].min()}–{df['award_year'].max()}")
+        print()
+
+    def to_csv(self, path: str) -> None:
+        self._check_loaded()
+        to_csv(self._df, path)
+
+    def to_sqlite(self, db_path: str, table: str = "awards") -> None:
+        self._check_loaded()
+        to_sqlite(self._df, db_path, table)
+
+    def _check_loaded(self) -> None:
+        if self._df is None:
+            raise RuntimeError(
+                "No data loaded. Call .load_from_file() or .load_sample() first."
+            )

cdfidata/sources/clr.py

@@ -0,0 +1,107 @@
+"""
+CLR (Consumer Loan Report) data loader.
+Aggregated to census tract level, 12 variables.
+"""
+import pandas as pd
+import numpy as np
+import random
+from typing import Optional
+
+from cdfidata.pipeline.cleaner import standardize
+from cdfidata.pipeline.exporter import to_csv, to_sqlite, to_parquet
+from cdfidata.utils.schema import CLR_COLUMNS, CLR_DTYPES
+
+
+class CLRLoader:
+    """
+    Loader for CDFI Fund Consumer Loan Report (CLR) data.
+
+    Usage:
+        loader = CLRLoader()
+        df = loader.load_sample()
+        df_il = loader.filter_state("IL")
+    """
+
+    def __init__(self):
+        self._df: Optional[pd.DataFrame] = None
+        self._year: Optional[int] = None
+
+    def load_from_file(self, path: str, year: int = 2022) -> pd.DataFrame:
+        """
+        Load CLR data from a local CSV file.
+
+        Args:
+            path: Path to the CLR CSV file
+            year: Fiscal year for reference
+
+        Returns:
+            Clean pandas DataFrame
+        """
+        self._year = year
+        print(f"Loading CLR data from {path}...")
+        df = pd.read_csv(path, dtype=str, low_memory=False)
+        print(f"Raw records: {len(df):,}")
+        df = standardize(df, CLR_COLUMNS, CLR_DTYPES,
+                         required_cols=["total_amount"])
+        print(f"Clean records: {len(df):,}")
+        self._df = df
+        return df
+
+    def load_sample(self, n: int = 1000) -> pd.DataFrame:
+        """Generate synthetic CLR sample data for testing."""
+        random.seed(42)
+        np.random.seed(42)
+
+        states = ["IL", "NY", "CA", "TX", "GA", "NC", "OH", "PA", "FL", "MI"]
+        loan_types = ["Auto Loan", "Personal Loan", "Credit Card",
+                      "Student Loan", "Home Improvement"]
+
+        records = []
+        for i in range(n):
+            n_loans = random.randint(1, 500)
+            avg = round(np.random.lognormal(8, 1))
+            records.append({
+                "fiscal_year": random.choice([2020, 2021, 2022]),
+                "state": random.choice(states),
+                "census_tract": f"{random.randint(1000, 9999):04d}",
+                "loan_type": random.choice(loan_types),
+                "number_of_loans": n_loans,
+                "total_amount": n_loans * avg,
+                "average_amount": avg,
+                "low_income_area": random.choice([True, False]),
+                "minority_area": random.choice([True, False]),
+                "rural_area": random.choice([True, False]),
+                "program": random.choice(["FA", "NACA", "RRP"]),
+                "award_type": "Financial Assistance",
+            })
+
+        self._df = pd.DataFrame(records)
+        return self._df
+
+    def filter_state(self, state: str) -> pd.DataFrame:
+        self._check_loaded()
+        return self._df[self._df["state"] == state.upper()].copy()
+
+    def summary(self) -> None:
+        self._check_loaded()
+        df = self._df
+        print(f"\nCLR Data Summary")
+        print(f"  Total records:    {len(df):,}")
+        print(f"  Total loans:      {df['number_of_loans'].sum():,.0f}")
+        print(f"  Total amount:     ${df['total_amount'].sum()/1e9:.2f}B")
+        print(f"  States covered:   {df['state'].nunique()}")
+        print()
+
+    def to_csv(self, path: str) -> None:
+        self._check_loaded()
+        to_csv(self._df, path)
+
+    def to_sqlite(self, db_path: str, table: str = "clr") -> None:
+        self._check_loaded()
+        to_sqlite(self._df, db_path, table)
+
+    def _check_loaded(self) -> None:
+        if self._df is None:
+            raise RuntimeError(
+                "No data loaded. Call .load_from_file() or .load_sample() first."
+            )

cdfidata/sources/tlr.py

@@ -0,0 +1,172 @@
+"""
+TLR (Transaction Level Report) data loader.
+Downloads, cleans, and returns CDFI Fund transaction-level loan data.
+"""
+import pandas as pd
+from pathlib import Path
+from typing import Optional
+
+from cdfidata.pipeline.downloader import download_tlr, extract_zip, cache_path
+from cdfidata.pipeline.cleaner import standardize
+from cdfidata.pipeline.exporter import to_csv, to_sqlite, to_parquet
+from cdfidata.utils.schema import TLR_COLUMNS, TLR_DTYPES
+
+
+class TLRLoader:
+    """
+    Loader for CDFI Fund Transaction Level Report (TLR) data.
+
+    Usage:
+        loader = TLRLoader()
+        df = loader.load(year=2022)
+        df_il = loader.filter_state("IL")
+    """
+
+    def __init__(self):
+        self._df: Optional[pd.DataFrame] = None
+        self._year: Optional[int] = None
+
+    def load(self, year: int = 2022, force: bool = False) -> pd.DataFrame:
+        """
+        Download and load TLR data for a given fiscal year.
+
+        Args:
+            year:  Fiscal year e.g. 2022
+            force: Re-download even if cached
+
+        Returns:
+            Clean pandas DataFrame with standardized columns
+        """
+        self._year = year
+
+        # Download zip
+        zip_path = download_tlr(year, force=force)
+
+        # Extract
+        extracted = extract_zip(zip_path)
+        tlr_files = [f for f in extracted if "TLR" in str(f).upper()
+                     and str(f).endswith(".csv")]
+
+        if not tlr_files:
+            # Try finding CSV directly in cache
+            tlr_files = list(cache_path("").parent.glob(f"*TLR*{year}*.csv"))
+
+        if not tlr_files:
+            raise FileNotFoundError(
+                f"Could not find TLR CSV file in extracted archive for FY{year}. "
+                f"Extracted files: {extracted}"
+            )
+
+        csv_path = tlr_files[0]
+        print(f"Loading TLR data from {csv_path}...")
+
+        df = pd.read_csv(csv_path, dtype=str, low_memory=False)
+        print(f"Raw records: {len(df):,}")
+
+        df = standardize(df, TLR_COLUMNS, TLR_DTYPES,
+                         required_cols=["amount"])
+
+        print(f"Clean records: {len(df):,}")
+        self._df = df
+        return df
+
+    def load_sample(self, n: int = 1000) -> pd.DataFrame:
+        """
+        Generate synthetic TLR sample data for testing and demos.
+        Does not require downloading real data.
+
+        Args:
+            n: Number of sample records
+
+        Returns:
+            Synthetic DataFrame with TLR schema
+        """
+        import numpy as np
+        import random
+        random.seed(42)
+        np.random.seed(42)
+
+        states = ["IL", "NY", "CA", "TX", "GA", "NC", "OH", "PA", "FL", "MI"]
+        loan_types = ["Business Loan", "Microenterprise Loan", "Home Mortgage",
+                      "Home Improvement", "Consumer Loan", "Commercial RE"]
+        purposes = ["Job Creation", "Affordable Housing", "Small Business",
+                    "Community Facility", "Microenterprise"]
+        programs = ["FA", "NACA", "RRP"]
+
+        records = []
+        for i in range(n):
+            records.append({
+                "fiscal_year": random.choice([2020, 2021, 2022]),
+                "award_number": f"FA-{i:06d}",
+                "financing_type": "Loan",
+                "loan_type": random.choice(loan_types),
+                "purpose": random.choice(purposes),
+                "amount": round(np.random.lognormal(10, 1.5)),
+                "term_months": random.choice([12, 24, 36, 60, 84, 120]),
+                "interest_rate": round(np.random.uniform(0.02, 0.08), 4),
+                "state": random.choice(states),
+                "census_tract": f"{random.randint(1000, 9999):04d}",
+                "low_income_area": random.choice([True, False]),
+                "distressed_area": random.choice([True, False]),
+                "minority_borrower": random.choice([True, False]),
+                "women_borrower": random.choice([True, False]),
+                "jobs_created": random.randint(0, 50),
+                "jobs_retained": random.randint(0, 100),
+                "program": random.choice(programs),
+            })
+
+        self._df = pd.DataFrame(records)
+        return self._df
+
+    def filter_state(self, state: str) -> pd.DataFrame:
+        """Filter loaded data to a specific state."""
+        self._check_loaded()
+        return self._df[self._df["state"] == state.upper()].copy()
+
+    def filter_loan_type(self, loan_type: str) -> pd.DataFrame:
+        """Filter by loan type (partial match)."""
+        self._check_loaded()
+        return self._df[
+            self._df["loan_type"].str.contains(loan_type, case=False, na=False)
+        ].copy()
+
+    def filter_amount(self, min_amount: float = 0,
+                      max_amount: float = float("inf")) -> pd.DataFrame:
+        """Filter by loan amount range."""
+        self._check_loaded()
+        return self._df[
+            (self._df["amount"] >= min_amount) &
+            (self._df["amount"] <= max_amount)
+        ].copy()
+
+    def summary(self) -> pd.DataFrame:
+        """Return a summary of the loaded TLR data."""
+        self._check_loaded()
+        df = self._df
+        print(f"\nTLR Data Summary — FY{self._year}")
+        print(f"  Total records:      {len(df):,}")
+        print(f"  Total amount:       ${df['amount'].sum()/1e9:.2f}B")
+        print(f"  Median loan size:   ${df['amount'].median():,.0f}")
+        print(f"  States covered:     {df['state'].nunique()}")
+        if "jobs_created" in df.columns:
+            print(f"  Total jobs created: {df['jobs_created'].sum():,.0f}")
+        print()
+        return df.describe()
+
+    def to_csv(self, path: str) -> None:
+        self._check_loaded()
+        to_csv(self._df, path)
+
+    def to_sqlite(self, db_path: str, table: str = "tlr") -> None:
+        self._check_loaded()
+        to_sqlite(self._df, db_path, table)
+
+    def to_parquet(self, path: str) -> None:
+        self._check_loaded()
+        to_parquet(self._df, path)
+
+    def _check_loaded(self) -> None:
+        if self._df is None:
+            raise RuntimeError(
+                "No data loaded. Call .load(year=2022) or .load_sample() first."
+            )

cdfidata/utils/schema.py

@@ -0,0 +1,174 @@
+"""
+Column mappings, data dictionaries, and constants for CDFI Fund public datasets.
+All source URLs and field definitions are based on official CDFI Fund documentation.
+"""
+
+# ── TLR (Transaction Level Report) ───────────────────────────────────────────
+# Source: cdfifund.gov — annual release, 61 variables, 1M+ loan observations
+# Masked to protect individual CDFI identity
+
+TLR_COLUMNS = {
+    "FISCAL_YEAR":              "fiscal_year",
+    "AWARD_NUMBER":             "award_number",
+    "AWARDEE_TYPE":             "awardee_type",
+    "FINANCING_TYPE":           "financing_type",
+    "LOAN_TYPE":                "loan_type",
+    "ACTIVITY_TYPE":            "activity_type",
+    "AMOUNT":                   "amount",
+    "TERM":                     "term_months",
+    "INTEREST_RATE":            "interest_rate",
+    "PURPOSE":                  "purpose",
+    "STATE":                    "state",
+    "CENSUS_TRACT":             "census_tract",
+    "COUNTY":                   "county",
+    "METROPOLITAN_AREA":        "metropolitan_area",
+    "LOW_INCOME_AREA":          "low_income_area",
+    "DISTRESSED_AREA":          "distressed_area",
+    "MINORITY_BORROWER":        "minority_borrower",
+    "WOMEN_BORROWER":           "women_borrower",
+    "LOW_INCOME_BORROWER":      "low_income_borrower",
+    "FIRST_TIME_BORROWER":      "first_time_borrower",
+    "JOBS_CREATED":             "jobs_created",
+    "JOBS_RETAINED":            "jobs_retained",
+    "AFFORDABLE_UNITS":         "affordable_units",
+    "PROGRAM":                  "program",
+}
+
+TLR_DTYPES = {
+    "fiscal_year":          "int",
+    "amount":               "float",
+    "term_months":          "float",
+    "interest_rate":        "float",
+    "jobs_created":         "float",
+    "jobs_retained":        "float",
+    "affordable_units":     "float",
+    "low_income_area":      "bool",
+    "distressed_area":      "bool",
+    "minority_borrower":    "bool",
+    "women_borrower":       "bool",
+    "low_income_borrower":  "bool",
+    "first_time_borrower":  "bool",
+}
+
+# ── CLR (Consumer Loan Report) ────────────────────────────────────────────────
+# Source: cdfifund.gov — aggregated to census tract, 12 variables
+
+CLR_COLUMNS = {
+    "FISCAL_YEAR":          "fiscal_year",
+    "STATE":                "state",
+    "CENSUS_TRACT":         "census_tract",
+    "LOAN_TYPE":            "loan_type",
+    "NUMBER_OF_LOANS":      "number_of_loans",
+    "TOTAL_AMOUNT":         "total_amount",
+    "AVERAGE_AMOUNT":       "average_amount",
+    "LOW_INCOME_AREA":      "low_income_area",
+    "MINORITY_AREA":        "minority_area",
+    "RURAL_AREA":           "rural_area",
+    "PROGRAM":              "program",
+    "AWARD_TYPE":           "award_type",
+}
+
+CLR_DTYPES = {
+    "fiscal_year":      "int",
+    "number_of_loans":  "int",
+    "total_amount":     "float",
+    "average_amount":   "float",
+    "low_income_area":  "bool",
+    "minority_area":    "bool",
+    "rural_area":       "bool",
+}
+
+# ── ILR (Institution Level Report) ───────────────────────────────────────────
+# Source: data.gov CIIS — 11 years (FY2003-2013), 728 CDFIs
+
+ILR_COLUMNS = {
+    "FISCAL_YEAR":              "fiscal_year",
+    "CDFI_NAME":                "cdfi_name",
+    "STATE":                    "state",
+    "INSTITUTION_TYPE":         "institution_type",
+    "TOTAL_ASSETS":             "total_assets",
+    "TOTAL_NET_ASSETS":         "total_net_assets",
+    "TOTAL_LOANS":              "total_loans",
+    "TOTAL_DEPOSITS":           "total_deposits",
+    "NET_INCOME":               "net_income",
+    "TOTAL_FINANCING":          "total_financing",
+    "TARGET_MARKET":            "target_market",
+    "CERTIFICATION_STATUS":     "certification_status",
+}
+
+ILR_DTYPES = {
+    "fiscal_year":      "int",
+    "total_assets":     "float",
+    "total_net_assets": "float",
+    "total_loans":      "float",
+    "total_deposits":   "float",
+    "net_income":       "float",
+    "total_financing":  "float",
+}
+
+# ── NMTC Allocatee Data ───────────────────────────────────────────────────────
+# Source: cdfifund.gov NMTC program allocatee database
+
+NMTC_COLUMNS = {
+    "CDE_NAME":             "cde_name",
+    "STATE":                "state",
+    "ALLOCATION_YEAR":      "allocation_year",
+    "ALLOCATION_AMOUNT":    "allocation_amount",
+    "CUMULATIVE_ALLOCATION": "cumulative_allocation",
+    "SERVICE_AREA":         "service_area",
+    "MISSION_FOCUS":        "mission_focus",
+}
+
+NMTC_DTYPES = {
+    "allocation_year":       "int",
+    "allocation_amount":     "float",
+    "cumulative_allocation": "float",
+}
+
+# ── Awards Database ───────────────────────────────────────────────────────────
+# Source: cdfifund.gov Awards Database
+
+AWARDS_COLUMNS = {
+    "AWARDEE_NAME":     "awardee_name",
+    "STATE":            "state",
+    "AWARD_YEAR":       "award_year",
+    "AWARD_AMOUNT":     "award_amount",
+    "PROGRAM":          "program",
+    "AWARD_TYPE":       "award_type",
+    "INSTITUTION_TYPE": "institution_type",
+}
+
+AWARDS_DTYPES = {
+    "award_year":   "int",
+    "award_amount": "float",
+}
+
+# ── Download URLs ─────────────────────────────────────────────────────────────
+CDFI_FUND_BASE = "https://www.cdfifund.gov"
+
+TLR_URLS = {
+    2022: "https://www.cdfifund.gov/sites/cdfi/files/2024-12/FY2022_TLR_CLR_Public_Data_File.zip",
+    2021: "https://www.cdfifund.gov/sites/cdfi/files/2023-09/FY2021_TLR_CLR_Public_Data_File.zip",
+}
+
+ILR_URL = "https://data.gov/dataset/data-on-cdfi-program-awardees"
+
+NMTC_URL = "https://www.cdfifund.gov/programs-training/programs/new-markets-tax-credit/allocatees"
+
+# ── Cache directory ───────────────────────────────────────────────────────────
+import os
+CACHE_DIR = os.path.join(os.path.expanduser("~"), ".cdfidata", "cache")
+
+# ── Boolean value mappings ────────────────────────────────────────────────────
+BOOL_TRUE_VALUES  = {"Y", "YES", "1", "TRUE", "X", "yes", "true", "y"}
+BOOL_FALSE_VALUES = {"N", "NO",  "0", "FALSE", "",  "no",  "false", "n"}
+
+# ── US State abbreviations ────────────────────────────────────────────────────
+US_STATES = {
+    "AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DE", "FL", "GA",
+    "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD",
+    "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ",
+    "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "RI", "SC",
+    "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY",
+    "DC", "GU", "PR", "VI",
+}

cdfidata-0.1.0.dist-info/METADATA

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: cdfidata
+Version: 0.1.0
+Summary: ETL pipeline for US Treasury CDFI Fund public datasets — TLR, CLR, ILR, NMTC, and Awards data
+License: MIT
+Project-URL: Homepage, https://github.com/Jaypatel1511/cdfi-data
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: pandas>=1.4.0
+Requires-Dist: numpy>=1.21.0
+Requires-Dist: requests>=2.27.0

cdfidata-0.1.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+cdfidata/__init__.py,sha256=nUODBEA_UCmmvQGlWpQmd4Idr4AlViEEJZzoeeEkeuI,458
+cdfidata/analysis/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cdfidata/pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cdfidata/pipeline/cleaner.py,sha256=FWvZ7-71R57oryeg8qkvrdJz6DQSPCpmzwawdASXocE,3272
+cdfidata/pipeline/downloader.py,sha256=LvHyxBmYRHeT7JjqH7ucFbQmladnAKTR0-i-dZluAls,3519
+cdfidata/pipeline/exporter.py,sha256=XNKvE6ZahnCkUOYU2NLB-I-9LF7AkP3leMkEJPXqsvw,2023
+cdfidata/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cdfidata/sources/awards.py,sha256=So3KrPgDhFRSFTcDB-0vFOLSS1uCytWPvTz-jMNFiYw,3465
+cdfidata/sources/clr.py,sha256=ThEUKyHbpzij-lQE8rTWBkkedi0DEENqQC71dOFDDm8,3648
+cdfidata/sources/tlr.py,sha256=YiAAgc1V1RD9EemHefmnTFgRYstx0U1ccNcePJPWQYw,6135
+cdfidata/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cdfidata/utils/schema.py,sha256=suAMxlvemblZClOWS2TCKCpdezGqWCUEWBwIfUqW9lo,7385
+tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tests/conftest.py,sha256=mjjNNhFkzjLPGaOSmlvK-bS00PjXQsStr75jOWD1mBE,580
+tests/test_awards.py,sha256=bghSiCPy1tJmGGYxFs9uhijpRWtFiBtxIuHxq5k2-t8,1284
+tests/test_cleaner.py,sha256=SXFsRUvVpc5gOiV1rOZoEcfw-MiKGS8pGNJNDUc1fvU,1672
+tests/test_clr.py,sha256=gyECfdpjXgl3bQzR4_zS28GSAAd1jxR5fPr_sEmWQNo,814
+tests/test_tlr.py,sha256=YSDLjgJcj3a24_wgerhHNDTxhEa1tjbldxwSMgNwsT4,1827
+cdfidata-0.1.0.dist-info/METADATA,sha256=2hgGvrWdU71W98bvVyYj7T3P3uVnTe9oty8Uy16Ow8U,388
+cdfidata-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+cdfidata-0.1.0.dist-info/top_level.txt,sha256=q-ti-k5gZ_-15nSh0kiVoq5UMK6uajqbb09uSXpHCag,15
+cdfidata-0.1.0.dist-info/RECORD,,

cdfidata-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

cdfidata-0.1.0.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+cdfidata
+tests

tests/conftest.py

@@ -0,0 +1,31 @@
+import pytest
+from cdfidata.sources.tlr import TLRLoader
+from cdfidata.sources.clr import CLRLoader
+from cdfidata.sources.awards import AwardsLoader
+
+
+@pytest.fixture
+def tlr_sample():
+    loader = TLRLoader()
+    return loader.load_sample(n=500)
+
+
+@pytest.fixture
+def tlr_loader():
+    loader = TLRLoader()
+    loader.load_sample(n=500)
+    return loader
+
+
+@pytest.fixture
+def clr_loader():
+    loader = CLRLoader()
+    loader.load_sample(n=500)
+    return loader
+
+
+@pytest.fixture
+def awards_loader():
+    loader = AwardsLoader()
+    loader.load_sample(n=200)
+    return loader

tests/test_awards.py

@@ -0,0 +1,50 @@
+import pytest
+import pandas as pd
+from cdfidata.sources.awards import AwardsLoader
+
+
+def test_load_sample_returns_dataframe():
+    loader = AwardsLoader()
+    df = loader.load_sample(n=100)
+    assert isinstance(df, pd.DataFrame)
+    assert len(df) == 100
+
+
+def test_load_sample_has_required_columns():
+    loader = AwardsLoader()
+    df = loader.load_sample(n=100)
+    required = ["awardee_name", "state", "award_year", "award_amount", "program"]
+    for col in required:
+        assert col in df.columns
+
+
+def test_filter_state(awards_loader):
+    df = awards_loader.filter_state("IL")
+    assert all(df["state"] == "IL")
+
+
+def test_filter_program(awards_loader):
+    df = awards_loader.filter_program("FA")
+    assert all(df["program"].str.contains("FA", case=False))
+
+
+def test_filter_year(awards_loader):
+    df = awards_loader.filter_year(2020)
+    assert all(df["award_year"] == 2020)
+
+
+def test_summary_runs(awards_loader):
+    awards_loader.summary()
+
+
+def test_not_loaded_raises():
+    loader = AwardsLoader()
+    with pytest.raises(RuntimeError, match="No data loaded"):
+        loader.filter_state("IL")
+
+
+def test_to_csv(awards_loader, tmp_path):
+    path = str(tmp_path / "awards.csv")
+    awards_loader.to_csv(path)
+    df = pd.read_csv(path)
+    assert len(df) == 200

tests/test_cleaner.py

@@ -0,0 +1,56 @@
+import pytest
+import pandas as pd
+from cdfidata.pipeline.cleaner import (
+    normalize_columns, cast_types, clean_strings,
+    drop_empty_rows, standardize
+)
+
+
+def test_normalize_columns():
+    df = pd.DataFrame({"AMOUNT": [1, 2], "STATE": ["IL", "NY"]})
+    col_map = {"AMOUNT": "amount", "STATE": "state"}
+    result = normalize_columns(df, col_map)
+    assert "amount" in result.columns
+    assert "state" in result.columns
+
+
+def test_cast_types_float():
+    df = pd.DataFrame({"amount": ["1000.50", "2000.00", "abc"]})
+    result = cast_types(df, {"amount": "float"})
+    assert result["amount"].dtype == float
+    assert pd.isna(result["amount"].iloc[2])
+
+
+def test_cast_types_bool():
+    df = pd.DataFrame({"flag": ["Y", "N", "YES", "NO", "1", "0"]})
+    result = cast_types(df, {"flag": "bool"})
+    assert result["flag"].iloc[0] == True
+    assert result["flag"].iloc[1] == False
+
+
+def test_clean_strings():
+    df = pd.DataFrame({"name": ["  IL  ", "  NY  "]})
+    result = clean_strings(df)
+    assert result["name"].iloc[0] == "IL"
+
+
+def test_drop_empty_rows():
+    df = pd.DataFrame({"amount": [100, None, 200], "state": ["IL", "NY", None]})
+    result = drop_empty_rows(df, required_cols=["amount"])
+    assert len(result) == 2
+
+
+def test_standardize_pipeline():
+    df = pd.DataFrame({
+        "AMOUNT": ["1000", "2000", None],
+        "STATE": [" IL ", " NY ", " CA "],
+    })
+    result = standardize(
+        df,
+        column_map={"AMOUNT": "amount", "STATE": "state"},
+        dtype_map={"amount": "float"},
+        required_cols=["amount"],
+    )
+    assert "amount" in result.columns
+    assert "state" in result.columns
+    assert len(result) == 2

tests/test_clr.py

@@ -0,0 +1,33 @@
+import pytest
+import pandas as pd
+from cdfidata.sources.clr import CLRLoader
+
+
+def test_load_sample_returns_dataframe():
+    loader = CLRLoader()
+    df = loader.load_sample(n=100)
+    assert isinstance(df, pd.DataFrame)
+    assert len(df) == 100
+
+
+def test_load_sample_has_required_columns():
+    loader = CLRLoader()
+    df = loader.load_sample(n=100)
+    required = ["fiscal_year", "state", "number_of_loans", "total_amount"]
+    for col in required:
+        assert col in df.columns
+
+
+def test_filter_state(clr_loader):
+    df = clr_loader.filter_state("IL")
+    assert all(df["state"] == "IL")
+
+
+def test_summary_runs(clr_loader):
+    clr_loader.summary()
+
+
+def test_not_loaded_raises():
+    loader = CLRLoader()
+    with pytest.raises(RuntimeError, match="No data loaded"):
+        loader.filter_state("IL")

tests/test_tlr.py

@@ -0,0 +1,71 @@
+import pytest
+import pandas as pd
+from cdfidata.sources.tlr import TLRLoader
+
+
+def test_load_sample_returns_dataframe():
+    loader = TLRLoader()
+    df = loader.load_sample(n=100)
+    assert isinstance(df, pd.DataFrame)
+    assert len(df) == 100
+
+
+def test_load_sample_has_required_columns():
+    loader = TLRLoader()
+    df = loader.load_sample(n=100)
+    required = ["fiscal_year", "amount", "state", "loan_type", "purpose"]
+    for col in required:
+        assert col in df.columns
+
+
+def test_load_sample_amounts_positive():
+    loader = TLRLoader()
+    df = loader.load_sample(n=100)
+    assert (df["amount"] > 0).all()
+
+
+def test_filter_state(tlr_loader):
+    df = tlr_loader.filter_state("IL")
+    assert all(df["state"] == "IL")
+
+
+def test_filter_state_empty(tlr_loader):
+    df = tlr_loader.filter_state("ZZ")
+    assert len(df) == 0
+
+
+def test_filter_loan_type(tlr_loader):
+    df = tlr_loader.filter_loan_type("Business")
+    assert all(df["loan_type"].str.contains("Business", case=False))
+
+
+def test_filter_amount(tlr_loader):
+    df = tlr_loader.filter_amount(min_amount=10_000, max_amount=100_000)
+    assert all(df["amount"] >= 10_000)
+    assert all(df["amount"] <= 100_000)
+
+
+def test_not_loaded_raises():
+    loader = TLRLoader()
+    with pytest.raises(RuntimeError, match="No data loaded"):
+        loader.filter_state("IL")
+
+
+def test_summary_runs(tlr_loader):
+    tlr_loader.summary()
+
+
+def test_to_csv(tlr_loader, tmp_path):
+    path = str(tmp_path / "tlr.csv")
+    tlr_loader.to_csv(path)
+    df = pd.read_csv(path)
+    assert len(df) == 500
+
+
+def test_to_sqlite(tlr_loader, tmp_path):
+    path = str(tmp_path / "tlr.db")
+    tlr_loader.to_sqlite(path, table="tlr")
+    import sqlite3
+    with sqlite3.connect(path) as conn:
+        df = pd.read_sql("SELECT * FROM tlr", conn)
+    assert len(df) == 500