datapungibea 2.0.0__py3-none-any.whl

datapungibea/__init__.py

@@ -0,0 +1,19 @@
+"""Connect to Bureau of Economic Analysis (BEA) API and enrich data output.
+
+Modernized version 2.0 with YAML-driven architecture.
+"""
+
+import logging
+from typing import Any, Dict, Optional
+
+import pandas
+import requests
+
+from datapungibea.api import data
+import datapungibea.tests as tests
+
+__version__ = '2.0.0'
+__all__ = ['data', 'tests', 'utils']
+
+# Set up logging
+logging.getLogger(__name__).addHandler(logging.NullHandler())

datapungibea/api.py

@@ -0,0 +1,616 @@
+r'''
+   Main user-facing API for datapungibea.
+   Uses factory pattern from datapungi_fed architecture.
+'''
+
+import logging
+from typing import Any, Dict, Optional
+
+import pandas as pd
+
+from datapungibea import generalSettings
+from datapungibea.core import driverCore
+
+logger = logging.getLogger(__name__)
+
+
+class data:
+    r'''
+    Main API class for accessing BEA data.
+    
+    Provides access to all BEA datasets through a unified interface.
+    Uses factory pattern to create query functions for each dataset group.
+    
+    Example:
+        import datapungibea as dpb
+        
+        # Initialize with API key
+        data = dpb.data("YOUR_API_KEY")
+        
+        # Or use environment variable
+        data = dpb.data()
+        
+        # Query datasets
+        data.nipa('T10101')
+        data.metadata()  # list all datasets
+    '''
+    
+    def __init__(
+        self,
+        key: Optional[str] = None,
+        keyfile: Optional[str] = None,
+        connectionParameters: Optional[Dict[str, Any]] = None,
+    ):
+        '''
+        Initialize the BEA data API.
+
+        Three ways to provide an API key:
+            1. Environment variable API_KEY_BEA (default):
+                export API_KEY_BEA=your_key_here
+                data = dpb.data()
+
+            2. Pass key directly:
+                data = dpb.data("YOUR_API_KEY")
+
+            3. Point to a keys file (YAML or JSON):
+                data = dpb.data(keyfile="/path/to/keys.yaml")
+
+        Args:
+            key: API key string passed directly
+            keyfile: path to a YAML or JSON file containing API keys
+            connectionParameters: explicit dict with 'key' and 'url' (advanced)
+        '''
+        # Load general settings
+        self.__connectInfo = generalSettings.getGeneralSettings(
+            key=key,
+            keyfile=keyfile,
+            connectionParameters=connectionParameters,
+        )
+
+        self._metadata = self.__connectInfo.packageMetadata
+        self._help = self.__connectInfo.datasourceOverview
+
+        # Load parameters for driver creation
+        loadInfo = {
+            'baseRequest': self.__connectInfo.baseRequest,
+            'connectionParameters': self.__connectInfo.connectionParameters
+        }
+        
+        # Initialize dataset groups using factory pattern
+        self._metadata_driver = driverCore('Metadata', **loadInfo)
+        self._nipa_driver = driverCore('NIPA', **loadInfo)
+        self._national_accounts_driver = driverCore('National Accounts', **loadInfo)
+        self._international_transactions_driver = driverCore('International Transactions', **loadInfo)
+        self._international_investment_driver = driverCore('International Investment Position', **loadInfo)
+        self._industry_driver = driverCore('Industry', **loadInfo)
+        self._regional_driver = driverCore('Regional', **loadInfo)
+        self._input_output_driver = driverCore('Input-Output', **loadInfo)
+        self._international_services_driver = driverCore('International Services Trade', **loadInfo)
+        self._multinational_driver = driverCore('Multinational Enterprises', **loadInfo)
+        
+        # Store last loaded driver for clipcode
+        self._lastCalledDriver = None
+        
+    def __str__(self) -> str:
+        '''List all available dataset groups.'''
+        groups = [
+            'Metadata',
+            'NIPA',
+            'National Accounts',
+            'International Transactions',
+            'International Investment Position',
+            'Industry',
+            'Regional',
+            'Input-Output',
+            'International Services Trade',
+            'Multinational Enterprises',
+            'NIPA Vintage (not API)',
+            'NIPA Summary (not API)'
+        ]
+        
+        output = "BEA Dataset Groups:\n"
+        output += "=" * 50 + "\n"
+        for group in groups:
+            output += f"  - {group}\n"
+        output += "\n"
+        output += "Usage: data.<group_name>() or data.<group_name>['<dataset>']()\n"
+        output += "Example: data.nipa('T10101')\n"
+        output += "\nRun data._help for more information.\n"
+        
+        return output
+    
+    # =========================================================================
+    # Metadata queries
+    # =========================================================================
+    
+    def datasetlist(self, verbose: bool = False, **kwargs: Any) -> Any:
+        '''Get list of available BEA datasets.'''
+        self._lastCalledDriver = self._metadata_driver
+        return self._metadata_driver['datasetlist'](verbose=verbose, **kwargs)
+    
+    def getParameterList(self, datasetname: str, verbose: bool = False, **kwargs: Any) -> Any:
+        '''Get parameters for a specific dataset.'''
+        self._lastCalledDriver = self._metadata_driver
+        return self._metadata_driver['parameterlist'](datasetname=datasetname, verbose=verbose, **kwargs)
+    
+    def getParameterValues(self, datasetName: str, parameterName: str, verbose: bool = False, **kwargs: Any) -> Any:
+        '''Get available values for a dataset parameter.'''
+        self._lastCalledDriver = self._metadata_driver
+        return self._metadata_driver['parametervalues'](
+            datasetname=datasetName,
+            parametername=parameterName,
+            verbose=verbose,
+            **kwargs
+        )
+    
+    # =========================================================================
+    # NIPA queries
+    # =========================================================================
+    
+    def NIPA(
+        self,
+        TableName: str,
+        Frequency: str = 'Q',
+        Year: str = 'X',
+        verbose: bool = False,
+        includeIndentations: bool = True,
+        **kwargs: Any
+    ) -> Any:
+        '''
+        Query NIPA (National Income and Product Accounts) data.
+        
+        Args:
+            TableName: NIPA table name (e.g., 'T10101')
+            Frequency: 'Q' (quarterly), 'A' (annual), or 'M' (monthly)
+            Year: specific year or 'X' for all years
+            verbose: if True, return dict with dataFrame, request, and code
+            includeIndentations: if True, add indentation markers to LineDescription
+            
+        Returns:
+            pandas DataFrame or dict (if verbose=True)
+            
+        Example:
+            data.NIPA('T10101')
+            data.NIPA('T10101', Frequency='A', Year='X', verbose=True)
+        '''
+        self._lastCalledDriver = self._nipa_driver
+        return self._nipa_driver(
+            TableName=TableName,
+            Frequency=Frequency,
+            Year=Year,
+            verbose=verbose,
+            includeIndentations=includeIndentations,
+            **kwargs
+        )
+    
+    def NIUnderlyingDetail(
+        self,
+        TableName: str,
+        Frequency: str = 'Q',
+        Year: str = 'X',
+        verbose: bool = False,
+        **kwargs: Any
+    ) -> Any:
+        '''
+        Query National Income underlying detail tables.
+        
+        Args:
+            TableName: table name
+            Frequency: 'Q', 'A', or 'M'
+            Year: specific year or 'X' for all years
+            verbose: if True, return dict with dataFrame, request, and code
+            
+        Returns:
+            pandas DataFrame or dict (if verbose=True)
+        '''
+        self._lastCalledDriver = self._nipa_driver
+        return self._nipa_driver['NIUnderlyingDetail'](
+            TableName=TableName,
+            Frequency=Frequency,
+            Year=Year,
+            verbose=verbose,
+            **kwargs
+        )
+    
+    # =========================================================================
+    # National Accounts queries
+    # =========================================================================
+    
+    def fixedAssets(self, TableName: str, Year: str, verbose: bool = False, **kwargs: Any) -> Any:
+        '''Query Fixed Assets data.'''
+        self._lastCalledDriver = self._national_accounts_driver
+        return self._national_accounts_driver(
+            TableName=TableName,
+            Year=Year,
+            verbose=verbose,
+            **kwargs
+        )
+    
+    # =========================================================================
+    # International Transactions queries
+    # =========================================================================
+    
+    def ITA(
+        self,
+        Indicator: str,
+        AreaOrCountry: str,
+        Frequency: str,
+        Year: str,
+        verbose: bool = False,
+        **kwargs: Any
+    ) -> Any:
+        '''Query International Transactions Accounts.'''
+        self._lastCalledDriver = self._international_transactions_driver
+        return self._international_transactions_driver(
+            Indicator=Indicator,
+            AreaOrCountry=AreaOrCountry,
+            Frequency=Frequency,
+            Year=Year,
+            verbose=verbose,
+            **kwargs
+        )
+    
+    # =========================================================================
+    # International Investment Position queries
+    # =========================================================================
+    
+    def IIP(
+        self,
+        TypeOfInvestment: str,
+        Component: str,
+        Frequency: str,
+        Year: str,
+        verbose: bool = False,
+        **kwargs: Any
+    ) -> Any:
+        '''Query International Investment Position.'''
+        self._lastCalledDriver = self._international_investment_driver
+        return self._international_investment_driver(
+            TypeOfInvestment=TypeOfInvestment,
+            Component=Component,
+            Frequency=Frequency,
+            Year=Year,
+            verbose=verbose,
+            **kwargs
+        )
+    
+    # =========================================================================
+    # Industry queries
+    # =========================================================================
+    
+    def GDPbyIndustry(
+        self,
+        Industry: str,
+        TableID: str,
+        Frequency: str,
+        Year: str,
+        verbose: bool = False,
+        **kwargs: Any
+    ) -> Any:
+        '''Query GDP by Industry.'''
+        self._lastCalledDriver = self._industry_driver
+        return self._industry_driver['GDPbyIndustry'](
+            Industry=Industry,
+            TableID=TableID,
+            Frequency=Frequency,
+            Year=Year,
+            verbose=verbose,
+            **kwargs
+        )
+    
+    def UnderlyingGDPbyIndustry(
+        self,
+        Industry: str,
+        TableID: str,
+        Frequency: str,
+        Year: str,
+        verbose: bool = False,
+        **kwargs: Any
+    ) -> Any:
+        '''Query Underlying GDP by Industry.'''
+        self._lastCalledDriver = self._industry_driver
+        return self._industry_driver['UnderlyingGDPbyIndustry'](
+            Industry=Industry,
+            TableID=TableID,
+            Frequency=Frequency,
+            Year=Year,
+            verbose=verbose,
+            **kwargs
+        )
+    
+    # =========================================================================
+    # Regional queries
+    # =========================================================================
+    
+    def Regional(
+        self,
+        GeoFips: str,
+        LineCode: str,
+        TableName: str,
+        Year: str,
+        verbose: bool = False,
+        **kwargs: Any
+    ) -> Any:
+        '''Query Regional economic data.'''
+        self._lastCalledDriver = self._regional_driver
+        return self._regional_driver(
+            GeoFips=GeoFips,
+            LineCode=LineCode,
+            TableName=TableName,
+            Year=Year,
+            verbose=verbose,
+            **kwargs
+        )
+    
+    # =========================================================================
+    # Input-Output queries
+    # =========================================================================
+    
+    def InputOutput(self, TableID: str, Year: str, verbose: bool = False, **kwargs: Any) -> Any:
+        '''Query Input-Output tables.'''
+        self._lastCalledDriver = self._input_output_driver
+        return self._input_output_driver(
+            TableID=TableID,
+            Year=Year,
+            verbose=verbose,
+            **kwargs
+        )
+    
+    # =========================================================================
+    # International Services Trade queries
+    # =========================================================================
+    
+    def IntlServTrade(
+        self,
+        TypeOfService: str,
+        TradeDirection: str,
+        Affiliation: str,
+        AreaOrCountry: str,
+        Year: str,
+        verbose: bool = False,
+        **kwargs: Any
+    ) -> Any:
+        '''Query International Services Trade.'''
+        self._lastCalledDriver = self._international_services_driver
+        return self._international_services_driver(
+            TypeOfService=TypeOfService,
+            TradeDirection=TradeDirection,
+            Affiliation=Affiliation,
+            AreaOrCountry=AreaOrCountry,
+            Year=Year,
+            verbose=verbose,
+            **kwargs
+        )
+    
+    def IntlServSTA(
+        self,
+        Channel: str = 'ALL',
+        Destination: str = 'ALL',
+        Industry: str = 'ALL',
+        AreaOrCountry: str = 'AllCountries',
+        Year: str = 'ALL',
+        verbose: bool = False,
+        **kwargs: Any
+    ) -> Any:
+        '''
+        Query International Services Supplied Through Affiliates.
+        
+        Args:
+            Channel: channel requested (default 'ALL')
+            Destination: destination requested (default 'ALL')
+            Industry: industry requested (default 'ALL')
+            AreaOrCountry: area or country requested (default 'AllCountries')
+            Year: year requested (default 'ALL')
+            verbose: if True, return dict with dataFrame, request, and code
+            
+        Returns:
+            pandas DataFrame or dict (if verbose=True)
+        '''
+        self._lastCalledDriver = self._international_services_driver
+        return self._international_services_driver['IntlServSTA'](
+            Channel=Channel,
+            Destination=Destination,
+            Industry=Industry,
+            AreaOrCountry=AreaOrCountry,
+            Year=Year,
+            verbose=verbose,
+            **kwargs
+        )
+    
+    # =========================================================================
+    # Multinational Enterprises queries
+    # =========================================================================
+    
+    def MNE(
+        self,
+        Frequency: str,
+        TableID: str,
+        DirectionOfInvestment: str,
+        OwnershipLevel: str,
+        NonbankAffiliatesOnly: str,
+        Classification: str,
+        Country: str,
+        Industry: str,
+        Year: str,
+        State: str,
+        SeriesID: str,
+        GetFootnotes: str,
+        Investment: str,
+        ParentInvestment: str,
+        verbose: bool = False,
+        **kwargs: Any
+    ) -> Any:
+        '''Query Activities of U.S. Multinational Enterprises.'''
+        self._lastCalledDriver = self._multinational_driver
+        return self._multinational_driver(
+            Frequency=Frequency,
+            TableID=TableID,
+            DirectionOfInvestment=DirectionOfInvestment,
+            OwnershipLevel=OwnershipLevel,
+            NonbankAffiliatesOnly=NonbankAffiliatesOnly,
+            Classification=Classification,
+            Country=Country,
+            Industry=Industry,
+            Year=Year,
+            State=State,
+            SeriesID=SeriesID,
+            GetFootnotes=GetFootnotes,
+            Investment=Investment,
+            ParentInvestment=ParentInvestment,
+            verbose=verbose,
+            **kwargs
+        )
+    
+    # =========================================================================
+    
+    # =========================================================================
+    # NIPA Summary (not API - custom computation)
+    # =========================================================================
+    
+    def NIPASummary(self, year: str, frequency: str, verbose: bool = False, **kwargs: Any) -> Any:
+        '''
+        Get NIPA Summary tables - overall view of NIPA accounts.
+        
+        Queries 6 NIPA accounts (Domestic Income, Private Enterprise, Personal Income,
+        Government, Foreign Transactions, Domestic Capital) and returns source/uses tables.
+        
+        Args:
+            year: year for summary (e.g., '2010')
+            frequency: 'Q' or 'A'
+            verbose: if True, return dict with dataFrame and request metadata
+            
+        Returns:
+            List of DataFrames (one per account source/use) or dict (if verbose)
+            
+        Example:
+            data.NIPASummary('2010', 'A')
+        '''
+        from datapungibea.config import CFGnipaSummary
+        import pandas as pd
+        
+        logger.info(f"Generating NIPA Summary for year={year}, frequency={frequency}")
+        
+        output = {'dataFrame': [], 'request': []}
+        
+        for account_name, account_config in CFGnipaSummary.tabparams.items():
+            logger.debug(f"Processing {account_name}")
+            
+            # Get source data
+            source_table = account_config['source']
+            source_df = self._getAccountUseOrSource(
+                tableName=source_table['tableName'],
+                year=year,
+                frequency=frequency if account_name != 'Account 2' else 'A',  # Account 2 is annual only
+                tableEntries=source_table['tableEntries']
+            )
+            source_df.insert(0, 'Account', f"{account_name} sources")
+            source_df.insert(1, 'Heading', account_config['meta']['heading'])
+            source_df.attrs['meta'] = {'caption': account_config['meta']['caption'], 'type': 'sources'}
+            
+            # Get uses data
+            uses_table = account_config['uses']
+            uses_df = self._getAccountUseOrSource(
+                tableName=uses_table['tableName'],
+                year=year,
+                frequency=frequency if account_name != 'Account 2' else 'A',
+                tableEntries=uses_table['tableEntries']
+            )
+            uses_df.insert(0, 'Account', f"{account_name} uses")
+            uses_df.insert(1, 'Heading', account_config['meta']['heading'])
+            uses_df.attrs['meta'] = {'caption': account_config['meta']['caption'], 'type': 'uses'}
+            
+            output['dataFrame'].append(source_df)
+            output['dataFrame'].append(uses_df)
+        
+        if verbose:
+            return output
+        else:
+            return output['dataFrame']
+    
+    def _getAccountUseOrSource(
+        self,
+        tableName: str,
+        year: str,
+        frequency: str,
+        tableEntries: list
+    ) -> pd.DataFrame:
+        '''Helper method to get and filter NIPA table data for summary.'''
+        import pandas as pd
+
+        # Query the NIPA table without indentation (get clean LineDescription)
+        readTable = self.NIPA(TableName=tableName, Frequency=frequency, Year=year, includeIndentations=False)
+        readTable = readTable.reset_index()
+
+        # Create restriction DataFrame
+        restrict = pd.DataFrame(tableEntries)
+
+        # Merge to get only the rows we need
+        output = pd.merge(restrict, readTable, on='SeriesCode', how='left')
+
+        # Sometimes there are two lines with the same SeriesCode on readTable 
+        # (Account 5 - Foreeign Transactions Current Account
+        #              Balance on Current account: A124RC)
+        # this leads to sequential duplication of rows - removing them here
+        output = output.loc[output['SeriesCode'].shift() != output['SeriesCode']]
+
+        # Apply indentation from config by adding dashes to LineDescription
+        output['LineDescription'] = output.apply(
+            lambda x: '-' * int(x['indentation']) + str(x.get('LineDescription', '')),
+            axis=1
+        )
+
+        # Clean up columns
+        output = output.drop(['LineNumber'], axis=1, errors='ignore')
+        
+        # Set index for sorting
+        output = output.set_index(['LineDescription', 'SeriesCode'])
+        output = output.reset_index()
+        
+        return output
+    
+    # =========================================================================
+    # Utility methods
+    # =========================================================================
+    
+    def _clipcode(self) -> None:
+        '''Copy the last query's code to clipboard.'''
+        if self._lastCalledDriver and hasattr(self._lastCalledDriver, '_ETDB'):
+            from datapungibea.codegen import transformIncludeCodeSnippet
+            codegen = transformIncludeCodeSnippet()
+            codegen.clipcode(self._lastCalledDriver._lastLoad)
+        else:
+            logger.warning('Get data using a driver first, e.g., data.NIPA("T10101", verbose=True)')
+    
+    def _docDriver(self, driverName: str) -> str:
+        '''Get documentation for a specific driver.'''
+        driver_map = {
+            'NIPA': self._nipa_driver,
+            'fixedAssets': self._national_accounts_driver,
+            'ITA': self._international_transactions_driver,
+            'IIP': self._international_investment_driver,
+            'GDPbyIndustry': self._industry_driver,
+            'Regional': self._regional_driver,
+            'InputOutput': self._input_output_driver,
+            'IntlServTrade': self._international_services_driver,
+            'MNE': self._multinational_driver,
+        }
+        
+        if driverName in driver_map:
+            return str(driver_map[driverName])
+        else:
+            return f"Driver '{driverName}' not found. Available: {list(driver_map.keys())}"
+
+
+if __name__ == '__main__':
+    # Test basic functionality
+    import sys
+    logging.basicConfig(level=logging.INFO)
+    
+    try:
+        d = data()
+        print(d)
+        print("\nTrying datasetlist...")
+        print(d.datasetlist())
+    except Exception as e:
+        print(f"Error: {e}")
+        sys.exit(1)