panelbox 0.2.0__py3-none-any.whl

panelbox/validation/serial_correlation/baltagi_wu.py

@@ -0,0 +1,220 @@
+"""
+Baltagi-Wu LBI test for serial correlation in panel data.
+
+LBI = Locally Best Invariant
+
+References
+----------
+Baltagi, B. H., & Wu, P. X. (1999). Unequally Spaced Panel Data Regressions
+with AR(1) Disturbances. Econometric Theory, 15(6), 814-823.
+
+Baltagi, B. H., & Li, Q. (1995). Testing AR(1) Against MA(1) Disturbances
+in an Error Component Model. Journal of Econometrics, 68(1), 133-151.
+"""
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+from panelbox.validation.base import ValidationTest, ValidationTestResult
+
+
+class BaltagiWuTest(ValidationTest):
+    """
+    Baltagi-Wu LBI test for first-order serial correlation in panel data.
+
+    This test is designed for unbalanced panels and tests for AR(1)
+    serial correlation in the idiosyncratic errors.
+
+    H0: No first-order serial correlation (rho = 0)
+    H1: AR(1) serial correlation present (rho ≠ 0)
+
+    The test is based on a modified Durbin-Watson statistic that
+    accounts for unbalanced panel structure.
+
+    Notes
+    -----
+    The test statistic is:
+
+    LBI = sum_i sum_t (e_it - e_{i,t-1})² / sum_i sum_t e_it²
+
+    Under H0, LBI ≈ 2 (similar to Durbin-Watson).
+    LBI < 2 suggests positive autocorrelation.
+    LBI > 2 suggests negative autocorrelation.
+
+    Unlike the standard Durbin-Watson test, the Baltagi-Wu test:
+    - Works with unbalanced panels
+    - Accounts for heterogeneous time series lengths
+    - Provides asymptotic normal distribution under H0
+
+    Examples
+    --------
+    >>> from panelbox.models.static.fixed_effects import FixedEffects
+    >>> fe = FixedEffects("y ~ x1 + x2", data, "entity", "time")
+    >>> results = fe.fit()
+    >>>
+    >>> from panelbox.validation.serial_correlation.baltagi_wu import BaltagiWuTest
+    >>> test = BaltagiWuTest(results)
+    >>> result = test.run()
+    >>> print(result)
+    """
+
+    def __init__(self, results: 'PanelResults'):
+        """
+        Initialize Baltagi-Wu test.
+
+        Parameters
+        ----------
+        results : PanelResults
+            Results from panel model estimation
+        """
+        super().__init__(results)
+
+    def run(self, alpha: float = 0.05) -> ValidationTestResult:
+        """
+        Run Baltagi-Wu test for serial correlation.
+
+        Parameters
+        ----------
+        alpha : float, default=0.05
+            Significance level
+
+        Returns
+        -------
+        ValidationTestResult
+            Test results
+
+        Raises
+        ------
+        ValueError
+            If panel has fewer than 2 time periods per entity
+
+        Notes
+        -----
+        The test uses asymptotic normality of the LBI statistic.
+        For large N and T, (LBI - 2) is approximately normally distributed.
+
+        The transformation to a standard normal test statistic uses:
+        z = (LBI - 2) / sqrt(var(LBI))
+
+        where var(LBI) is estimated from the panel structure.
+        """
+        # Get residuals with entity and time structure
+        resid_df = self._prepare_residual_data()
+
+        # Sort by entity and time
+        resid_df = resid_df.sort_values(['entity', 'time'])
+
+        # Compute lagged residuals within each entity
+        resid_df['resid_lag'] = resid_df.groupby('entity')['resid'].shift(1)
+
+        # Compute differences
+        resid_df['resid_diff'] = resid_df['resid'] - resid_df['resid_lag']
+
+        # Drop missing values (first observation of each entity)
+        resid_df_clean = resid_df.dropna(subset=['resid_diff', 'resid_lag'])
+
+        if len(resid_df_clean) == 0:
+            raise ValueError(
+                "No valid observations after computing differences. "
+                "Ensure each entity has at least 2 time periods."
+            )
+
+        # Compute LBI statistic
+        # LBI = sum(diff²) / sum(resid²)
+        numerator = np.sum(resid_df_clean['resid_diff'] ** 2)
+        denominator = np.sum(resid_df['resid'] ** 2)  # Use all residuals
+
+        if denominator == 0:
+            raise ValueError("Sum of squared residuals is zero (perfect fit)")
+
+        lbi_stat = numerator / denominator
+
+        # Compute approximate variance of LBI
+        # Under H0: E[LBI] ≈ 2
+        # Var(LBI) ≈ 4 / (N*T_bar) where T_bar is average time periods
+        n_entities = resid_df['entity'].nunique()
+        n_obs_total = len(resid_df)
+        t_bar = n_obs_total / n_entities
+
+        # More refined variance estimate
+        # Account for unbalanced structure
+        entity_counts = resid_df.groupby('entity').size()
+        t_i = entity_counts.values
+
+        # Variance formula for unbalanced panels
+        # Var(LBI) ≈ 4 * sum(1/T_i) / N
+        var_lbi = 4 * np.sum(1 / t_i) / n_entities
+
+        # Standard error
+        se_lbi = np.sqrt(var_lbi)
+
+        # Test statistic: (LBI - 2) / SE(LBI)
+        # Under H0, this is approximately N(0,1)
+        if se_lbi == 0:
+            raise ValueError("Standard error is zero")
+
+        z_stat = (lbi_stat - 2) / se_lbi
+
+        # Two-sided p-value
+        pvalue = 2 * (1 - stats.norm.cdf(abs(z_stat)))
+
+        # Metadata
+        # Estimate rho from LBI: rho ≈ 1 - LBI/2
+        rho_estimate = 1 - lbi_stat / 2
+
+        metadata = {
+            'lbi_statistic': float(lbi_stat),
+            'z_statistic': float(z_stat),
+            'rho_estimate': float(rho_estimate),
+            'n_entities': int(n_entities),
+            'n_obs_total': int(n_obs_total),
+            'n_obs_used': len(resid_df_clean),
+            'avg_time_periods': float(t_bar),
+            'min_time_periods': int(t_i.min()),
+            'max_time_periods': int(t_i.max()),
+            'variance_lbi': float(var_lbi),
+            'se_lbi': float(se_lbi),
+            'interpretation': (
+                'LBI < 2: positive autocorrelation, '
+                'LBI ≈ 2: no autocorrelation, '
+                'LBI > 2: negative autocorrelation'
+            )
+        }
+
+        result = ValidationTestResult(
+            test_name="Baltagi-Wu LBI Test for Serial Correlation",
+            statistic=z_stat,
+            pvalue=pvalue,
+            null_hypothesis="No first-order serial correlation",
+            alternative_hypothesis="First-order serial correlation present",
+            alpha=alpha,
+            df=None,  # Asymptotic test, no df
+            metadata=metadata
+        )
+
+        return result
+
+    def _prepare_residual_data(self) -> pd.DataFrame:
+        """
+        Prepare residual data with entity and time identifiers.
+
+        Returns
+        -------
+        pd.DataFrame
+            DataFrame with columns: entity, time, resid
+        """
+        if hasattr(self.results, 'entity_index') and hasattr(self.results, 'time_index'):
+            resid_flat = self.resid.ravel() if hasattr(self.resid, 'ravel') else self.resid
+
+            resid_df = pd.DataFrame({
+                'entity': self.results.entity_index,
+                'time': self.results.time_index,
+                'resid': resid_flat
+            })
+
+            return resid_df
+        else:
+            raise AttributeError(
+                "Results object must have 'entity_index' and 'time_index' attributes"
+            )

panelbox/validation/serial_correlation/breusch_godfrey.py

@@ -0,0 +1,260 @@
+"""
+Breusch-Godfrey LM test for serial correlation in panel data.
+
+References
+----------
+Breusch, T. S. (1978). Testing for autocorrelation in dynamic linear models.
+Australian Economic Papers, 17(31), 334-355.
+
+Godfrey, L. G. (1978). Testing against general autoregressive and moving average
+error models when the regressors include lagged dependent variables.
+Econometrica, 46(6), 1293-1301.
+"""
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+from panelbox.validation.base import ValidationTest, ValidationTestResult
+
+
+class BreuschGodfreyTest(ValidationTest):
+    """
+    Breusch-Godfrey LM test for serial correlation.
+    
+    Tests the null hypothesis of no serial correlation against the
+    alternative of AR(p) serial correlation in the errors.
+    
+    H0: No serial correlation
+    H1: AR(p) serial correlation present
+    
+    The test regresses the residuals on lagged residuals and the original
+    regressors, then tests if the lagged residuals are jointly significant.
+    
+    Notes
+    -----
+    Unlike the Durbin-Watson test, the BG test:
+    - Can test for higher-order serial correlation
+    - Is valid when regressors include lagged dependent variables
+    - Provides an LM test statistic ~ Chi2(p)
+    
+    For panel data, the test is applied accounting for the panel structure.
+    
+    Examples
+    --------
+    >>> from panelbox.models.static.fixed_effects import FixedEffects
+    >>> fe = FixedEffects("y ~ x1 + x2", data, "entity", "time")
+    >>> results = fe.fit()
+    >>> 
+    >>> from panelbox.validation.serial_correlation.breusch_godfrey import BreuschGodfreyTest
+    >>> test = BreuschGodfreyTest(results)
+    >>> result = test.run(lags=1)  # Test for AR(1)
+    >>> print(result)
+    """
+    
+    def __init__(self, results: 'PanelResults'):
+        """
+        Initialize Breusch-Godfrey test.
+        
+        Parameters
+        ----------
+        results : PanelResults
+            Results from panel model estimation
+        """
+        super().__init__(results)
+    
+    def run(self, lags: int = 1, alpha: float = 0.05) -> ValidationTestResult:
+        """
+        Run Breusch-Godfrey LM test for serial correlation.
+        
+        Parameters
+        ----------
+        lags : int, default=1
+            Number of lags to test (order of AR process)
+        alpha : float, default=0.05
+            Significance level
+        
+        Returns
+        -------
+        ValidationTestResult
+            Test results
+        
+        Raises
+        ------
+        ValueError
+            If required data is not available or lags < 1
+        
+        Notes
+        -----
+        The test procedure:
+        1. Obtain residuals from original model
+        2. Regress residuals on lagged residuals (up to lag p) and original X
+        3. Compute LM = n*R² from this auxiliary regression
+        4. Compare to Chi2(p) distribution
+        """
+        if lags < 1:
+            raise ValueError(f"lags must be >= 1, got {lags}")
+        
+        # Get residuals with entity and time structure
+        resid_df = self._prepare_residual_data()
+        
+        # Get design matrix
+        X = self._get_design_matrix()
+        
+        if X is None:
+            raise ValueError(
+                "Design matrix not available for Breusch-Godfrey test"
+            )
+        
+        # Create lagged residuals
+        resid_df = resid_df.sort_values(['entity', 'time'])
+        
+        for lag in range(1, lags + 1):
+            resid_df[f'resid_lag{lag}'] = resid_df.groupby('entity')['resid'].shift(lag)
+        
+        # Drop missing values (first lags obs per entity)
+        lag_cols = [f'resid_lag{i}' for i in range(1, lags + 1)]
+        resid_df = resid_df.dropna(subset=lag_cols)
+        
+        if len(resid_df) == 0:
+            raise ValueError("No valid observations after creating lags")
+        
+        # Get residuals and lagged residuals as arrays
+        resid = resid_df['resid'].values
+        X_lags = resid_df[lag_cols].values
+        
+        # Match X to the reduced sample (after dropping NAs)
+        # We need to align X with the residuals we kept
+        # This is tricky - we need the original indices
+        
+        # Simpler approach: use all X but only for non-missing resid indices
+        if len(X) == len(resid) + lags * resid_df['entity'].nunique():
+            # Need to match indices properly
+            # For now, assume X already matches the full data
+            # and we need to select the rows that correspond to non-missing resid
+            
+            # Get indices of non-missing residuals
+            valid_indices = resid_df.index.values
+            
+            # This assumes resid_df index corresponds to original data indices
+            if max(valid_indices) < len(X):
+                X_matched = X[valid_indices, :]
+            else:
+                # Fallback: use last len(resid) rows
+                X_matched = X[-len(resid):, :]
+        else:
+            # Assume X and resid are aligned
+            X_matched = X[:len(resid), :]
+        
+        # Auxiliary regression: resid on [X, resid_lag1, ..., resid_lagp]
+        X_aug = np.column_stack([X_matched, X_lags])
+        
+        # OLS
+        try:
+            XtX = X_aug.T @ X_aug
+            Xty = X_aug.T @ resid
+            beta_aux = np.linalg.solve(XtX, Xty)
+        except np.linalg.LinAlgError:
+            beta_aux = np.linalg.lstsq(X_aug, resid, rcond=None)[0]
+        
+        # Fitted values
+        fitted_aux = X_aug @ beta_aux
+
+        # R² from auxiliary regression using explained sum of squares
+        # This is more numerically stable
+        mean_resid = np.mean(resid)
+        SST = np.sum((resid - mean_resid) ** 2)
+        SSE = np.sum((fitted_aux - mean_resid) ** 2)
+
+        if SST > 0:
+            R2_aux = SSE / SST
+        else:
+            R2_aux = 0.0
+
+        # Ensure R² is in [0, 1]
+        R2_aux = np.clip(R2_aux, 0.0, 1.0)
+
+        # LM statistic for panel data
+        # IMPORTANT: For panel data, we use the number of CROSS-SECTIONAL UNITS (N)
+        # not the total number of observations (N*T) or the reduced sample size.
+        #
+        # The Breusch-Godfrey test for panels (pbgtest in plm) uses:
+        # LM = N * R²
+        # where N is the number of cross-sectional units.
+        #
+        # This is different from the time-series version which uses n = T observations.
+        #
+        # Reference: Baltagi & Li (1995), "Testing AR(1) against MA(1) disturbances
+        # in an error component model"
+
+        n_entities = resid_df['entity'].nunique()
+        lm_stat = n_entities * R2_aux
+
+        # Sanity check
+        if lm_stat < 0:
+            lm_stat = 0.0
+
+        # Degrees of freedom = number of lags
+        df = lags
+        
+        # P-value
+        pvalue = 1 - stats.chi2.cdf(lm_stat, df)
+        
+        # Metadata
+        n_obs = len(resid)
+        metadata = {
+            'lags': lags,
+            'R2_auxiliary': R2_aux,
+            'n_obs_auxiliary': n_obs,
+            'n_entities': n_entities,
+            'note': 'Panel BG test uses LM = N * R² where N = number of entities'
+        }
+        
+        result = ValidationTestResult(
+            test_name=f"Breusch-Godfrey LM Test for Serial Correlation (AR({lags}))",
+            statistic=lm_stat,
+            pvalue=pvalue,
+            null_hypothesis="No serial correlation",
+            alternative_hypothesis=f"AR({lags}) serial correlation present",
+            alpha=alpha,
+            df=df,
+            metadata=metadata
+        )
+        
+        return result
+    
+    def _prepare_residual_data(self) -> pd.DataFrame:
+        """Prepare residual data with entity and time identifiers."""
+        if hasattr(self.results, 'entity_index') and hasattr(self.results, 'time_index'):
+            resid_flat = self.resid.ravel() if hasattr(self.resid, 'ravel') else self.resid
+            
+            resid_df = pd.DataFrame({
+                'entity': self.results.entity_index,
+                'time': self.results.time_index,
+                'resid': resid_flat
+            })
+            
+            return resid_df
+        else:
+            raise AttributeError(
+                "Results object must have 'entity_index' and 'time_index' attributes"
+            )
+    
+    def _get_design_matrix(self) -> np.ndarray:
+        """Get the design matrix X."""
+        if not hasattr(self.results, '_model'):
+            return None
+        
+        model = self.results._model
+        
+        if hasattr(model, 'formula_parser') and hasattr(model, 'data'):
+            try:
+                _, X = model.formula_parser.build_design_matrices(
+                    model.data.data,
+                    return_type='array'
+                )
+                return X
+            except Exception:
+                pass
+        
+        return None

panelbox/validation/serial_correlation/wooldridge_ar.py

@@ -0,0 +1,200 @@
+"""
+Wooldridge test for autocorrelation in panel data.
+
+References
+----------
+Wooldridge, J. M. (2002). Econometric Analysis of Cross Section and Panel Data.
+MIT Press, Section 10.4.1.
+
+Stata command: xtserial
+"""
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+from panelbox.validation.base import ValidationTest, ValidationTestResult
+
+
+class WooldridgeARTest(ValidationTest):
+    """
+    Wooldridge test for first-order autocorrelation in panel data.
+    
+    This test is specifically designed for fixed effects models and tests
+    for AR(1) autocorrelation in the idiosyncratic errors.
+    
+    The test is based on regressing the first-differenced residuals on their
+    own lag and testing if the coefficient equals -0.5 (which is the value
+    under H0 of no serial correlation).
+    
+    Notes
+    -----
+    The test statistic is approximately distributed as F(1, N-1) under the null
+    of no first-order serial correlation.
+    
+    This test requires at least T >= 3 time periods.
+    
+    Examples
+    --------
+    >>> from panelbox.models.static.fixed_effects import FixedEffects
+    >>> fe = FixedEffects("y ~ x1 + x2", data, "entity", "time")
+    >>> results = fe.fit()
+    >>> 
+    >>> from panelbox.validation.serial_correlation.wooldridge_ar import WooldridgeARTest
+    >>> test = WooldridgeARTest(results)
+    >>> result = test.run()
+    >>> print(result)
+    """
+    
+    def __init__(self, results: 'PanelResults'):
+        """
+        Initialize Wooldridge AR test.
+        
+        Parameters
+        ----------
+        results : PanelResults
+            Results from panel model estimation (preferably Fixed Effects)
+        """
+        super().__init__(results)
+        
+        # Check if model is suitable
+        if 'Fixed Effects' not in self.model_type:
+            import warnings
+            warnings.warn(
+                "Wooldridge test is designed for Fixed Effects models. "
+                f"Current model: {self.model_type}"
+            )
+    
+    def run(self, alpha: float = 0.05) -> ValidationTestResult:
+        """
+        Run Wooldridge test for AR(1) autocorrelation.
+        
+        Parameters
+        ----------
+        alpha : float, default=0.05
+            Significance level
+        
+        Returns
+        -------
+        ValidationTestResult
+            Test results
+        
+        Raises
+        ------
+        ValueError
+            If panel has fewer than 3 time periods
+        """
+        # Get residuals as DataFrame with entity and time info
+        # We need to reconstruct the panel structure
+        resid_df = self._prepare_residual_data()
+        
+        # Check minimum time periods
+        min_T = resid_df.groupby('entity').size().min()
+        if min_T < 3:
+            raise ValueError(
+                f"Wooldridge test requires at least 3 time periods. "
+                f"Minimum found: {min_T}"
+            )
+        
+        # Compute first differences of residuals
+        resid_df = resid_df.sort_values(['entity', 'time'])
+        resid_df['resid_diff'] = resid_df.groupby('entity')['resid'].diff()
+        resid_df['resid_diff_lag'] = resid_df.groupby('entity')['resid_diff'].shift(1)
+        
+        # Drop missing values (first two obs per entity are lost)
+        resid_df = resid_df.dropna(subset=['resid_diff', 'resid_diff_lag'])
+        
+        if len(resid_df) == 0:
+            raise ValueError("No valid observations after differencing")
+        
+        # Regression: Δe_it on Δe_{i,t-1}
+        y = resid_df['resid_diff'].values
+        X = resid_df['resid_diff_lag'].values
+        
+        # OLS regression
+        n = len(y)
+        beta = np.sum(X * y) / np.sum(X * X)
+        
+        # Residuals
+        fitted = beta * X
+        resid_reg = y - fitted
+        
+        # Standard error of beta
+        s2 = np.sum(resid_reg ** 2) / (n - 1)
+        se_beta = np.sqrt(s2 / np.sum(X * X))
+        
+        # Test H0: beta = -0.5 (no serial correlation)
+        # Under H0, if no autocorrelation, E[Δe_it * Δe_{i,t-1}] = -sigma²/2
+        # So coefficient should be -0.5
+        t_stat = (beta - (-0.5)) / se_beta
+        
+        # F statistic (F = t²)
+        f_stat = t_stat ** 2
+        
+        # P-value from F distribution
+        # Number of entities
+        n_entities = resid_df['entity'].nunique()
+        df_num = 1
+        df_denom = n_entities - 1
+        
+        pvalue = 1 - stats.f.cdf(f_stat, df_num, df_denom)
+        
+        # Metadata
+        metadata = {
+            'coefficient': beta,
+            'std_error': se_beta,
+            't_statistic': t_stat,
+            'n_entities': n_entities,
+            'n_obs_used': n
+        }
+        
+        result = ValidationTestResult(
+            test_name="Wooldridge Test for Autocorrelation",
+            statistic=f_stat,
+            pvalue=pvalue,
+            null_hypothesis="No first-order autocorrelation",
+            alternative_hypothesis="First-order autocorrelation present",
+            alpha=alpha,
+            df=(df_num, df_denom),
+            metadata=metadata
+        )
+        
+        return result
+    
+    def _prepare_residual_data(self) -> pd.DataFrame:
+        """
+        Prepare residual data with entity and time identifiers.
+        
+        Returns
+        -------
+        pd.DataFrame
+            DataFrame with columns: entity, time, resid
+        """
+        # Try to get entity and time from model metadata
+        # This assumes the model stored the original data structure
+        
+        # For now, we'll try to extract from the results object
+        # This requires that the model kept track of entity/time indices
+        
+        # If results has entity_index and time_index attributes
+        if hasattr(self.results, 'entity_index') and hasattr(self.results, 'time_index'):
+            # Ensure resid is 1D
+            resid_flat = self.resid.ravel() if hasattr(self.resid, 'ravel') else self.resid
+
+            resid_df = pd.DataFrame({
+                'entity': self.results.entity_index,
+                'time': self.results.time_index,
+                'resid': resid_flat
+            })
+        else:
+            # Fallback: try to reconstruct from model's data attribute
+            # This assumes the results object has reference to the original model
+            # which has the PanelData object
+            
+            # For now, raise informative error
+            raise AttributeError(
+                "Results object must have 'entity_index' and 'time_index' attributes. "
+                "Please ensure your model stores these during estimation."
+            )
+        
+        return resid_df

panelbox/validation/specification/__init__.py

@@ -0,0 +1,16 @@
+"""
+Specification tests for panel models.
+"""
+
+from panelbox.validation.specification.hausman import HausmanTest, HausmanTestResult
+from panelbox.validation.specification.mundlak import MundlakTest
+from panelbox.validation.specification.reset import RESETTest
+from panelbox.validation.specification.chow import ChowTest
+
+__all__ = [
+    'HausmanTest',
+    'HausmanTestResult',
+    'MundlakTest',
+    'RESETTest',
+    'ChowTest',
+]