taxcalc 5.3.0__py3-none-any.whl → 6.1.0__py3-none-any.whl

taxcalc/tests/test_puf_var_stats.py

@@ -1,194 +0,0 @@
-"""
-Test generates statistics for puf.csv variables.
-"""
-# CODING-STYLE CHECKS:
-# pycodestyle test_puf_var_stats.py
-# pylint --disable=locally-disabled test_puf_var_stats.py
-
-import os
-import json
-import copy
-import numpy as np
-import pandas as pd
-import pytest
-from taxcalc.policy import Policy
-from taxcalc.records import Records
-from taxcalc.calculator import Calculator
-
-
-def create_base_table(test_path):
-    """
-    Create and return base table.
-    """
-    # specify calculated variable names and descriptions
-    calc_dict = {'eitc': 'Federal EITC',
-                 'iitax': 'Federal income tax liability',
-                 'payrolltax': 'Payroll taxes (ee+er) for OASDI+HI',
-                 'c00100': 'Federal AGI',
-                 'c02500': 'OASDI benefits in AGI',
-                 'c04600': 'Post-phase-out personal exemption',
-                 'c21040': 'Itemized deduction that is phased out',
-                 'c04470': 'Post-phase-out itemized deduction',
-                 'c04800': 'Federal regular taxable income',
-                 'c05200': 'Regular tax on taxable income',
-                 'c07220': 'Child tax credit (adjusted)',
-                 'c11070': 'Extra child tax credit (refunded)',
-                 'c07180': 'Child care credit',
-                 'c09600': 'Federal AMT liability'}
-    # specify read variable names and descriptions
-    unused_var_set = set(['DSI', 'EIC',
-                          'h_seq', 'a_lineno', 'ffpos', 'fips', 'agi_bin',
-                          'FLPDYR', 'FLPDMO', 'f2441', 'f3800', 'f6251',
-                          'f8582', 'f8606', 'f8829', 'f8910', 'f8936', 'n20',
-                          'n24', 'n25', 'n30', 'PREP', 'SCHB', 'SCHCF', 'SCHE',
-                          'TFORM', 'IE', 'TXST', 'XFPT', 'XFST', 'XOCAH',
-                          'XOCAWH', 'XOODEP', 'XOPAR', 'XTOT', 'MARS', 'MIDR',
-                          'RECID', 'gender', 'wage_head', 'wage_spouse',
-                          'earnsplit', 'agedp1', 'agedp2', 'agedp3',
-                          's006', 's008', 's009', 'WSAMP', 'TXRT',
-                          'matched_weight', 'e00200p', 'e00200s',
-                          'e00900p', 'e00900s', 'e02100p', 'e02100s',
-                          'age_head', 'age_spouse',
-                          'nu18', 'n1820', 'n21',
-                          'ssi_ben', 'snap_ben', 'other_ben',
-                          'mcare_ben', 'mcaid_ben', 'vet_ben',
-                          'housing_ben', 'tanf_ben', 'wic_ben',
-                          'blind_head', 'blind_spouse',
-                          'PT_SSTB_income',
-                          'PT_binc_w2_wages',
-                          'PT_ubia_property'])
-    records_varinfo = Records(data=None)
-    read_vars = list(records_varinfo.USABLE_READ_VARS - unused_var_set)
-    # get read variable information from JSON file
-    rec_vars_path = os.path.join(test_path, '..', 'records_variables.json')
-    with open(rec_vars_path, 'r', encoding='utf-8') as rvfile:
-        read_var_dict = json.load(rvfile)
-    # create table_dict with sorted read vars followed by sorted calc vars
-    table_dict = {}
-    for var in sorted(read_vars):
-        if 'taxdata_puf' in read_var_dict['read'][var]['availability']:
-            table_dict[var] = read_var_dict['read'][var]['desc']
-        else:
-            pass
-    sorted_calc_vars = sorted(calc_dict.keys())
-    for var in sorted_calc_vars:
-        table_dict[var] = calc_dict[var]
-    # construct DataFrame table from table_dict
-    table = pd.DataFrame.from_dict(table_dict, orient='index')
-    table.columns = ['description']
-    return table
-
-
-def calculate_corr_stats(calc, table):
-    """
-    Calculate correlation coefficient matrix.
-    """
-    errmsg = ''
-    for varname1 in table.index:
-        var1 = calc.array(varname1)
-        var1_cc = []
-        for varname2 in table.index:
-            var2 = calc.array(varname2)
-            try:
-                cor = np.corrcoef(var1, var2)[0][1]
-            except FloatingPointError:
-                msg = f'corr-coef error for {varname1} and {varname2}\n'
-                errmsg += msg
-                cor = 9.99  # because could not compute it
-            var1_cc.append(cor)
-        table[varname1] = var1_cc
-    if errmsg:
-        raise ValueError('\n' + errmsg)
-
-
-def calculate_mean_stats(calc, table, year):
-    """
-    Calculate weighted means for year.
-    """
-    total_weight = calc.total_weight()
-    means = []
-    for varname in table.index:
-        wmean = calc.weighted_total(varname) / total_weight
-        means.append(wmean)
-    table[str(year)] = means
-
-
-def differences(new_filename, old_filename, stat_kind):
-    """
-    Return message string if differences detected by np.allclose();
-    otherwise return empty string.
-    """
-    new_df = pd.read_csv(new_filename)
-    old_df = pd.read_csv(old_filename)
-    diffs = False
-    if list(new_df.columns.values) == list(old_df.columns.values):
-        for col in new_df.columns[1:]:
-            if col == 'description':
-                continue  # skip description column
-            if not np.allclose(new_df[col], old_df[col]):
-                diffs = True
-    else:
-        diffs = True
-    if diffs:
-        new_name = os.path.basename(new_filename)
-        old_name = os.path.basename(old_filename)
-        msg = f'{stat_kind} RESULTS DIFFER:\n'
-        msg += '-------------------------------------------------'
-        msg += '-------------\n'
-        msg += f'--- NEW RESULTS IN {new_name} FILE ---\n'
-        msg += f'--- if new OK, copy {new_name} to\n'
-        msg += f'---                 {old_name}   \n'
-        msg += '---            and rerun test.      '
-        msg += '-------------------------------------------------'
-        msg += '-------------\n'
-    else:
-        msg = ''
-        os.remove(new_filename)
-    return msg
-
-
-MEAN_FILENAME = 'puf_var_wght_means_by_year.csv'
-CORR_FILENAME = 'puf_var_correl_coeffs_2016.csv'
-
-
-@pytest.mark.requires_pufcsv
-def test_puf_var_stats(tests_path, puf_fullsample):
-    """
-    Main logic of test.
-    """
-    # create a baseline Policy object containing 2017_law.json parameters
-    pre_tcja_jrf = os.path.join(tests_path, '..', 'reforms', '2017_law.json')
-    pre_tcja = Policy.read_json_reform(pre_tcja_jrf)
-    baseline_policy = Policy()
-    baseline_policy.implement_reform(pre_tcja)
-    # create a Calculator object using baseline_policy and full puf.csv sample
-    rec = Records(data=puf_fullsample)
-    calc = Calculator(policy=baseline_policy, records=rec, verbose=False)
-    # create base tables
-    table_mean = create_base_table(tests_path)
-    table_corr = copy.deepcopy(table_mean)
-    del table_corr['description']
-    # add statistics to tables
-    year_headers = ['description']
-    for year in range(Policy.JSON_START_YEAR, 2024 + 1):
-        assert year == calc.current_year
-        year_headers.append(str(year))
-        calc.calc_all()
-        calculate_mean_stats(calc, table_mean, year)
-        if year == 2016:
-            calculate_corr_stats(calc, table_corr)
-        if year < 2034:
-            calc.increment_year()
-    # write tables to new CSV files
-    mean_path = os.path.join(tests_path, MEAN_FILENAME + '-new')
-    table_mean.sort_index(inplace=True)
-    table_mean.to_csv(mean_path, header=year_headers, float_format='%8.0f')
-    corr_path = os.path.join(tests_path, CORR_FILENAME + '-new')
-    table_corr.sort_index(inplace=True)
-    table_corr.to_csv(corr_path, float_format='%8.2f',
-                      columns=table_corr.index)
-    # compare new and old CSV files for differences
-    mean_msg = differences(mean_path, mean_path[:-4], 'MEAN')
-    corr_msg = differences(corr_path, corr_path[:-4], 'CORR')
-    if mean_msg or corr_msg:
-        raise ValueError(mean_msg + corr_msg)

taxcalc/tests/test_pufcsv.py

@@ -1,328 +0,0 @@
-"""
-Tests of Tax-Calculator using puf.csv input.
-
-Note that the puf.csv file that is required to run this program has
-been constructed by the Tax-Calculator development team by merging
-information from the most recent publicly available IRS SOI PUF file
-and from the Census CPS file for the corresponding year.  If you have
-acquired from IRS the most recent SOI PUF file and want to execute
-this program, contact the Tax-Calculator development team to discuss
-your options.
-"""
-# CODING-STYLE CHECKS:
-# pycodestyle test_pufcsv.py
-# pylint --disable=locally-disabled test_pufcsv.py
-
-import os
-import json
-import pytest
-import numpy as np
-import pandas as pd
-from taxcalc.policy import Policy
-from taxcalc.records import Records
-from taxcalc.calculator import Calculator
-
-
-START_YEAR = 2017
-NUM_YEARS = 19
-
-
-@pytest.mark.pufcsv_agg
-@pytest.mark.requires_pufcsv
-def test_agg(tests_path, puf_fullsample):
-    """
-    Test Tax-Calculator aggregate taxes with no policy reform using
-    the full-sample puf.csv and a small sub-sample of puf.csv
-    """
-    # pylint: disable=too-many-locals,too-many-statements
-    nyrs = NUM_YEARS
-    # create a baseline Policy object with current-law policy parameters
-    baseline_policy = Policy()
-    # create a Records object (rec) containing all puf.csv input records
-    recs = Records(data=puf_fullsample)
-    # create a Calculator object using baseline policy and puf records
-    calc = Calculator(policy=baseline_policy, records=recs)
-    calc.advance_to_year(START_YEAR)
-    calc_start_year = calc.current_year
-    # create aggregate diagnostic table (adt) as a Pandas DataFrame object
-    adt = calc.diagnostic_table(nyrs).round(1)  # column labels are int
-    taxes_fullsample = adt.loc['Combined Liability ($b)']
-    # compare actual DataFrame, adt, with the expected DataFrame, edt
-    aggres_path = os.path.join(tests_path, 'pufcsv_agg_expect.csv')
-    edt = pd.read_csv(aggres_path, index_col=False)  # column labels are str
-    edt.drop('Unnamed: 0', axis='columns', inplace=True)
-    assert len(adt.columns.values) == len(edt.columns.values)
-    diffs = False
-    for icol in adt.columns.values:
-        if not np.allclose(adt[icol].values, edt[str(icol)].values):
-            diffs = True
-    if diffs:
-        new_filename = f'{aggres_path[:-10]}actual.csv'
-        adt.to_csv(new_filename, float_format='%.1f')
-        msg = 'PUFCSV AGG RESULTS DIFFER FOR FULL-SAMPLE\n'
-        msg += '-------------------------------------------------\n'
-        msg += '--- NEW RESULTS IN pufcsv_agg_actual.csv FILE ---\n'
-        msg += '--- if new OK, copy pufcsv_agg_actual.csv to  ---\n'
-        msg += '---                 pufcsv_agg_expect.csv     ---\n'
-        msg += '---            and rerun test.                ---\n'
-        msg += '---       (both are in taxcalc/tests)         ---\n'
-        msg += '-------------------------------------------------\n'
-        raise ValueError(msg)
-    # create aggregate diagnostic table using unweighted sub-sample of records
-    fullsample = puf_fullsample
-    rn_seed = 2222  # to ensure sub-sample is always the same
-    subfrac = 0.05  # sub-sample fraction
-    subsample = fullsample.sample(frac=subfrac, random_state=rn_seed)
-    recs_subsample = Records(data=subsample)
-    calc_subsample = Calculator(policy=baseline_policy, records=recs_subsample)
-    calc_subsample.advance_to_year(START_YEAR)
-    adt_subsample = calc_subsample.diagnostic_table(nyrs)
-    # compare combined tax liability from full and sub samples for each year
-    taxes_subsample = adt_subsample.loc['Combined Liability ($b)']
-    msg = ''
-    for cyr in range(calc_start_year, calc_start_year + nyrs):
-        reltol = 0.045  # maximum allowed relative difference in tax liability
-        if not np.allclose(taxes_subsample[cyr], taxes_fullsample[cyr],
-                           atol=0.0, rtol=reltol):
-            reldiff = (taxes_subsample[cyr] / taxes_fullsample[cyr]) - 1.
-            line1 = '\nPUFCSV AGG SUB-vs-FULL RESULTS DIFFER IN {}'
-            line2 = '\n  when subfrac={:.3f}, rtol={:.4f}, seed={}'
-            line3 = '\n  with sub={:.3f}, full={:.3f}, rdiff={:.4f}'
-            msg += line1.format(cyr)
-            msg += line2.format(subfrac, reltol, rn_seed)
-            msg += line3.format(taxes_subsample[cyr],
-                                taxes_fullsample[cyr],
-                                reldiff)
-    if msg:
-        raise ValueError(msg)
-
-
-MTR_TAX_YEAR = 2013
-MTR_NEG_DIFF = False  # set True to subtract (rather than add) small amount
-# specify payrolltax mtr histogram bin boundaries (or edges):
-PTAX_MTR_BIN_EDGES = [0.0, 0.02, 0.04, 0.06, 0.08,
-                      0.10, 0.12, 0.14, 0.16, 0.18, 1.0]
-#        the bin boundaries above are arbitrary, so users
-#        may want to experiment with alternative boundaries
-# specify incometax mtr histogram bin boundaries (or edges):
-ITAX_MTR_BIN_EDGES = [-1.0, -0.30, -0.20, -0.10, 0.0,
-                      0.10, 0.20, 0.30, 0.40, 0.50, 1.0]
-#        the bin boundaries above are arbitrary, so users
-#        may want to experiment with alternative boundaries
-
-
-def mtr_bin_counts(mtr_data, bin_edges, recid):
-    """
-    Compute mtr histogram bin counts and return results as a string.
-    """
-    res = ''
-    (bincount, _) = np.histogram(mtr_data.round(decimals=4), bins=bin_edges)
-    sum_bincount = np.sum(bincount)
-    res += f'{sum_bincount} :'
-    for idx in range(len(bin_edges) - 1):
-        res += f' {bincount[idx]:6d}'
-    res += '\n'
-    if sum_bincount < mtr_data.size:
-        res += 'WARNING: sum of bin counts is too low\n'
-        mtr_min = mtr_data.min()
-        mtr_max = mtr_data.max()
-        bin_min = min(bin_edges)
-        bin_max = max(bin_edges)
-        if mtr_min < bin_min:
-            res += f'         min(mtr)={mtr_min:.2f}\n'
-            for idx in range(mtr_data.size):
-                if mtr_data[idx] < bin_min:
-                    res += (
-                        f'         mtr={mtr_data[idx]:.2f} '
-                        f'for recid={recid[idx]}\n'
-                    )
-        if mtr_max > bin_max:
-            res += f'         max(mtr)={mtr_max:.2f}\n'
-            for idx in range(mtr_data.size):
-                if mtr_data[idx] > bin_max:
-                    res += (
-                        f'         mtr={mtr_data[idx]:.2f} '
-                        f'for recid={recid[idx]}\n'
-                    )
-    return res
-
-
-def nonsmall_diffs(linelist1, linelist2, small=0.0):
-    """
-    Return True if line lists differ significantly; otherwise return False.
-    Significant numerical difference means one or more numbers differ (between
-    linelist1 and linelist2) by more than the specified small amount.
-    """
-    # embedded function used only in nonsmall_diffs function
-    def isfloat(value):
-        """
-        Return True if value can be cast to float; otherwise return False.
-        """
-        try:
-            float(value)
-            return True
-        except ValueError:
-            return False
-    # begin nonsmall_diffs logic
-    assert isinstance(linelist1, list)
-    assert isinstance(linelist2, list)
-    if len(linelist1) != len(linelist2):
-        return True
-    assert 0.0 <= small <= 1.0
-    epsilon = 1e-6
-    smallamt = small + epsilon
-    for line1, line2 in zip(linelist1, linelist2):
-        if line1 == line2:
-            continue
-        tokens1 = line1.replace(',', '').split()
-        tokens2 = line2.replace(',', '').split()
-        for tok1, tok2 in zip(tokens1, tokens2):
-            tok1_isfloat = isfloat(tok1)
-            tok2_isfloat = isfloat(tok2)
-            if tok1_isfloat and tok2_isfloat:
-                if abs(float(tok1) - float(tok2)) <= smallamt:
-                    continue
-                return True
-            if not tok1_isfloat and not tok2_isfloat:
-                if tok1 == tok2:
-                    continue
-                return True
-            return True
-        return False
-
-
-@pytest.mark.requires_pufcsv
-def test_mtr(tests_path, puf_path):
-    """
-    Test Tax-Calculator marginal tax rates with no policy reform using puf.csv
-
-    Compute histograms for each marginal tax rate income type using
-    sample input from the puf.csv file and writing output to a string,
-    which is then compared for differences with EXPECTED_MTR_RESULTS.
-    """
-    # pylint: disable=too-many-locals,too-many-statements
-    assert len(PTAX_MTR_BIN_EDGES) == len(ITAX_MTR_BIN_EDGES)
-    # construct actual results string, res
-    res = ''
-    if MTR_NEG_DIFF:
-        res += 'MTR computed using NEGATIVE finite_diff '
-    else:
-        res += 'MTR computed using POSITIVE finite_diff '
-    res += f'for tax year {MTR_TAX_YEAR}\n'
-    # create a Policy object (clp) containing current-law policy parameters
-    clp = Policy()
-    clp.set_year(MTR_TAX_YEAR)
-    # create a Records object (puf) containing puf.csv input records
-    puf = Records(data=puf_path)
-    recid = puf.RECID  # pylint: disable=no-member
-    # create a Calculator object using clp policy and puf records
-    calc = Calculator(policy=clp, records=puf)
-    res += f'Total number of data records = {puf.array_length}\n'
-    res += 'PTAX mtr histogram bin edges:\n'
-    res += f'     {PTAX_MTR_BIN_EDGES}\n'
-    res += 'ITAX mtr histogram bin edges:\n'
-    res += f'     {ITAX_MTR_BIN_EDGES}\n'
-    variable_header = 'PTAX and ITAX mtr histogram bin counts for'
-    # compute marginal tax rate (mtr) histograms for each mtr variable
-    for var_str in Calculator.MTR_VALID_VARIABLES:
-        zero_out = var_str == 'e01400'
-        (mtr_ptax, mtr_itax, _) = calc.mtr(variable_str=var_str,
-                                           negative_finite_diff=MTR_NEG_DIFF,
-                                           zero_out_calculated_vars=zero_out,
-                                           wrt_full_compensation=False)
-        if zero_out:
-            # check that calculated variables are consistent
-            assert np.allclose((calc.array('iitax') +
-                                calc.array('payrolltax')),
-                               calc.array('combined'))
-            assert np.allclose(calc.array('ptax_was'),
-                               calc.array('payrolltax'))
-            assert np.allclose(calc.array('c21060') - calc.array('c21040'),
-                               calc.array('c04470'))
-            assert np.allclose(calc.array('taxbc') + calc.array('c09600'),
-                               calc.array('c05800'))
-            assert np.allclose((calc.array('c05800') +
-                                calc.array('othertaxes') -
-                                calc.array('c07100')),
-                               calc.array('c09200'))
-            assert np.allclose(calc.array('c09200') - calc.array('refund'),
-                               calc.array('iitax'))
-        if var_str == 'e00200s':
-            # only MARS==2 filing units have valid MTR values
-            mtr_ptax = mtr_ptax[calc.array('MARS') == 2]
-            mtr_itax = mtr_itax[calc.array('MARS') == 2]
-        res += f'{variable_header} {var_str}:\n'
-        res += mtr_bin_counts(mtr_ptax, PTAX_MTR_BIN_EDGES, recid)
-        res += mtr_bin_counts(mtr_itax, ITAX_MTR_BIN_EDGES, recid)
-    # check for differences between actual and expected results
-    mtrres_path = os.path.join(tests_path, 'pufcsv_mtr_expect.txt')
-    with open(mtrres_path, 'r', encoding='utf-8') as expected_file:
-        txt = expected_file.read()
-    expected_results = txt.rstrip('\n\t ') + '\n'  # cleanup end of file txt
-    if nonsmall_diffs(res.splitlines(True), expected_results.splitlines(True)):
-        new_filename = f'{mtrres_path[:-10]}actual.txt'
-        with open(new_filename, 'w', encoding='utf-8') as new_file:
-            new_file.write(res)
-        msg = 'PUFCSV MTR RESULTS DIFFER\n'
-        msg += '-------------------------------------------------\n'
-        msg += '--- NEW RESULTS IN pufcsv_mtr_actual.txt FILE ---\n'
-        msg += '--- if new OK, copy pufcsv_mtr_actual.txt to  ---\n'
-        msg += '---                 pufcsv_mtr_expect.txt     ---\n'
-        msg += '---            and rerun test.                ---\n'
-        msg += '-------------------------------------------------\n'
-        raise ValueError(msg)
-
-
-@pytest.mark.requires_pufcsv
-def test_credit_reforms(puf_subsample):
-    """
-    Test personal credit reforms using puf.csv subsample
-    """
-    rec = Records(data=puf_subsample)
-    reform_year = 2017
-    # create current-law Calculator object, calc1
-    pol = Policy()
-    calc1 = Calculator(policy=pol, records=rec)
-    calc1.advance_to_year(reform_year)
-    calc1.calc_all()
-    itax1 = calc1.weighted_total('iitax')
-    # create personal-refundable-credit-reform Calculator object, calc2
-    reform = {'II_credit': {reform_year: [1000, 1000, 1000, 1000, 1000]}}
-    pol.implement_reform(reform)
-    calc2 = Calculator(policy=pol, records=rec)
-    calc2.advance_to_year(reform_year)
-    calc2.calc_all()
-    itax2 = calc2.weighted_total('iitax')
-    # create personal-nonrefundable-credit-reform Calculator object, calc3
-    reform = {'II_credit_nr': {reform_year: [1000, 1000, 1000, 1000, 1000]}}
-    pol = Policy()
-    pol.implement_reform(reform)
-    calc3 = Calculator(policy=pol, records=rec)
-    calc3.advance_to_year(reform_year)
-    calc3.calc_all()
-    itax3 = calc3.weighted_total('iitax')
-    # check income tax revenues generated by the three Calculator objects
-    assert itax2 < itax1  # because refundable credits lower revenues
-    assert itax3 > itax2  # because nonrefundable credits lower revenues less
-    assert itax3 < itax1  # because nonrefundable credits lower revenues some
-
-
-@pytest.mark.requires_pufcsv
-def test_puf_availability(tests_path, puf_path):
-    """
-    Cross-check records_variables.json data with variables in puf.csv file
-    """
-    # make set of variable names in puf.csv file
-    pufdf = pd.read_csv(puf_path)
-    pufvars = set(list(pufdf))
-    # make set of variable names that are marked as puf.csv available
-    rvpath = os.path.join(tests_path, '..', 'records_variables.json')
-    with open(rvpath, 'r', encoding='utf-8') as rvfile:
-        rvdict = json.load(rvfile)
-    recvars = set()
-    for vname, vdict in rvdict['read'].items():
-        if 'taxdata_puf' in vdict.get('availability', ''):
-            recvars.add(vname)
-    # check that pufvars and recvars sets are the same
-    assert (pufvars - recvars) == set()
-    assert (recvars - pufvars) == set()