mergeron 2024.738963.0__tar.gz → 2024.738972.0__tar.gz

{mergeron-2024.738963.0 → mergeron-2024.738972.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mergeron
-Version: 2024.738963.0
+Version: 2024.738972.0
 Summary: Analysis of standards defined in Horizontal Merger Guidelines
 License: MIT
 Keywords: merger policy analysis,merger guidelines,merger screening,policy presumptions,concentration standards,upward pricing pressure,GUPPI

{mergeron-2024.738963.0 → mergeron-2024.738972.0}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "mergeron"
 # See ./get_version_str.py
-version = "2024.738963.0"
+version = "2024.738972.0"
 description = "Analysis of standards defined in Horizontal Merger Guidelines"
 keywords = [
     "merger policy analysis",

{mergeron-2024.738963.0 → mergeron-2024.738972.0}/src/mergeron/core/__init__.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from importlib.metadata import version
 
-from attrs import Attribute, define, field, validators
+from attrs import Attribute, field, frozen, validators
 
 from .. import _PKG_NAME, RECConstants, UPPAggrSelector  # noqa: TID252
 
@@ -38,7 +38,7 @@ def _rec_spec_validator(
         )
 
 
-@define(slots=True, frozen=True)
+@frozen
 class UPPBoundarySpec:
     share_ratio: float = field(
         kw_only=False,

{mergeron-2024.738963.0 → mergeron-2024.738972.0}/src/mergeron/core/guidelines_boundaries.py

@@ -5,13 +5,12 @@ with a canvas on which to draw boundaries for Guidelines standards.
 """
 
 import decimal
-from collections.abc import Callable
 from dataclasses import dataclass
 from importlib.metadata import version
 from typing import Any, Literal, TypeAlias
 
 import numpy as np
-from attrs import define, field
+from attrs import field, frozen
 from mpmath import mp, mpf  # type: ignore
 from numpy.typing import NDArray
 
@@ -24,7 +23,7 @@ __version__ = version(_PKG_NAME)
 mp.prec = 80
 mp.trap_complex = True
 
-HMGPubYear: TypeAlias = Literal[1992, 2010, 2023]
+HMGPubYear: TypeAlias = Literal[1992, 2004, 2010, 2023]
 
 
 @dataclass(slots=True, frozen=True)
@@ -43,25 +42,28 @@ class GuidelinesBoundary:
     area: float
 
 
-@dataclass(slots=True, frozen=True)
-class GuidelinesBoundaryCallable:
-    boundary_function: Callable[[NDArray[np.float64]], NDArray[np.float64]]
-    area: float
-    s_naught: float = 0
-
-
-@define(slots=True, frozen=True)
+@frozen
 class GuidelinesThresholds:
     """
     Guidelines threholds by Guidelines publication year
 
     ΔHHI, Recapture Rate, GUPPI, Diversion ratio, CMCR, and IPR thresholds
-    constructed from concentration standards.
+    constructed from concentration standards in Guidelines published in
+    1992, 2004, 2010, and 2023.
+
+    The 2004 Guidelines refernced here are the EU Commission
+    guidelines on assessment of horizontal mergers. These
+    guidelines also define a presumption for mergers with
+    post-merger HHI in [1000, 2000) and ΔHHI >= 250 points,
+    whi is not modeled here.
+
+    All other Guidelines modeled here are U.S. merger guidelines.
+
     """
 
     pub_year: HMGPubYear
     """
-    Year of publication of the U.S. Horizontal Merger Guidelines (HMG)
+    Year of publication of the Guidelines
     """
 
     safeharbor: HMGThresholds = field(kw_only=True, default=None)
@@ -99,6 +101,7 @@ class GuidelinesThresholds:
         _hhi_p, _dh_s, _dh_p = {
             1992: (0.18, 0.005, 0.01),
             2010: (0.25, 0.01, 0.02),
+            2004: (0.20, 0.015, 0.015),
             2023: (0.18, 0.01, 0.01),
         }[self.pub_year]
 

{mergeron-2024.738963.0 → mergeron-2024.738972.0}/src/mergeron/core/guidelines_boundaries_specialized_functions.py

@@ -6,21 +6,19 @@ to have poor performance
 
 """
 
+from collections.abc import Callable
+from dataclasses import dataclass
 from importlib.metadata import version
 from typing import Literal
 
 import numpy as np
 from mpmath import mp, mpf  # type: ignore
+from numpy.typing import NDArray
 from scipy.spatial.distance import minkowski as distance_function  # type: ignore
 from sympy import lambdify, simplify, solve, symbols
 
 from .. import _PKG_NAME  # noqa: TID252
-from .guidelines_boundaries import (
-    GuidelinesBoundary,
-    GuidelinesBoundaryCallable,
-    _shrratio_boundary_intcpt,
-    lerp,
-)
+from .guidelines_boundaries import GuidelinesBoundary, _shrratio_boundary_intcpt, lerp
 
 __version__ = version(_PKG_NAME)
 
@@ -29,6 +27,13 @@ mp.prec = 80
 mp.trap_complex = True
 
 
+@dataclass(slots=True, frozen=True)
+class GuidelinesBoundaryCallable:
+    boundary_function: Callable[[NDArray[np.float64]], NDArray[np.float64]]
+    area: float
+    s_naught: float = 0
+
+
 def delta_hhi_boundary_qdtr(_dh_val: float = 0.01, /) -> GuidelinesBoundaryCallable:
     """
     Generate the list of share combination on the ΔHHI boundary.

{mergeron-2024.738963.0 → mergeron-2024.738972.0}/src/mergeron/gen/__init__.py

@@ -5,23 +5,21 @@ Defines constants and containers for industry data generation and testing
 
 from __future__ import annotations
 
-from importlib.metadata import version
-
-from .. import _PKG_NAME, RECConstants, UPPAggrSelector  # noqa: TID252
-
-__version__ = version(_PKG_NAME)
-
-
 import enum
 from dataclasses import dataclass
-from typing import ClassVar, Literal, Protocol, TypeVar, Union
+from importlib.metadata import version
+from typing import ClassVar, Protocol, TypeVar
 
 import numpy as np
-from attrs import Attribute, define, field, validators
+from attrs import Attribute, define, field, frozen, validators
 from numpy.typing import NBitBase, NDArray
 
+from .. import _PKG_NAME, RECConstants, UPPAggrSelector  # noqa: TID252
 from ..core.pseudorandom_numbers import DIST_PARMS_DEFAULT  # noqa: TID252
 
+__version__ = version(_PKG_NAME)
+
+
 EMPTY_ARRAY_DEFAULT = np.zeros(2)
 FCOUNT_WTS_DEFAULT = ((_nr := np.arange(1, 6)[::-1]) / _nr.sum()).astype(np.float64)
 
@@ -74,7 +72,7 @@ class SHRConstants(enum.StrEnum):
     """
 
 
-@define(slots=True, frozen=True)
+@frozen
 class ShareSpec:
     """Market share specification
 
@@ -152,7 +150,7 @@ class FM2Constants(enum.StrEnum):
     SYM = "symmetric"
 
 
-@define(slots=True, frozen=True)
+@frozen
 class PCMSpec:
     """Price-cost margin (PCM) specification
 
@@ -224,9 +222,9 @@ class SSZConstants(float, enum.ReprEnum):
     """When initial set of draws is not restricted in any way."""
 
 
-# Validators for selected attributes of MarketSampleSpec
+# Validators for selected attributes of MarketSpec
 def _sample_size_validator(
-    _object: MarketSampleSpec, _attribute: Attribute[int], _value: int, /
+    _object: MarketSpec, _attribute: Attribute[int], _value: int, /
 ) -> None:
     if _value < 10**6:
         raise ValueError(
@@ -235,7 +233,7 @@ def _sample_size_validator(
 
 
 def _share_spec_validator(
-    _instance: MarketSampleSpec, _attribute: Attribute[ShareSpec], _value: ShareSpec, /
+    _instance: MarketSpec, _attribute: Attribute[ShareSpec], _value: ShareSpec, /
 ) -> None:
     _r_bar = _value.recapture_rate
     if _r_bar and not (0 < _r_bar <= 1):
@@ -274,7 +272,7 @@ def _share_spec_validator(
 
 
 def _pcm_spec_validator(
-    _instance: MarketSampleSpec, _attribute: Attribute[PCMSpec], _value: PCMSpec, /
+    _instance: MarketSpec, _attribute: Attribute[PCMSpec], _value: PCMSpec, /
 ) -> None:
     if (
         _instance.share_spec.recapture_form == RECConstants.FIXED
@@ -308,15 +306,10 @@ def _pcm_spec_validator(
             )
 
 
-@define(slots=True, frozen=True)
-class MarketSampleSpec:
+@define(slots=False)
+class MarketSpec:
     """Parameter specification for market data generation."""
 
-    sample_size: int = field(
-        default=10**6, validator=(validators.instance_of(int), _sample_size_validator)
-    )
-    """sample size generated"""
-
     share_spec: ShareSpec = field(
         kw_only=True,
         default=ShareSpec(RECConstants.INOUT, 0.855, SHRConstants.UNI, None, None),
@@ -353,17 +346,16 @@ class INVResolution(enum.StrEnum):
     BOTH = "both"
 
 
-@define(slots=True, frozen=True)
+@frozen
 class UPPTestRegime:
     resolution: INVResolution = field(
         default=INVResolution.ENFT, validator=validators.instance_of(INVResolution)
     )
     guppi_aggregator: UPPAggrSelector = field(
-        default=UPPAggrSelector.MAX, validator=validators.instance_of(UPPAggrSelector)
+        default=UPPAggrSelector.MIN, validator=validators.instance_of(UPPAggrSelector)
     )
     divr_aggregator: UPPAggrSelector | None = field(
-        default=guppi_aggregator,
-        validator=validators.instance_of((UPPAggrSelector, type(None))),
+        default=None, validator=validators.instance_of((UPPAggrSelector, type(None)))
     )
 
 
@@ -469,7 +461,7 @@ class MarginDataSample:
 
 @dataclass(slots=True, frozen=True)
 class UPPTestsRaw:
-    """arrays marking test failures and successes
+    """Container for arrays marking test failures and successes
 
     A test success is a draw ("market") that meeets the
     specified test criterion, and a test failure is
@@ -494,9 +486,9 @@ class UPPTestsRaw:
 
 @dataclass(slots=True, frozen=True)
 class UPPTestsCounts:
-    """counts of markets resolved as specified
+    """Counts of markets resolved as specified
 
-    Resolution is specified in a UPPTestRegime object.
+    Resolution may be either "enforcement" or "clearance".
     """
 
     by_firm_count: NDArray[np.int64]

mergeron-2024.738963.0/src/mergeron/gen/_data_generation_functions_nonpublic.py → mergeron-2024.738972.0/src/mergeron/gen/_data_generation_functions.py

@@ -19,11 +19,12 @@ from ..core.pseudorandom_numbers import (  # noqa: TID252
     prng,
 )
 from . import (
+    EMPTY_ARRAY_DEFAULT,
     FCOUNT_WTS_DEFAULT,
     TF,
     FM2Constants,
     MarginDataSample,
-    MarketSampleSpec,
+    MarketSpec,
     PCMConstants,
     PriceDataSample,
     PRIConstants,
@@ -36,7 +37,8 @@ __version__ = version(_PKG_NAME)
 
 
 def _gen_share_data(
-    _mkt_sample_spec: MarketSampleSpec,
+    _sample_size: int,
+    _mkt_sample_spec: MarketSpec,
     _fcount_rng_seed_seq: SeedSequence | None,
     _mktshr_rng_seed_seq: SeedSequence,
     _nthreads: int = 16,
@@ -66,7 +68,7 @@ def _gen_share_data(
         for _f in ("recapture_form", "dist_type", "dist_parms", "firm_counts_weights")
     )
 
-    _ssz = _mkt_sample_spec.sample_size
+    _ssz = _sample_size
 
     match _dist_type_mktshr:
         case SHRConstants.UNI:
@@ -396,24 +398,22 @@ def _gen_market_shares_dirichlet(
 def _gen_price_data(
     _frmshr_array: NDArray[np.float64],
     _nth_firm_share: NDArray[np.float64],
-    _mkt_sample_spec: MarketSampleSpec,
+    _mkt_sample_spec: MarketSpec,
     _seed_seq: SeedSequence | None = None,
     /,
 ) -> PriceDataSample:
-    _ssz = len(_frmshr_array)
-
     _hsr_filing_test_type = _mkt_sample_spec.hsr_filing_test_type
 
     _price_array, _price_ratio_array, _hsr_filing_test = (
         np.ones_like(_frmshr_array, np.float64),
         np.empty_like(_frmshr_array, np.float64),
-        np.empty(_ssz, bool),
+        np.empty(len(_frmshr_array), bool),
     )
 
     _pr_max_ratio = 5.0
     match _mkt_sample_spec.price_spec:
         case PRIConstants.SYM:
-            _nth_firm_price = np.ones((_ssz, 1))
+            _nth_firm_price = np.ones((len(_frmshr_array), 1))
         case PRIConstants.POS:
             _price_array, _nth_firm_price = (
                 np.ceil(_p * _pr_max_ratio) for _p in (_frmshr_array, _nth_firm_share)
@@ -477,14 +477,14 @@ def _gen_price_data(
             # del _nth_firm_rev, _rev_ratio_to_nth
         case _:
             # Otherwise, all draws meet the filing test
-            _hsr_filing_test = np.ones(_ssz, dtype=bool)
+            _hsr_filing_test = np.ones(len(_frmshr_array), dtype=bool)
 
     return PriceDataSample(_price_array, _hsr_filing_test)
 
 
 def _gen_pcm_data(
     _frmshr_array: NDArray[np.floating[TF]],
-    _mkt_sample_spec: MarketSampleSpec,
+    _mkt_sample_spec: MarketSpec,
     _price_array: NDArray[np.floating[TF]],
     _aggregate_purchase_prob: NDArray[np.floating[TF]],
     _pcm_rng_seed_seq: SeedSequence,
@@ -504,27 +504,25 @@ def _gen_pcm_data(
     _mnl_test_array = np.empty((len(_frmshr_array), 2), dtype=int)
 
     _beta_min, _beta_max = [None] * 2  # placeholder
-    _dist_parms = np.ones(2, np.float64)
     if _dist_type_pcm == PCMConstants.EMPR:
         _pcm_array = resample_mgn_data(
             _pcm_array.shape,  # type: ignore
             seed_sequence=_pcm_rng_seed_seq,
         )
     else:
-        if _dist_type_pcm == PCMConstants.UNI:
-            _dist_parms = (
-                DIST_PARMS_DEFAULT if _dist_parms_pcm is None else _dist_parms_pcm
-            )
-        elif _dist_type_pcm == PCMConstants.BETA:
-            # Error-checking (could move to validators in definition of MarketSampleSpec)
-
+        if _dist_type_pcm == PCMConstants.BETA:
             if _dist_parms_pcm is None:
-                _dist_parms_pcm = _dist_parms
+                _dist_parms_pcm = np.ones(2, np.float64)
 
         elif _dist_type_pcm == PCMConstants.BETA_BND:  # Bounded beta
             if _dist_parms_pcm is None:
                 _dist_parms_pcm = np.array([0, 1, 0, 1], np.float64)
                 _dist_parms = beta_located_bound(_dist_parms_pcm)
+        else:
+            #  _dist_type_pcm == PCMConstants.UNI
+            _dist_parms = (
+                DIST_PARMS_DEFAULT if _dist_parms_pcm is None else _dist_parms_pcm
+            )
 
         _pcm_rng = MultithreadedRNG(
             _pcm_array,
@@ -564,6 +562,66 @@ def _gen_pcm_data(
     return MarginDataSample(_pcm_array, _mnl_test_array)
 
 
+def _gen_divr_array(
+    _recapture_form: RECConstants,
+    _recapture_rate: float | None,
+    _frmshr_array: NDArray[np.float64],
+    _aggregate_purchase_prob: NDArray[np.float64] = EMPTY_ARRAY_DEFAULT,
+    /,
+) -> NDArray[np.float64]:
+    """
+    Given merging-firm shares and related parameters, return diverion ratios.
+
+    If recapture is specified as "Outside-in" (RECConstants.OUTIN), then the
+    choice-probability for the outside good must be supplied.
+
+    Parameters
+    ----------
+    _recapture_form
+        Enum specifying Fixed (proportional), Inside-out, or Outside-in
+
+    _recapture_rate
+        If recapture is proportional or inside-out, the recapture rate
+        for the firm with the smaller share.
+
+    _frmshr_array
+        Merging-firm shares.
+
+    _aggregate_purchase_prob
+        1 minus probability that the outside good is chosen; converts
+        market shares to choice probabilities by multiplication.
+
+    Returns
+    -------
+        Merging-firm diversion ratios for mergers in the sample.
+
+    """
+
+    _divr_array: NDArray[np.float64]
+    if _recapture_form == RECConstants.FIXED:
+        _divr_array = _recapture_rate * _frmshr_array[:, ::-1] / (1 - _frmshr_array)  # type: ignore
+
+    else:
+        _purchprob_array = _aggregate_purchase_prob * _frmshr_array
+        _divr_array = _purchprob_array[:, ::-1] / (1 - _purchprob_array)
+
+    _divr_assert_test = (
+        (np.round(np.einsum("ij->i", _frmshr_array), 15) == 1)
+        | (np.argmin(_frmshr_array, axis=1) == np.argmax(_divr_array, axis=1))
+    )[:, None]
+    if not all(_divr_assert_test):
+        raise ValueError(
+            "{} {} {} {}".format(
+                "Data construction fails tests:",
+                "the index of min(s_1, s_2) must equal",
+                "the index of max(d_12, d_21), for all draws.",
+                "unless frmshr_array sums to 1.00.",
+            )
+        )
+
+    return _divr_array
+
+
 def _beta_located(
     _mu: float | NDArray[np.float64], _sigma: float | NDArray[np.float64], /
 ) -> NDArray[np.float64]:

{mergeron-2024.738963.0 → mergeron-2024.738972.0}/src/mergeron/gen/data_generation.py

@@ -7,7 +7,6 @@ from __future__ import annotations
 
 from importlib.metadata import version
 
-import attrs
 import numpy as np
 from numpy.random import SeedSequence
 from numpy.typing import NDArray
@@ -17,12 +16,12 @@ from . import (
     EMPTY_ARRAY_DEFAULT,
     FM2Constants,
     MarketDataSample,
-    MarketSampleSpec,
+    MarketSpec,
     PRIConstants,
     SHRConstants,
     SSZConstants,
 )
-from ._data_generation_functions_nonpublic import (
+from ._data_generation_functions import (
     _gen_market_shares_dirichlet,  # noqa: F401 easter-egg for external modules
     _gen_market_shares_uniform,  # noqa: F401 easter-egg for external modules
     _gen_pcm_data,
@@ -34,9 +33,10 @@ __version__ = version(_PKG_NAME)
 
 
 def gen_market_sample(
-    _mkt_sample_spec: MarketSampleSpec,
+    _mkt_sample_spec: MarketSpec,
     /,
     *,
+    sample_size: int = 10**6,
     seed_seq_list: list[SeedSequence] | None = None,
     nthreads: int = 16,
 ) -> MarketDataSample:
@@ -59,6 +59,8 @@ def gen_market_sample(
     ----------
     _mkt_sample_spec
         class specifying parameters for data generation
+    sample_size
+        number of draws to generate
     seed_seq_list
         tuple of SeedSequences to ensure replicable data generation with
         appropriately independent random streams
@@ -72,7 +74,7 @@ def gen_market_sample(
 
     """
 
-    _mkt_sample_spec = _mkt_sample_spec or MarketSampleSpec()
+    _mkt_sample_spec = _mkt_sample_spec or MarketSpec()
 
     _recapture_form = _mkt_sample_spec.share_spec.recapture_form
     _recapture_rate = _mkt_sample_spec.share_spec.recapture_rate
@@ -89,19 +91,20 @@ def gen_market_sample(
         seed_seq_list, _dist_type_mktshr, _mkt_sample_spec.price_spec
     )
 
-    _shr_sample_size = 1.0 * _mkt_sample_spec.sample_size
+    _shr_sample_size = 1.0 * sample_size
     # Scale up sample size to offset discards based on specified criteria
     _shr_sample_size *= _hsr_filing_test_type
     if _dist_firm2_pcm == FM2Constants.MNL:
         _shr_sample_size *= SSZConstants.MNL_DEP
-    _mkt_sample_spec_here = attrs.evolve(
-        _mkt_sample_spec, sample_size=int(_shr_sample_size)
-    )
-    del _shr_sample_size
+    _shr_sample_size = int(_shr_sample_size)
 
     # Generate share data
     _mktshr_data = _gen_share_data(
-        _mkt_sample_spec_here, _fcount_rng_seed_seq, _mktshr_rng_seed_seq, nthreads
+        _shr_sample_size,
+        _mkt_sample_spec,
+        _fcount_rng_seed_seq,
+        _mktshr_rng_seed_seq,
+        nthreads,
     )
 
     _mktshr_array, _fcounts, _aggregate_purchase_prob, _nth_firm_share = (
@@ -116,7 +119,7 @@ def gen_market_sample(
 
     # Generate merging-firm price data
     _price_data = _gen_price_data(
-        _mktshr_array[:, :2], _nth_firm_share, _mkt_sample_spec_here, _pr_rng_seed_seq
+        _mktshr_array[:, :2], _nth_firm_share, _mkt_sample_spec, _pr_rng_seed_seq
     )
 
     _price_array, _hsr_filing_test = (
@@ -138,7 +141,7 @@ def gen_market_sample(
     # Generate margin data
     _pcm_data = _gen_pcm_data(
         _mktshr_array[:, :2],
-        _mkt_sample_spec_here,
+        _mkt_sample_spec,
         _price_array,
         _aggregate_purchase_prob,
         _pcm_rng_seed_seq,
@@ -148,7 +151,7 @@ def gen_market_sample(
         getattr(_pcm_data, _f) for _f in ("pcm_array", "mnl_test_array")
     )
 
-    _s_size = _mkt_sample_spec.sample_size  # originally-specified sample size
+    _s_size = sample_size  # originally-specified sample size
     if _dist_firm2_pcm == FM2Constants.MNL:
         _mktshr_array = _mktshr_array[_mnl_test_rows][:_s_size]
         _pcm_array = _pcm_array[_mnl_test_rows][:_s_size]

mergeron-2024.738972.0/src/mergeron/gen/market_sample.py

@@ -0,0 +1,79 @@
+"""
+Methods to generate data for analyzing merger enforcement policy.
+
+"""
+
+from __future__ import annotations
+
+from importlib.metadata import version
+
+from attrs import define
+from numpy.random import SeedSequence
+
+from .. import _PKG_NAME  # noqa: TID252
+from ..core import guidelines_boundaries as gbl  # noqa: TID252
+from . import MarketSpec, UPPTestRegime
+from .data_generation import gen_market_sample
+from .upp_tests import SaveData, invres_cnts, save_data_to_hdf5, sim_invres_cnts_ll
+
+__version__ = version(_PKG_NAME)
+
+
+@define(slots=False)
+class MarketSample(MarketSpec):
+    def generate_sample(
+        self,
+        /,
+        *,
+        sample_size: int = 10**6,
+        seed_seq_list: list[SeedSequence] | None,
+        nthreads: int,
+        save_data_to_file: SaveData = False,
+    ) -> None:
+        self.data = gen_market_sample(
+            self,
+            sample_size=sample_size,
+            seed_seq_list=seed_seq_list,
+            nthreads=nthreads,
+        )
+        _invalid_array_names = (
+            ("fcounts", "choice_prob_outgd", "nth_firm_share", "hhi_post")
+            if self.share_spec.dist_type == "Uniform"
+            else ()
+        )
+        if save_data_to_file:
+            save_data_to_hdf5(
+                self.data,
+                excluded_attrs=_invalid_array_names,
+                save_data_to_file=save_data_to_file,
+            )
+
+    def estimate_invres_counts(
+        self,
+        _invres_parm_vec: gbl.HMGThresholds,
+        _upp_test_regime: UPPTestRegime,
+        /,
+        *,
+        sample_size: int = 10**6,
+        seed_seq_list: list[SeedSequence] | None,
+        nthreads: int,
+        save_data_to_file: SaveData = False,
+    ) -> None:
+        if getattr(self, "market_data_sample", None) is None:
+            self.invres_counts = sim_invres_cnts_ll(
+                self,
+                _invres_parm_vec,
+                _upp_test_regime,
+                save_data_to_file=save_data_to_file,
+                sample_size=sample_size,
+                seed_seq_list=seed_seq_list,
+                nthreads=nthreads,
+            )
+        else:
+            self.invres_counts = invres_cnts(
+                self.data, _invres_parm_vec, _upp_test_regime
+            )
+            if save_data_to_file:
+                save_data_to_hdf5(
+                    self.invres_counts, save_data_to_file=save_data_to_file
+                )

{mergeron-2024.738963.0 → mergeron-2024.738972.0}/src/mergeron/gen/upp_tests.py

@@ -12,7 +12,6 @@ from typing import Literal, TypeAlias, TypedDict
 
 import numpy as np
 import tables as ptb  # type: ignore
-from attrs import evolve
 from joblib import Parallel, cpu_count, delayed  # type: ignore
 from numpy.random import SeedSequence
 from numpy.typing import NDArray
@@ -26,7 +25,7 @@ from . import (
     DataclassInstance,
     INVResolution,
     MarketDataSample,
-    MarketSampleSpec,
+    MarketSpec,
     UPPTestRegime,
     UPPTestsCounts,
     UPPTestsRaw,
@@ -43,21 +42,27 @@ ptb.parameters.MAX_BLOSC_THREADS = 4
 SaveData: TypeAlias = Literal[False] | tuple[Literal[True], ptb.File, ptb.Group]
 
 
-class IVNRESCntsArgs(TypedDict, total=False):
+class INVRESCntsArgs(TypedDict, total=False):
     "Keyword arguments of function, :code:`sim_invres_cnts`"
 
-    sim_test_regime: UPPTestRegime
     saved_array_name_suffix: str
     save_data_to_file: SaveData
-    seed_seq_list: list[SeedSequence]
+    sample_size: int
+    seed_seq_list: list[SeedSequence] | None
     nthreads: int
 
 
 def sim_invres_cnts_ll(
-    _mkt_sample_spec: MarketSampleSpec,
+    _mkt_sample_spec: MarketSpec,
     _invres_parm_vec: gbl.HMGThresholds,
-    _sim_invres_cnts_kwargs: IVNRESCntsArgs,
+    _sim_test_regime: UPPTestRegime,
     /,
+    *,
+    saved_array_name_suffix: str = "",
+    save_data_to_file: SaveData = False,
+    sample_size: int = 10**6,
+    seed_seq_list: list[SeedSequence] | None = None,
+    nthreads: int = 16,
 ) -> UPPTestsCounts:
     """A function to parallelize data-generation and testing
 
@@ -78,22 +83,34 @@ def sim_invres_cnts_ll(
     _mkt_sample_spec
         Configuration to use for generating sample data to test
 
-    _sim_invres_cnts_kwargs
-        Arguments to downstream test function `sim_invres_cnts`
+    _sim_test_regime
+        Configuration to use for testing
+
+    saved_array_name_suffix
+        Suffix to add to the array names in the HDF5 file
+
+    save_data_to_file
+        Whether to save data to an HDF5 file, and where to save it
+
+    sample_size
+        Number of draws to simulate
+
+    seed_seq_list
+        List of seed sequences, to assure independent samples in each thread
+
+    nthreads
+        Number of parallel processes to use
 
     Returns
     -------
         Arrays of UPPTestCounts
 
     """
-    _sample_sz = _mkt_sample_spec.sample_size
+    _sample_sz = sample_size
     _subsample_sz = 10**6
     _iter_count = int(_sample_sz / _subsample_sz) if _subsample_sz < _sample_sz else 1
     _thread_count = cpu_count()
 
-    # Crate a copy, to avoid side effects in the outer scope
-    _mkt_sample_spec_here = evolve(_mkt_sample_spec, sample_size=_subsample_sz)
-
     if (
         _mkt_sample_spec.share_spec.recapture_form != RECConstants.OUTIN
         and _mkt_sample_spec.share_spec.recapture_rate != _invres_parm_vec.rec
@@ -107,26 +124,24 @@ def sim_invres_cnts_ll(
         )
 
     _rng_seed_seq_list = [None] * _iter_count
-    if _sim_invres_cnts_kwargs:
-        if _sseql := _sim_invres_cnts_kwargs.get("seed_seq_list"):
-            _rng_seed_seq_list = list(
-                zip(*[g.spawn(_iter_count) for g in _sseql], strict=True)  # type: ignore
-            )
+    if seed_seq_list:
+        _rng_seed_seq_list = list(
+            zip(*[g.spawn(_iter_count) for g in seed_seq_list], strict=True)  # type: ignore
+        )
 
-        _sim_invres_cnts_kwargs: IVNRESCntsArgs = {  # type: ignore
-            _k: _v
-            for _k, _v in _sim_invres_cnts_kwargs.items()
-            if _k != "seed_seq_list"
-        }
-    else:
-        _sim_invres_cnts_kwargs = {}
+    _sim_invres_cnts_kwargs: INVRESCntsArgs = INVRESCntsArgs({
+        "sample_size": _subsample_sz,
+        "save_data_to_file": save_data_to_file,
+        "nthreads": nthreads,
+    })
 
     _res_list = Parallel(n_jobs=_thread_count, prefer="threads")(
         delayed(sim_invres_cnts)(
-            _mkt_sample_spec_here,
+            _mkt_sample_spec,
             _invres_parm_vec,
+            _sim_test_regime,
             **_sim_invres_cnts_kwargs,
-            saved_array_name_suffix=f"{_iter_id:0{2 + int(np.ceil(np.log10(_iter_count)))}d}",
+            saved_array_name_suffix=f"{saved_array_name_suffix}_{_iter_id:0{2 + int(np.ceil(np.log10(_iter_count)))}d}",
             seed_seq_list=_rng_seed_seq_list_ch,
         )
         for _iter_id, _rng_seed_seq_list_ch in enumerate(_rng_seed_seq_list)
@@ -151,19 +166,23 @@ def sim_invres_cnts_ll(
 
 
 def sim_invres_cnts(
-    _mkt_sample_spec: MarketSampleSpec,
+    _mkt_sample_spec: MarketSpec,
     _upp_test_parms: gbl.HMGThresholds,
+    _sim_test_regime: UPPTestRegime,
     /,
     *,
-    sim_test_regime: UPPTestRegime,
     saved_array_name_suffix: str = "",
     save_data_to_file: SaveData = False,
+    sample_size: int = 10**6,
     seed_seq_list: list[SeedSequence] | None = None,
     nthreads: int = 16,
 ) -> UPPTestsCounts:
     # Generate market data
-    _market_data = dgl.gen_market_sample(
-        _mkt_sample_spec, seed_seq_list=seed_seq_list, nthreads=nthreads
+    _market_data_sample = dgl.gen_market_sample(
+        _mkt_sample_spec,
+        sample_size=sample_size,
+        seed_seq_list=seed_seq_list,
+        nthreads=nthreads,
     )
 
     _invalid_array_names = (
@@ -173,28 +192,42 @@ def sim_invres_cnts(
     )
 
     save_data_to_hdf5(
-        _market_data,
-        saved_array_name_suffix,
-        _invalid_array_names,
+        _market_data_sample,
+        saved_array_name_suffix=saved_array_name_suffix,
+        excluded_attrs=_invalid_array_names,
         save_data_to_file=save_data_to_file,
     )
 
-    _upp_tests_data = gen_upp_arrays(
-        _market_data,
-        _upp_test_parms,
-        sim_test_regime,
+    _upp_test_arrays = invres_cnts(
+        _market_data_sample, _upp_test_parms, _sim_test_regime
+    )
+
+    save_data_to_hdf5(
+        _upp_test_arrays,
         saved_array_name_suffix=saved_array_name_suffix,
         save_data_to_file=save_data_to_file,
     )
 
+    return _upp_test_arrays
+
+
+def invres_cnts(
+    _market_data_sample: MarketDataSample,
+    _upp_test_parms: gbl.HMGThresholds,
+    _upp_test_regime: UPPTestRegime,
+    /,
+) -> UPPTestsCounts:
+    _upp_test_arrays = gen_upp_test_arrays(
+        _market_data_sample, _upp_test_parms, _upp_test_regime
+    )
+
     _fcounts, _hhi_delta, _hhi_post = (
-        getattr(_market_data, _g) for _g in ["fcounts", "hhi_delta", "hhi_post"]
+        getattr(_market_data_sample, _g) for _g in ("fcounts", "hhi_delta", "hhi_post")
     )
-    del _market_data
 
     _stats_rowlen = 6
     # Clearance/enforcement counts --- by firm count
-    _firm_counts_weights = _mkt_sample_spec.share_spec.firm_counts_weights
+    _firm_counts_weights = np.unique(_fcounts)
     if _firm_counts_weights is not None and np.all(_firm_counts_weights >= 0):
         _max_firm_count = len(_firm_counts_weights)
 
@@ -210,9 +243,9 @@ def sim_invres_cnts(
                     *[
                         np.einsum(
                             "ij->",
-                            1 * (_firm_count_test & getattr(_upp_tests_data, _f)),
+                            1 * (_firm_count_test & getattr(_upp_test_arrays, _f)),
                         )
-                        for _f in _upp_tests_data.__dataclass_fields__
+                        for _f in _upp_test_arrays.__dataclass_fields__
                     ],
                 ]),
             ))
@@ -236,9 +269,9 @@ def sim_invres_cnts(
                 np.einsum("ij->", 1 * _hhi_delta_test),
                 *[
                     np.einsum(
-                        "ij->", 1 * (_hhi_delta_test & getattr(_upp_tests_data, _f))
+                        "ij->", 1 * (_hhi_delta_test & getattr(_upp_test_arrays, _f))
                     )
-                    for _f in _upp_tests_data.__dataclass_fields__
+                    for _f in _upp_test_arrays.__dataclass_fields__
                 ],
             ]),
         ))
@@ -273,9 +306,9 @@ def sim_invres_cnts(
                     np.einsum("ij->", 1 * _conc_test),
                     *[
                         np.einsum(
-                            "ij->", 1 * (_conc_test & getattr(_upp_tests_data, _f))
+                            "ij->", 1 * (_conc_test & getattr(_upp_test_arrays, _f))
                         )
-                        for _f in _upp_tests_data.__dataclass_fields__
+                        for _f in _upp_test_arrays.__dataclass_fields__
                     ],
                 ]),
             ))
@@ -293,19 +326,26 @@ def sim_invres_cnts(
     )
 
 
-def gen_upp_arrays(
+def gen_upp_test_arrays(
     _market_data: MarketDataSample,
     _upp_test_parms: gbl.HMGThresholds,
     _sim_test_regime: UPPTestRegime,
     /,
-    *,
-    saved_array_name_suffix: str = "",
-    save_data_to_file: SaveData = False,
 ) -> UPPTestsRaw:
     """
     Generate UPP tests arrays for given configuration and market sample
 
     Given a standards vector, market
+
+    Parameters
+    ----------
+    _market_data
+        market data sample
+    _upp_test_parms
+        guidelines thresholds for testing UPP and related statistics
+    _sim_test_regime
+        configuration to use for generating UPP tests
+
     """
     _g_bar, _divr_bar, _cmcr_bar, _ipr_bar = (
         getattr(_upp_test_parms, _f) for _f in ("guppi", "divr", "cmcr", "ipr")
@@ -407,29 +447,21 @@ def gen_upp_arrays(
         _divr_test_vector = _market_data.divr_array.max(axis=1, keepdims=True)
 
     if _invres_resolution == INVResolution.ENFT:
-        _upp_tests_data = UPPTestsRaw(
+        _upp_test_arrays = UPPTestsRaw(
             _guppi_test_vector >= _g_bar,
             (_guppi_test_vector >= _g_bar) | (_divr_test_vector >= _divr_bar),
             _cmcr_test_vector >= _cmcr_bar,
             _ipr_test_vector >= _ipr_bar,
         )
     else:
-        _upp_tests_data = UPPTestsRaw(
+        _upp_test_arrays = UPPTestsRaw(
             _guppi_test_vector < _g_bar,
             (_guppi_test_vector < _g_bar) & (_divr_test_vector < _divr_bar),
             _cmcr_test_vector < _cmcr_bar,
             _ipr_test_vector < _ipr_bar,
         )
-    del _guppi_test_vector, _divr_test_vector, _cmcr_test_vector, _ipr_test_vector
-
-    save_data_to_hdf5(
-        _upp_tests_data,
-        saved_array_name_suffix,
-        (),
-        save_data_to_file=save_data_to_file,
-    )
 
-    return _upp_tests_data
+    return _upp_test_arrays
 
 
 def initialize_hd5(
@@ -449,24 +481,25 @@ def initialize_hd5(
 
 def save_data_to_hdf5(
     _dclass: DataclassInstance,
-    _saved_array_name_suffix: str = "",
-    _excl_attrs: Sequence[str] = (),
     /,
     *,
+    saved_array_name_suffix: str | None = "",
+    excluded_attrs: Sequence[str] | None = (),
     save_data_to_file: SaveData = False,
 ) -> None:
     if save_data_to_file:
         _, _h5_file, _h5_group = save_data_to_file
         # Save market data arrays
+        excluded_attrs = excluded_attrs or ()
         for _array_name in _dclass.__dataclass_fields__:
-            if _array_name in _excl_attrs:
+            if _array_name in excluded_attrs:
                 continue
             save_array_to_hdf5(
                 getattr(_dclass, _array_name),
                 _array_name,
                 _h5_group,
                 _h5_file,
-                saved_array_name_suffix=_saved_array_name_suffix,
+                saved_array_name_suffix=saved_array_name_suffix,
             )