dpd-stream 0.2.0__tar.gz → 0.2.1__tar.gz

{dpd_stream-0.2.0 → dpd_stream-0.2.1}/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 0.2.1
+
+- **Lower memory reading a panel.** Checking that (panel, time) identifies the observations no
+  longer holds a set of every row; repeated times are found as the blocks are written, with the
+  same error and the same results. On 9.6 million rows the fit's memory no longer rises to
+  1.55 GB at that step, and a fit now stays near 1.1 GB from 10 to 24 million rows.
+- **Memory tests.** `pytest -m resource` fits difference and system GMM on a 24-million-row
+  panel under a fixed memory limit; CI runs them on Linux, macOS and Windows. The default
+  `pytest` run leaves them out.
+- Requires hdfe-stream 0.9.0 or later, which reads Parquet in less memory.
+
 ## 0.2.0
 
 - **Fit resources (CPUs and memory) are constrained to your job's limits automatically.** On a

{dpd_stream-0.2.0 → dpd_stream-0.2.1}/CITATION.cff

@@ -7,8 +7,8 @@ authors:
   - family-names: Tucker
     given-names: Lee
     email: lee.c.tucker@census.gov
-version: 0.2.0
-date-released: 2026-10-31
+version: 0.2.1
+date-released: 2026-11-05
 license: MIT
 repository-code: "https://github.com/leetucker/dpd-stream"
 url: "https://github.com/leetucker/dpd-stream"

{dpd_stream-0.2.0 → dpd_stream-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dpd-stream
-Version: 0.2.0
+Version: 0.2.1
 Summary: Out-of-core dynamic panel GMM: Arellano-Bond difference GMM and Blundell-Bond system GMM, by streamed passes
 Author: Lee Tucker
 License-Expression: MIT
@@ -25,7 +25,7 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: hdfe-stream[formula]>=0.8.0
+Requires-Dist: hdfe-stream[formula]>=0.9.0
 Requires-Dist: stream-limits>=0.1.1
 Requires-Dist: polars!=1.41.*,>=1.37
 Requires-Dist: numpy>=1.24
@@ -44,6 +44,7 @@ Requires-Dist: sphinx-copybutton>=0.5; extra == "docs"
 Requires-Dist: sphinx-design>=0.6; extra == "docs"
 Provides-Extra: test
 Requires-Dist: pytest>=7.0; extra == "test"
+Requires-Dist: stream-limits[testing]>=0.2.1; extra == "test"
 Requires-Dist: pandas>=2.0; extra == "test"
 Dynamic: license-file
 

{dpd_stream-0.2.0 → dpd_stream-0.2.1}/dpd_stream/__init__.py

@@ -40,7 +40,7 @@ from .results import DPDMulti, DPDResult
 try:
     __version__ = _version("dpd-stream")
 except PackageNotFoundError:  # pragma: no cover
-    __version__ = "0.2.0"
+    __version__ = "0.2.1"
 
 __all__ = [
     "DPDMulti",

{dpd_stream-0.2.0 → dpd_stream-0.2.1}/dpd_stream/store.py

@@ -86,18 +86,18 @@ def build(lf: pl.LazyFrame, bases, panel, *, tdelta=None, cluster=(), weights=No
             raise ValueError(f"time variable {tvar!r} must be numeric (integer periods), "
                              f"got {schema[tvar]}")
     lf = lf.filter(pl.col(pid).is_not_null() & pl.col(tvar).is_not_null())
+    # (repeated times within a panel are found while the blocks are written,
+    # below: counting the distinct (panel, time) pairs here would hold a set
+    # of every row in memory)
     info = engine.collect(lf.select(
         pl.len().alias("n"), pl.col(tvar).min().alias("tmin"), pl.col(tvar).max().alias("tmax"),
-        pl.struct(pid, tvar).n_unique().alias("nu"), pl.col(pid).n_unique().alias("np"),
+        pl.col(pid).n_unique().alias("np"),
         (pl.col(tvar) != pl.col(tvar).round(0)).any().alias("frac")), "panel structure")
     n = int(info["n"][0])
     if n == 0:
         raise ValueError("no observations with a panel and a time")
     if info["frac"][0]:
         raise ValueError(f"time variable {tvar!r} must hold whole numbers")
-    if int(info["nu"][0]) != n:
-        raise ValueError(f"repeated time values within panel: ({pid}, {tvar}) does not "
-                         "identify the observations")
     tmin, tmax = int(info["tmin"][0]), int(info["tmax"][0])
     if tdelta is None:
         tdelta = 1
@@ -194,6 +194,11 @@ def build(lf: pl.LazyFrame, bases, panel, *, tdelta=None, cluster=(), weights=No
         frame = frame.sort([pid, _T])       # whole panels: sorted by time within each
         ids = frame[pid]
         starts = (ids != ids.shift(1)).fill_null(True).to_numpy()
+        # a block holds whole panels, so a repeated time is two neighbouring rows
+        if ((~starts[1:]) & (np.diff(frame[_T].to_numpy()) == 0)).any():
+            sample.unlink(missing_ok=True)
+            raise ValueError(f"repeated time values within panel: ({pid}, {tvar}) does not "
+                             "identify the observations")
         pcode = np.cumsum(starts).astype(np.int64) - 1 + code
         code = int(pcode[-1]) + 1
         B = (frame.select(base_names).to_numpy().astype(np.float64)

{dpd_stream-0.2.0 → dpd_stream-0.2.1}/dpd_stream.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dpd-stream
-Version: 0.2.0
+Version: 0.2.1
 Summary: Out-of-core dynamic panel GMM: Arellano-Bond difference GMM and Blundell-Bond system GMM, by streamed passes
 Author: Lee Tucker
 License-Expression: MIT
@@ -25,7 +25,7 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: hdfe-stream[formula]>=0.8.0
+Requires-Dist: hdfe-stream[formula]>=0.9.0
 Requires-Dist: stream-limits>=0.1.1
 Requires-Dist: polars!=1.41.*,>=1.37
 Requires-Dist: numpy>=1.24
@@ -44,6 +44,7 @@ Requires-Dist: sphinx-copybutton>=0.5; extra == "docs"
 Requires-Dist: sphinx-design>=0.6; extra == "docs"
 Provides-Extra: test
 Requires-Dist: pytest>=7.0; extra == "test"
+Requires-Dist: stream-limits[testing]>=0.2.1; extra == "test"
 Requires-Dist: pandas>=2.0; extra == "test"
 Dynamic: license-file
 

{dpd_stream-0.2.0 → dpd_stream-0.2.1}/dpd_stream.egg-info/SOURCES.txt

@@ -41,4 +41,5 @@ tests/test_oracle.py
 tests/test_profile.py
 tests/test_published.py
 tests/test_pydynpd.py
+tests/test_resources.py
 tests/test_stata.py

{dpd_stream-0.2.0 → dpd_stream-0.2.1}/dpd_stream.egg-info/requires.txt

@@ -1,4 +1,4 @@
-hdfe-stream[formula]>=0.8.0
+hdfe-stream[formula]>=0.9.0
 stream-limits>=0.1.1
 polars!=1.41.*,>=1.37
 numpy>=1.24
@@ -20,4 +20,5 @@ sphinx-design>=0.6
 
 [test]
 pytest>=7.0
+stream-limits[testing]>=0.2.1
 pandas>=2.0

{dpd_stream-0.2.0 → dpd_stream-0.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dpd-stream"
-version = "0.2.0"
+version = "0.2.1"
 description = "Out-of-core dynamic panel GMM: Arellano-Bond difference GMM and Blundell-Bond system GMM, by streamed passes"
 readme = "README.md"
 license = "MIT"
@@ -30,7 +30,7 @@ requires-python = ">=3.10"
 dependencies = [
     # the row store, run directories, the formula compiler and the result
     # base classes (hdfe_stream.core's stable names)
-    "hdfe-stream[formula]>=0.8.0",   # core.resources and core.profile are new in 0.8.0
+    "hdfe-stream[formula]>=0.9.0",   # 0.9.0: lower-memory Parquet batch reads (iter_groups)
     "stream-limits>=0.1.1",     # thread and memory limits for jobs on a scheduler (0.1.1: Windows memory)
     "polars>=1.37,!=1.41.*",
     "numpy>=1.24",
@@ -65,6 +65,7 @@ docs = [
 ]
 test = [
     "pytest>=7.0",
+    "stream-limits[testing]>=0.2.1",   # resource_runner, for tests/test_resources.py
     "pandas>=2.0",
 ]
 
@@ -76,6 +77,8 @@ include = ["dpd_stream*"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+# memory tests (tests/test_resources.py) are slow: run them with `pytest -m resource`
+addopts = "-m 'not resource'"
 filterwarnings = [
     "ignore::DeprecationWarning",
     "ignore::FutureWarning",

{dpd_stream-0.2.0 → dpd_stream-0.2.1}/tests/test_features.py

@@ -59,6 +59,20 @@ def test_fweights_must_be_constant(panel_data):
                    verbose=False)
 
 
+@pytest.mark.parametrize("where", ["first", "middle", "last"])
+def test_repeated_time_within_a_panel(panel_data, where):
+    """Found while the blocks are written (each holds whole panels, sorted by
+    time), in any panel and with the rows in any order; a time shared by two
+    panels is not a repeat."""
+    ids = panel_data["id"].unique().sort()
+    pid = {"first": ids[0], "middle": ids[len(ids) // 2], "last": ids[-1]}[where]
+    df = pl.concat([panel_data, panel_data.filter(pl.col("id") == pid).head(1)])
+    with pytest.raises(ValueError, match="repeated time values within panel"):
+        dpd_stream(FML, df.sample(fraction=1.0, shuffle=True, seed=2), panel=("id", "year"),
+                   verbose=False, block_mb=0.01)
+    assert dpd_stream(FML, panel_data, panel=("id", "year"), verbose=False).n_obs > 0
+
+
 def test_iterated(panel_data):
     two = dpd_stream(FML, panel_data, panel=("id", "year"), verbose=False)
     it0 = dpd_stream(FML, panel_data, panel=("id", "year"), steps="iterated", maxiter=0,

dpd_stream-0.2.1/tests/test_resources.py

@@ -0,0 +1,96 @@
+"""Memory tests: the streamed fits stay inside a memory limit on data that
+would not fit in it.
+
+Run with ``pytest -m resource`` (the default run leaves them out: they take
+about two minutes). Each fit runs in a child process under stream-limits'
+``resource_runner``: a watchdog on every platform, or a kernel limit when
+``STREAM_LIMITS_TEST_CGROUP_PARENT`` names a cgroup set up for it.
+
+The panel is 3 million firms over 8 years, 24 million rows, 0.96 GB in
+memory, written as six Parquet files and read as a glob. With
+``max_memory_gb=1.0`` difference and system GMM hold about 1.1 GB of private
+memory (not counting memory-mapped scratch files, which the kernel can
+reclaim), most of it buffers whose sizes are capped: the same on 9.6 million
+rows. The limit, 1.6 GB, leaves 0.5 GB for platform differences; a fit that
+held the panel in memory would need at least 2 GB. Measured on Linux (WSL2,
+2 threads): difference GMM 1.09 GB, system GMM 1.07 GB.
+
+The limit is on private memory, which stream-limits measures on Linux and
+Windows. On macOS it measures resident memory, which counts the mapped
+scratch files, so there the limit is a looser safety net.
+On Windows, private memory is committed memory: a buffer or an allocator
+segment counts in full once allocated, touched or not (an hdfe-stream logit
+fit had 1.61 GB committed while 1.14 GB was resident). Measured in CI
+(GitHub's windows-latest, 2 threads): difference GMM 1.46 GB, system GMM
+1.38 GB; the limit there is 1.8 GB, which a fit holding the panel would
+still exceed.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+
+import polars as pl
+import pytest
+
+from dpd_stream.simulate import simulate_dynamic
+
+pytestmark = pytest.mark.resource
+
+PARTS, PANELS_PER_PART, T = 6, 500_000, 8       # 24 million rows
+BUDGET_GB = 1.0                                 # the fit's max_memory_gb
+LIMIT_GB = {"darwin": 2.6, "win32": 1.8}.get(sys.platform, 1.6)
+FML = "y ~ l(y, 1) + x + w | year | gmm(y, 2:4) + gmm(x, 1:3) + iv(w)"
+
+
+@pytest.fixture(scope="module")
+def big_panel(tmp_path_factory):
+    """Simulated in parts (the whole panel at once needs 7 GB to simulate),
+    each part's firms numbered after the last part's."""
+    folder = tmp_path_factory.mktemp("big")
+    size = 0
+    for i in range(PARTS):
+        d = simulate_dynamic(PANELS_PER_PART, T, seed=100 + i)
+        d = d.with_columns(pl.col("id") + i * PANELS_PER_PART)
+        d.write_parquet(folder / f"part{i}.parquet")
+        size += d.estimated_size()
+    assert size > 0.9e9
+    return folder / "*.parquet"
+
+
+def _fit_in_child(resource_runner, system: bool, path, workdir) -> dict:
+    code = f"""
+import json
+from dpd_stream import dpd_stream
+r = dpd_stream({FML!r}, {str(path)!r}, panel=("id", "year"), system={system},
+               workdir={str(workdir)!r}, verbose=False, max_memory_gb={BUDGET_GB})
+print(json.dumps(dict(n_obs=int(r.n_obs), coef={{k: float(v) for k, v in r.coef().items()}})))
+"""
+    result = resource_runner([sys.executable, "-c", code], memory_gb=LIMIT_GB,
+                             threads=2, timeout=900)
+    report = {k: result.as_dict()[k] for k in ("status", "mode", "memory_measure",
+                                               "peak_private_bytes", "peak_rss_bytes",
+                                               "elapsed_s")}
+    print(json.dumps(report))
+    assert result.status == "ok", (report, result.stderr[-3000:])
+    if result.kernel:
+        assert result.kernel["events"].get("oom_kill", 0) == 0
+    return json.loads(result.stdout.strip().splitlines()[-1])
+
+
+# The checks against the simulated coefficients (simulate_dynamic's defaults:
+# rho 0.5, beta 1) are sanity checks only: agreement with xtabond2 is the rest
+# of the suite's job.
+
+def test_difference_gmm_streams_inside_the_limit(resource_runner, big_panel, tmp_path):
+    out = _fit_in_child(resource_runner, False, big_panel, tmp_path / "work")
+    assert out["n_obs"] == PARTS * PANELS_PER_PART * (T - 2)
+    assert abs(out["coef"]["l(y, 1)"] - 0.5) < 0.02
+    assert abs(out["coef"]["x"] - 1.0) < 0.02
+
+
+def test_system_gmm_streams_inside_the_limit(resource_runner, big_panel, tmp_path):
+    out = _fit_in_child(resource_runner, True, big_panel, tmp_path / "work")
+    assert abs(out["coef"]["l(y, 1)"] - 0.5) < 0.02
+    assert abs(out["coef"]["x"] - 1.0) < 0.02