acteval 0.1.0__tar.gz → 0.1.2__tar.gz

acteval-0.1.2/.github/workflows/benchmark.yml

@@ -0,0 +1,40 @@
+name: Benchmark
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: astral-sh/setup-uv@v4
+
+      - name: Install Python 3.13
+        run: uv python install 3.13
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Run benchmarks
+        run: >
+          uv run pytest tests/test_bench_evaluate.py tests/test_bench_pairwise.py
+          --benchmark-only
+          --benchmark-json output.json
+
+      - name: Store benchmark results
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: pytest
+          output-file-path: output.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          alert-threshold: "120%"
+          comment-on-alert: true
+          fail-on-alert: true
+          benchmark-data-dir-path: dev/bench

{acteval-0.1.0 → acteval-0.1.2}/.github/workflows/ci.yml

@@ -7,11 +7,12 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
 
     steps:
       - uses: actions/checkout@v4
@@ -25,10 +26,12 @@ jobs:
         run: uv sync
 
       - name: Lint
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'
         run: uv run ruff check src/ tests/
 
       - name: Check formatting
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'
         run: uv run black --check src/ tests/
 
       - name: Run tests
-        run: uv run pytest tests/ --cov=src/acteval
+        run: uv run pytest tests/ --ignore=tests/test_bench_evaluate.py --cov=src/acteval

{acteval-0.1.0 → acteval-0.1.2}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.4
 Name: acteval
-Version: 0.1.0
+Version: 0.1.2
 Summary: Standalone library for evaluating activity sequences
-Requires-Python: >=3.8
+Requires-Python: >=3.10
 Requires-Dist: matplotlib
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: pot
-Requires-Dist: scipy<=1.13
+Requires-Dist: scipy
 Requires-Dist: tabulate
 Requires-Dist: tomli>=1.1.0; python_version < '3.11'
+Requires-Dist: tslearn

acteval-0.1.2/README.md

@@ -0,0 +1,319 @@
+# acteval
+
+A Python library for evaluating synthetic activity schedules by comparing them to observed data. Given a population of daily activity sequences (who did what and when), `acteval` measures how well a synthetic population reproduces the observed distribution across multiple dimensions: activity frequencies, timing, transitions, participation rates, and novelty.
+
+## Install
+
+```bash
+pip install acteval
+```
+
+Or with [uv](https://github.com/astral-sh/uv):
+
+```bash
+uv add acteval
+```
+
+## CLI
+
+`acteval` ships with a command-line interface for comparing models without writing Python.
+
+```bash
+# Compare one model to observed data
+acteval observed.csv --model my_model synthetic.csv
+
+# Compare multiple models side-by-side
+acteval observed.csv \
+  --model model_a synthetic_a.csv \
+  --model model_b synthetic_b.csv
+
+# Save full results to CSV files
+acteval observed.csv --model my_model synthetic.csv --output results/
+
+# Show group-level detail instead of domain summary
+acteval observed.csv --model my_model synthetic.csv --level groups
+
+# Split evaluation by attribute (e.g. gender)
+acteval observed.csv \
+  --target-attrs target_attrs.csv --split-on gender \
+  --model my_model synthetic.csv --attrs my_model synth_attrs.csv
+
+# Use a custom config file
+acteval observed.csv --model my_model synthetic.csv --config custom.toml
+```
+
+Input files can be CSV or Parquet (detected by extension). Run `acteval --help` for the full option list.
+
+## Quick start
+
+```python
+import pandas as pd
+from acteval import compare
+
+observed = pd.DataFrame([
+    {"pid": 0, "act": "home", "start": 0,  "end": 8,  "duration": 8},
+    {"pid": 0, "act": "work", "start": 8,  "end": 16, "duration": 8},
+    {"pid": 0, "act": "home", "start": 16, "end": 24, "duration": 8},
+    {"pid": 1, "act": "home", "start": 0,  "end": 12, "duration": 12},
+    {"pid": 1, "act": "shop", "start": 12, "end": 13, "duration": 1},
+    {"pid": 1, "act": "home", "start": 13, "end": 24, "duration": 11},
+])
+
+synthetic = pd.DataFrame([
+    {"pid": 0, "act": "home", "start": 0,  "end": 9,  "duration": 9},
+    {"pid": 0, "act": "work", "start": 9,  "end": 17, "duration": 8},
+    {"pid": 0, "act": "home", "start": 17, "end": 24, "duration": 7},
+    {"pid": 1, "act": "home", "start": 0,  "end": 8,  "duration": 8},
+    {"pid": 1, "act": "work", "start": 8,  "end": 16, "duration": 8},
+    {"pid": 1, "act": "home", "start": 16, "end": 24, "duration": 8},
+    {"pid": 2, "act": "home", "start": 0,  "end": 8,  "duration": 8},
+    {"pid": 2, "act": "home", "start": 8, "end": 24, "duration": 16},
+])
+
+result = compare(observed, {"my_model": synthetic})
+print(result.summary())
+# creativity      0.166667
+# feasibility     0.500000
+# participations  0.162037
+# timing          0.082728
+# transitions     0.380952
+```
+
+## Input format
+
+Data is passed as a pandas DataFrame with one row per activity episode:
+
+| column | type | description |
+|--------|------|-------------|
+| `pid` | int/str | Person identifier |
+| `act` | str | Activity label (e.g. `"home"`, `"work"`, `"shop"`) |
+| `start` | numeric | Start time (any consistent unit, e.g. hours) |
+| `end` | numeric | End time |
+| `duration` | numeric | Duration (`end - start`); can be omitted when both `start` and `end` are provided |
+
+Any two of `start`, `end`, and `duration` are sufficient — the third is derived automatically.
+
+
+```python
+import pandas as pd
+
+observed = pd.DataFrame([
+    {"pid": 0, "act": "home", "start": 0,  "end": 6,  "duration": 6},
+    {"pid": 0, "act": "work", "start": 6,  "end": 14, "duration": 8},
+    {"pid": 0, "act": "home", "start": 14, "end": 24, "duration": 10},
+    {"pid": 1, "act": "home", "start": 0,  "end": 10, "duration": 10},
+    {"pid": 1, "act": "work", "start": 10, "end": 24, "duration": 14},
+])
+```
+
+## API
+
+### `compare(observed, synthetic, **kwargs)`
+
+Compare one or more synthetic populations to an observed population.
+
+```python
+from acteval import compare
+
+# Single synthetic population
+result = compare(observed, synthetic)
+
+# Multiple models side-by-side
+result = compare(observed, {"model_a": synthetic_a, "model_b": synthetic_b})
+```
+
+`result` is an `EvalResult` object. See [Reading the results](#reading-the-results) for how to access distances and descriptions at feature, group, and domain level.
+
+### `Evaluator`
+
+Use `Evaluator` when comparing multiple synthetic populations against the same observed data — it computes and caches the observed features once. Each `compare()` call is independent.
+
+```python
+from acteval import Evaluator
+
+evaluator = Evaluator(observed)
+
+result_v1 = evaluator.compare({"v1": synthetic_v1})
+result_v2 = evaluator.compare({"v2": synthetic_v2})
+```
+
+### `pairwise_distances(schedules, specs=None)`
+
+Compute a single NxN distance matrix between individual schedules. Useful for clustering, outlier detection, or directly comparing a small batch of schedules.
+
+```python
+from acteval import pairwise_distances
+
+result = pairwise_distances(schedules)
+result.matrix          # numpy array, shape (N, N)
+result.pids            # original pid values, length N
+
+# Get a labeled DataFrame with original pid values as index/columns
+df = result.to_dataframe()
+
+# Example: find the two most similar schedules
+import numpy as np
+dist = result.to_dataframe()
+dist.values[np.arange(len(dist)), np.arange(len(dist))] = np.inf
+i, j = np.unravel_index(dist.values.argmin(), dist.shape)
+print(f"Most similar: {dist.index[i]} and {dist.columns[j]}")
+```
+
+The result matrix is symmetric with zeros on the diagonal. All values are in **0–1**.
+
+#### Pluggable distance specs
+
+By default, three equal-weight semantic-distance specs are used (participations, transitions, timing via MAE). Pass a custom `specs` list to change the metrics or their relative weights:
+
+```python
+from acteval.pairwise import chamfer_spec, soft_dtw_spec, default_pairwise_specs
+
+# Chamfer distance on EOS-padded activity sequences
+result = pairwise_distances(schedules, specs=[chamfer_spec()])
+
+# Soft-DTW on EOS-padded activity sequences
+result = pairwise_distances(schedules, specs=[soft_dtw_spec(gamma=1.0)])
+
+# Mix metrics with custom weights
+result = pairwise_distances(schedules, specs=[
+    *default_pairwise_specs(),       # weight=1.0 each
+    chamfer_spec(weight=1.0),
+    soft_dtw_spec(weight=2.0),
+])
+```
+
+Each spec defines a `feature_fn` (extracts a `(N, ...)` array from the population) and a `distance_fn` (computes the `(N, N)` matrix). The final matrix is a weighted average across all active specs.
+
+| Factory | Description |
+|---------|-------------|
+| `default_pairwise_specs()` | MAE on participation counts, bi-gram counts, mean durations |
+| `chamfer_spec(max_len, weight)` | Chamfer distance on EOS-padded `(N, L, 2)` sequences |
+| `soft_dtw_spec(max_len, gamma, weight)` | Soft-DTW on EOS-padded `(N, L, 2)` sequences |
+
+### `Population`
+
+For direct access to the underlying data structure:
+
+```python
+from acteval import Population
+
+pop = Population(observed)
+print(pop.acts)       # activity labels per episode
+print(pop.durations)  # durations as numpy array
+```
+
+## Reading the results
+
+### Accessing results
+
+`compare()` returns an `EvalResult` with distances and descriptions at three levels of aggregation. Each level is accessed via a property that returns a view object with `.distances` and `.descriptions` DataFrames:
+
+| Property | Index | Content |
+|----------|-------|---------|
+| `result.features.combined.distances` | `(domain, feature, segment)` | Per-feature distances — lower is closer to observed |
+| `result.features.combined.descriptions` | `(domain, feature, segment)` | Per-feature descriptive statistics (e.g. average start time) |
+| `result.groups.combined.distances` | `(domain, feature)` | Distances averaged across segments |
+| `result.groups.combined.descriptions` | `(domain, feature)` | Descriptions averaged across segments |
+| `result.domains.combined.distances` | `(domain,)` | Distances averaged across features — one row per domain |
+| `result.domains.combined.descriptions` | `(domain,)` | Descriptions averaged across features |
+
+Save all levels to CSV at once with `result.save("output_dir/")`.
+
+Distances are in the range **0–1** (lower is better). A distance of `0.0` means the synthetic distribution perfectly matches observed; `1.0` is the maximum penalty.
+
+> **Note on timing features:** A distance of `1.0` for a timing feature means the activity is *entirely absent* from the synthetic population — not just timed differently. This is treated as a maximum-penalty missing feature rather than a distributional difference.
+
+### Evaluation domains
+
+| Domain | What it measures |
+|--------|-----------------|
+| `participations` | Who does what and how often — participation rates, joint participation, sequence lengths |
+| `transitions` | Activity sequences — 2-, 3-, and 4-gram transition patterns |
+| `timing` | When and how long — start times, durations, and their joint distributions |
+| `creativity` | How novel and diverse the synthetic schedules are relative to observed |
+| `feasibility` | Structural validity — home-based schedules, no consecutive duplicate activities |
+
+### Ranking models
+
+```python
+result = compare(observed, {"model_a": df_a, "model_b": df_b})
+
+# Mean domain distance per model (lower is better)
+print(result.rank_models())
+# model_a    0.12
+# model_b    0.19
+
+# Best model
+print(result.best_model)   # "model_a"
+
+# Domain-level summary table
+print(result.summary())
+#                   model_a  model_b
+# domain
+# creativity           0.08     0.14
+# feasibility          0.03     0.05
+# participations       0.11     0.18
+# timing               0.16     0.22
+# transitions          0.22     0.35
+```
+
+## Development
+
+```bash
+# Install dependencies (including dev tools)
+uv sync
+
+# Run tests (benchmarks excluded by default)
+uv run pytest tests/
+
+# Run tests with coverage
+uv run pytest --cov=src/acteval tests/
+
+# Lint with ruff
+ruff check src/ tests/
+
+# Auto-fix lint issues
+ruff check --fix src/ tests/
+
+# Check formatting with black
+black --check src/ tests/
+
+# Auto-format
+black src/ tests/
+```
+
+## Benchmarks
+
+Two benchmark suites are included, both using [pytest-benchmark](https://pytest-benchmark.readthedocs.io/).
+
+### Population evaluation (`compare`)
+
+Tests `compare()` at 1k, 20k, and 100k rows (observed + synthetic populations):
+
+```bash
+uv run pytest tests/test_bench_evaluate.py --benchmark-only -v
+```
+
+### Pairwise distances (`pairwise_distances`)
+
+Tests `pairwise_distances()` at 256, 512, and 1024 schedules:
+
+```bash
+uv run pytest tests/test_bench_pairwise.py --benchmark-only -v
+```
+
+### Both suites together
+
+```bash
+uv run pytest tests/test_bench_evaluate.py tests/test_bench_pairwise.py --benchmark-only -v
+```
+
+### Comparing runs
+
+```bash
+# Save a baseline
+uv run pytest tests/test_bench_evaluate.py tests/test_bench_pairwise.py --benchmark-only --benchmark-save=baseline
+
+# Compare against it after making changes
+uv run pytest tests/test_bench_evaluate.py tests/test_bench_pairwise.py --benchmark-only --benchmark-compare=baseline
+```

acteval-0.1.2/llms.txt

@@ -0,0 +1,68 @@
+# acteval
+
+> Python library for evaluating synthetic activity schedules: given daily sequences of who-did-what-and-when, measures how well a synthetic population reproduces observed distributions across frequencies, timing, transitions, participation rates, and novelty. Distances are 0–1 (lower is better).
+
+## Install
+
+pip install acteval
+
+## Input format
+
+pandas DataFrame, one row per activity episode:
+- pid — person identifier (int or str)
+- act — activity label (str, e.g. "home", "work", "shop")
+- start — episode start time (numeric, any consistent unit)
+- end — episode end time (numeric)
+- duration — episode duration (numeric)
+
+## Key API
+
+- compare(observed, synthetic, **kwargs) → EvalResult
+  Compare one or more synthetic DataFrames to observed.
+  synthetic can be a single DataFrame or {"name": df, ...} dict.
+
+- Evaluator(observed) — cache observed features for repeated comparisons.
+  evaluator.compare({"v1": df_v1}) re-uses cached observed features.
+
+- pairwise_distances(schedules, specs=None) → PairwiseResult
+  NxN distance matrix between individual schedules.
+  result.matrix (numpy), result.to_dataframe() (labeled).
+
+## EvalResult structure
+
+Six DataFrames at three aggregation levels:
+- distances / descriptions              — per-feature (index: domain, feature, segment)
+- group_distances / group_descriptions  — per feature group (index: domain, feature)
+- domain_distances / domain_descriptions — per domain (index: domain)
+
+Convenience: result.summary(), result.rank_models(), result.best_model
+
+## Evaluation domains
+
+- participations — who does what and how often; participation rates, joint participation
+- transitions    — activity sequence patterns (2-, 3-, 4-gram bigrams)
+- timing         — start times and durations (EMD via Wasserstein distance)
+- creativity     — novelty and diversity of synthetic schedules vs observed
+- feasibility    — structural validity: home-based, no consecutive duplicate activities
+
+Missing activity penalty: 1.0 (maximum) for timing; None (compute on available data) for participation/transitions.
+
+## Pairwise distance specs
+
+from acteval.pairwise import chamfer_spec, soft_dtw_spec, default_pairwise_specs
+- default_pairwise_specs() — MAE on participation counts, bigram counts, mean durations
+- chamfer_spec(max_len, weight) — Chamfer on EOS-padded (N, L, 2) sequences
+- soft_dtw_spec(max_len, gamma, weight) — Soft-DTW on EOS-padded sequences
+
+## Source layout
+
+src/acteval/
+- evaluate.py       — public API: compare(), Evaluator, evaluate()
+- _pipeline.py      — evaluation engine; builds features → groups → domains output
+- population.py     — Population: core data structure, integer-encodes activities/persons
+- _jobs.py          — JobSpec dataclass, get_jobs(); reads config.toml
+- pairwise.py       — pairwise_distances(), PairwiseResult, spec factories
+- features/         — structural, frequency, times, transitions, participation, creativity
+- distance/         — wasserstein.py (EMD via POT), scalar.py (MAE/MSE/MAPE)
+- describe/         — descriptive stats alongside distance metrics
+- _filters.py       — filter_novel(): removes sequences seen in base population

{acteval-0.1.0 → acteval-0.1.2}/pyproject.toml

@@ -1,20 +1,33 @@
 [project]
 name = "acteval"
-version = "0.1.0"
+version = "0.1.2"
 description = "Standalone library for evaluating activity sequences"
-requires-python = ">=3.8"
+requires-python = ">=3.10"
 dependencies = [
     "pandas",
     "numpy",
-    "scipy<=1.13",
+    "scipy",
     "pot",
     "matplotlib",
     "tabulate",
     "tomli>=1.1.0; python_version < '3.11'",
+    "tslearn",
 ]
 
 [tool.uv]
-dev-dependencies = ["pytest", "pytest-cov", "ruff", "black"]
+dev-dependencies = [
+    "pytest",
+    "pytest-cov",
+    "pytest-benchmark",
+    "ruff",
+    "black",
+    "matplotlib>=3.7.5",
+    "seaborn>=0.13.2",
+    "polars>=1.8.2",
+]
+
+[project.scripts]
+acteval = "acteval.cli:main"
 
 [build-system]
 requires = ["hatchling"]
@@ -25,9 +38,10 @@ packages = ["src/acteval"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+addopts = "--ignore=tests/test_bench_evaluate.py --ignore=tests/test_bench_pairwise.py --benchmark-columns=mean,stddev,rounds --benchmark-sort=name"
 
 [tool.ruff]
-target-version = "py39"
+target-version = "py310"
 
 [tool.ruff.lint]
 select = ["E", "F", "I", "W"]
@@ -37,4 +51,4 @@ ignore = ["E501"]
 "tests/test_distance_emd.py" = ["E402"]
 
 [tool.black]
-target-version = ["py39"]
+target-version = ["py310"]