dwind 0.3__py3-none-any.whl → 0.3.2__py3-none-any.whl

dwind/__init__.py

@@ -1,3 +1,3 @@
 from .config import Configuration
 
-__version__ = "0.3"
+__version__ = "0.3.2"

dwind/btm_sizing.py

@@ -5,7 +5,7 @@ import logging
 import numpy as np
 import pandas as pd
 
-from dwind import Configuration, helper
+from dwind import Configuration
 
 
 log = logging.getLogger("dwfs")
@@ -117,7 +117,7 @@ def sizer(agents: pd.DataFrame, config: Configuration):
             agents.drop_duplicates(subset=["gid"], inplace=True)
 
         # make small
-        agents = helper.memory_downcaster(agents)
+        # agents = array.memory_downcaster(agents)
 
         return agents
 

dwind/cli/collect.py

@@ -0,0 +1,114 @@
+"""Enables running dwind as a CLI tool, the primary interface for working with dwind."""
+
+from __future__ import annotations
+
+from typing import Optional, Annotated
+from pathlib import Path
+
+import typer
+import pandas as pd
+
+from dwind.cli import utils
+
+
+app = typer.Typer()
+
+DWIND = Path("/projects/dwind/agents")
+
+
+@app.command()
+def combine_chunks(
+    dir_out: Annotated[
+        str,
+        typer.Argument(
+            help=(
+                "Path to where the chunked outputs should be saved. Should be the same that was"
+                " passed to the run command."
+            )
+        ),
+    ],
+    file_name: Annotated[
+        Optional[str],  # noqa
+        typer.Option(
+            "--file-name",
+            "-f",
+            help="Custom filename, without the extension (e.g. .pqt), for the results data.",
+        ),
+    ] = None,
+    remove_results_chunks: Annotated[
+        bool,
+        typer.Option(
+            "--remove-results-chunks/--no-remove-results-chunks",
+            "-rr/-RR",
+            help="Delete the individual chunk files after saving the combined results.",
+        ),
+    ] = True,
+):
+    """Combine the results of a multi-job run based on the run's TOML configuration. Please note
+    this has the potential to combine multiple runs as it does not respect the jobs ran during a
+    single cycle.
+    """
+    dir_out = Path.cwd() if dir_out is None else Path(dir_out).resolve()
+    out_path = dir_out / "chunk_files"
+    result_files = [f for f in out_path.iterdir() if f.suffix == (".pqt")]
+
+    if len(result_files) > 0:
+        file_name = "results" if file_name is None else file_name
+        result_agents = pd.concat([pd.read_parquet(f) for f in result_files])
+        f_out = dir_out / f"{file_name}.pqt"
+        result_agents.to_parquet(f_out)
+        print(f"Aggregated results saved to: {f_out}")
+        return None
+
+        if remove_results_chunks:
+            for f in result_files:
+                f.unlink()
+                print(f"Removed: {f}")
+    else:
+        print(f"No chunked results found in: {out_path}.")
+
+
+@app.command()
+def cleanup_agents(
+    dir_out: Annotated[
+        str,
+        typer.Argument(
+            help=(
+                "Path to where the chunked agents were saved. Should be the same that was"
+                " passed to the run command."
+            )
+        ),
+    ],
+):
+    """Deletes the temporary agent chunk files generated at runtime.
+
+    Args:
+        dir_out (str): The base output directory, which should be the same as that passed to the
+            run command.
+    """
+    utils.cleanup_chunks(dir_out, which="agents")
+
+
+@app.command()
+def cleanup_results(
+    dir_out: Annotated[
+        str,
+        typer.Argument(
+            help=(
+                "Path to where the results agents were saved. Should be the same that was"
+                " passed to the run command."
+            )
+        ),
+    ],
+):
+    """Deletes the chunked results files generated at runtime.
+
+    Args:
+        dir_out (str): The base output directory, which should be the same as that passed to the
+            run command.
+    """
+    utils.cleanup_chunks(dir_out, which="results")
+
+
+if __name__ == "__main__":
+    app()

dwind/cli/debug.py

@@ -0,0 +1,137 @@
+"""Enables running dwind as a CLI tool, the primary interface for working with dwind."""
+
+from __future__ import annotations
+
+import itertools
+from typing import Optional, Annotated
+from pathlib import Path
+
+import typer
+import pandas as pd
+from rich.console import Console
+
+from dwind.cli import utils
+from dwind.utils import hpc
+
+
+# Create the main app with the primary "run", "debug", and "collect" commands
+app = typer.Typer()
+
+console = Console()
+
+DWIND = Path("/projects/dwind/agents")
+
+
+@app.command()
+def job_summary(
+    jobs: Annotated[list[str], typer.Argument(help="Job ID(s) to check for the final run status.")],
+    chunks: Annotated[
+        Optional[list[int]],  # noqa
+        typer.Option(
+            "--chunks",
+            "-c",
+            help="Corresponding chunk indexes for each job ID.",
+        ),
+    ] = None,
+):
+    """Print a table summary of the provided job id(s) and their final run status."""
+    if chunks is not None:
+        N = len(jobs)
+        M = len(chunks)
+        if N != M:
+            raise ValueError(f"Length of inputs to 'jobs' ({N}) not equal to 'chunks' ({M})")
+
+    jobs = hpc.get_finished_run_status(jobs)
+    summary = (
+        pd.DataFrame.from_dict(jobs, orient="index", columns=["status"])
+        .reset_index()
+        .rename(columns={"index": "job"})
+    )
+    if chunks is None:
+        utils.print_status_table(summary)
+    else:
+        summary["chunk"] = chunks
+        utils.print_status_table(summary, chunk_col="chunk")
+
+
+@app.command()
+def missing_agents_from_chunks(
+    dir_out: Annotated[
+        str,
+        typer.Argument(
+            help=(
+                "Path to where the chunked outputs were saved. Should match the inputs to the"
+                " run command."
+            )
+        ),
+    ],
+    chunks: Annotated[
+        Optional[list[int]], typer.Option(help="If used, specify chunked indicies to compare.")  # noqa
+    ] = None,
+):
+    """Compare the agent "gid" field between the chunked agents and results files, and save the
+    missing chunk index and agent gid as a 1-column CSV file.
+    """
+    dir_out = Path(dir_out).resolve()
+    results_path = dir_out / "chunk_files"
+    agents_path = results_path / "agent_chunks"
+
+    missing = []
+    agent_files = {
+        f.name.split("_")[-1]: f
+        for f in agents_path.iterdir()
+        if f.suffix == ".pqt" and f.name.startswith("agents")
+    }
+    results_files = {f.name.split("_")[-1]: f for f in results_path.iterdir() if f.suffix == ".pqt"}
+    shared_chunks = list({*agent_files}.intersection([*results_files]))
+    if chunks is not None:
+        shared_chunks = [c for c in shared_chunks if c in chunks]
+
+    for chunk in shared_chunks:
+        agents = pd.read_parquet(agent_files[chunk])[["gid"]]
+        results = pd.read_parquet(results_files[chunk])[["gid"]]
+        missing_gid = set(agents.gid.values).difference(results.gid.values)
+        missing.extend(list(zip(itertools.repeat(chunk), missing_gid)))
+
+    if missing:
+        df = pd.DataFrame(missing, columns=["chunk", "gid"])
+        df.to_csv(dir_out / "missing_agents_by_chunk.csv", index=False)
+        print(f"Saved missing agents to: {dir_out / 'missing_agents_by_chunk.csv'}")
+    else:
+        print("All agents produced results.")
+
+
+@app.command()
+def missing_agents(
+    agents_file: Annotated[str, typer.Argument(help="Full file path and name of the agents file.")],
+    results_file: Annotated[
+        str, typer.Argument(help="Full file path and name of the results file.")
+    ],
+    dir_out: Annotated[
+        str,
+        typer.Argument(
+            help="Full file path and name for where to save the list of missing agent gids."
+        ),
+    ],
+):
+    """Compare the agent "gid" field between the chunked agents and results files, and save the
+    missing chunk index and agent gid as a 2-column CSV file.
+    """
+    results_file = Path(results_file).resolve()
+    agents_file = Path(agents_file).resolve()
+    dir_out = Path(dir_out).resolve()
+
+    agents = pd.read_parquet(agents_file)[["gid"]]
+    results = pd.read_parquet(results_file)[["gid"]]
+    missing_gid = set(agents.gid.values).difference(results.gid.values)
+
+    if missing_gid:
+        df = pd.DataFrame(missing_gid, columns=["gid"])
+        df.to_csv(dir_out / "missing_agents.csv", index=False)
+        print(f"Saved missing agents to: {dir_out / 'missing_agents.csv'}")
+    else:
+        print("All agents produced results.")
+
+
+if __name__ == "__main__":
+    app()

dwind/cli/run.py

@@ -0,0 +1,288 @@
+"""Enables running dwind as a CLI tool, the primary interface for working with dwind."""
+
+from __future__ import annotations
+
+import sys
+from typing import Optional, Annotated
+from pathlib import Path
+
+import typer
+from rich.pretty import pprint
+
+from dwind.cli import debug, utils, collect
+from dwind.model import Model
+from dwind.config import Sector, Scenario
+
+
+# fmt: off
+if sys.version_info >= (3, 11):  # noqa
+    import tomllib
+else:
+    import tomli as tomllib
+# fmt: on
+
+app = typer.Typer()
+
+
+@app.command()
+def hpc(
+    location: Annotated[
+        str, typer.Argument(help="The state, state_county, or priority region to run.")
+    ],
+    sector: Annotated[
+        Sector, typer.Argument(help="One of fom (front of meter) or btm (back-of-the-meter).")
+    ],
+    scenario: Annotated[
+        Scenario,
+        typer.Argument(help="The scenario to run (baseline is the current only option)."),
+    ],
+    year: Annotated[
+        int,
+        typer.Argument(
+            callback=utils.year_callback,
+            help="The assumption year for the analysis. Options are 2022, 2024, and 2025.",
+        ),
+    ],
+    repository: Annotated[
+        str, typer.Argument(help="Path to the dwind repository to use when running the model.")
+    ],
+    nodes: Annotated[
+        int,
+        typer.Argument(
+            help="Number of HPC nodes or CPU nodes to run on. -1 indicates 75% of CPU limit."
+        ),
+    ],
+    allocation: Annotated[str, typer.Argument(help="HPC allocation name.")],
+    memory: Annotated[int, typer.Argument(help="Node memory, in GB (HPC only).")],
+    walltime: Annotated[int, typer.Argument(help="Node walltime request, in hours.")],
+    feature: Annotated[
+        str,
+        typer.Argument(
+            help=(
+                "Additional flags for the SLURM job, using formatting such as"
+                " --qos=high or --depend=[state:job_id]."
+            )
+        ),
+    ],
+    env: Annotated[
+        str,
+        typer.Argument(
+            help="The path to the dwind Python environment that should be used to run the model."
+        ),
+    ],
+    model_config: Annotated[
+        str, typer.Argument(help="Complete file name and path of the model configuration file")
+    ],
+    dir_out: Annotated[
+        str, typer.Argument(help="Path to where the chunked outputs should be saved.")
+    ],
+    stdout_path: Annotated[
+        Optional[str], typer.Option(help="The path to write stdout logs.")  # noqa
+    ] = None,
+    combine: Annotated[
+        bool,
+        typer.Option(
+            "--combine/--no-combine",
+            "-c/-C",
+            help="Automatically combine the chunked results after analysis completion.",
+        ),
+    ] = True,
+    remove_agent_chunks: Annotated[
+        bool,
+        typer.Option(
+            "--remove-agent-chunks/--no-remove-agent-chunks",
+            "-ra/-RA",
+            help="Delete the temporary agent chunk files.",
+        ),
+    ] = True,
+    remove_results_chunks: Annotated[
+        bool,
+        typer.Option(
+            "--remove-results-chunks/--no-remove-results-chunks",
+            "-rr/-RR",
+            help="Delete the chunked results files. Ignored if --combine is not passed.",
+        ),
+    ] = True,
+):
+    """Run dwind via the HPC multiprocessing interface."""
+    # sys.path.append(repository)
+    from dwind.mp import MultiProcess
+
+    # NOTE: collect_by_priority has been removed but may need to be reinstated
+
+    mp = MultiProcess(
+        location=location,
+        sector=sector,
+        scenario=scenario,
+        year=year,
+        env=env,
+        n_nodes=nodes,
+        memory=memory,
+        walltime=walltime,
+        allocation=allocation,
+        feature=feature,
+        repository=repository,
+        model_config=model_config,
+        dir_out=dir_out,
+        stdout_path=stdout_path,
+    )
+
+    agent_df = utils.load_agents(
+        location=location, sector=sector, model_config=model_config, prepare=True
+    )
+    job_chunk_map = mp.run_jobs(agent_df)
+    debug.job_summary(jobs=job_chunk_map.keys(), chunks=job_chunk_map.values())
+
+    if remove_agent_chunks:
+        utils.cleanup_chunks(dir_out, which="agents")
+
+    if combine:
+        run_name = f"{location}_{sector}_{scenario}_{year}"
+        collect.combine_chunks(
+            dir_out=dir_out, file_name=run_name, remove_results_chunks=remove_results_chunks
+        )
+
+
+@app.command()
+def interactive(
+    location: Annotated[
+        str, typer.Argument(help="The state, state_county, or priority region to run.")
+    ],
+    sector: Annotated[
+        Sector, typer.Argument(help="One of fom (front of meter) or btm (back-of-the-meter).")
+    ],
+    scenario: Annotated[Scenario, typer.Argument(help="The scenario to run, such as 'baseline'.")],
+    year: Annotated[
+        int, typer.Argument(callback=utils.year_callback, help="The year basis of the scenario.")
+    ],
+    dir_out: Annotated[str, typer.Argument(help="save path")],
+    repository: Annotated[
+        str, typer.Argument(help="Path to the dwind repository to use when running the model.")
+    ],
+    model_config: Annotated[
+        str, typer.Argument(help="Complete file name and path of the model configuration file")
+    ],
+    **kwargs: Annotated[
+        str, typer.Option(help="Do not pass arguments here, this is for internal overflow only.")
+    ],
+):
+    """Run dwind locally, or via an interactive session where a SLURM job is not scheduled."""
+    agents = utils.load_agents(
+        location=location, sector=sector, model_config=model_config, prepare=True
+    )
+    model = Model(
+        agents=agents,
+        location=location,
+        sector=sector,
+        scenario=scenario,
+        year=year,
+        out_path=dir_out,
+        model_config=model_config,
+    )
+    model.run()
+
+
+@app.command()
+def config(
+    config_path: Annotated[
+        str, typer.Argument(help="Path to configuration TOML with run and model parameters.")
+    ],
+    use_hpc: Annotated[
+        bool,
+        typer.Option(help="Run via sbatch on the HPC (--use-hpc) or interactively (--no-use-hpc)."),
+    ] = True,
+    combine: Annotated[
+        bool,
+        typer.Option(
+            "--combine/--no-combine",
+            "-c/-C",
+            help=(
+                "Automatically combine the chunked results after analysis completion. Ignored"
+                " if --no-use-hpc."
+            ),
+        ),
+    ] = True,
+    remove_agent_chunks: Annotated[
+        bool,
+        typer.Option(
+            "--remove-agent-chunks/--no-remove-agent-chunks",
+            "-ra/-RA",
+            help="Delete the temporary agent chunk files. Ignored if --no-use-hpc.",
+        ),
+    ] = True,
+    remove_results_chunks: Annotated[
+        bool,
+        typer.Option(
+            "--remove-results-chunks/--no-remove-results-chunks",
+            "-rr/-RR",
+            help=(
+                "Delete the chunked results files. Ignored if --combine is not passed or"
+                " --no-use-hpc is passed."
+            ),
+        ),
+    ] = True,
+):
+    """Run dwind via the HPC multiprocessing interface from a configuration file."""
+    config_path = Path(config_path).resolve()
+    with config_path.open("rb") as f:
+        config = tomllib.load(f)
+    print("Running the following configuration:")
+    pprint(config)
+
+    if use_hpc:
+        hpc(
+            combine=combine,
+            remove_agent_chunks=remove_agent_chunks,
+            remove_results_chunks=remove_results_chunks,
+            **config,
+        )
+    else:
+        interactive(**config)
+
+
+@app.command()
+def chunk(
+    chunk_ix: Annotated[int, typer.Argument(help="Chunk number/index. Used for logging.")],
+    location: Annotated[
+        str, typer.Argument(help="The state, state_county, or priority region to run.")
+    ],
+    sector: Annotated[
+        Sector, typer.Argument(help="One of fom (front of meter) or btm (back-of-the-meter).")
+    ],
+    scenario: Annotated[Scenario, typer.Argument(help="The scenario to run, such as baseline.")],
+    year: Annotated[
+        int, typer.Argument(callback=utils.year_callback, help="The year basis of the scenario.")
+    ],
+    out_path: Annotated[str, typer.Argument(help="save path")],
+    repository: Annotated[
+        str, typer.Argument(help="Path to the dwind repository to use when running the model.")
+    ],
+    model_config: Annotated[
+        str, typer.Argument(help="Complete file name and path of the model configuration file")
+    ],
+):
+    """Run a chunk of a dwind model. Internal only, do not run outside the context of a
+    chunked analysis or debugging.
+    """
+    # Import the correct version of the library
+    sys.path.append(repository)
+    from dwind.model import Model
+
+    agent_file = Path(out_path).resolve() / f"agent_chunks/agents_{chunk_ix}.pqt"
+    agents = utils.load_agents(file_name=agent_file, prepare=False)
+
+    model = Model(
+        agents=agents,
+        location=location,
+        sector=sector,
+        scenario=scenario,
+        year=year,
+        chunk_ix=chunk_ix,
+        out_path=out_path,
+        model_config=model_config,
+    )
+    model.run()
+
+
+if __name__ == "__main__":
+    app()