pipstools 0.2.1__py3-none-any.whl

pipstools/__init__.py

@@ -0,0 +1,5 @@
+from .annotation import annotate_problem as annotate_problem
+from .conversion import convert_problem as convert_problem
+from .scoring import score_problem as score_problem
+from .validation import validate_problem as validate_problem
+from .visualisation import visualize_problem as visualize_problem

pipstools/annotation.py

@@ -0,0 +1,204 @@
+import gzip
+import importlib.util
+import shutil
+from pathlib import Path
+from time import perf_counter
+
+from pipstools.io import read_gdx, read_mps, read_parquet, write_gdx, write_parquet
+from pipstools.partitioning import get_partitions
+from pipstools.scoring import get_score
+from pipstools.utils import get_stem
+
+
+def annotate_problem(
+    f_input,
+    k,
+    f_output=None,
+    fileformat="gdx",
+    var_dense=None,
+    equ_dense=None,
+    presolve=False,
+    mpsreader=None,
+    method="hypergraph",
+    hypergraph="col",
+    hg_objective="soed",
+    regex_pattern=None,
+    regex_order=None,
+    regex_mapfile=None,
+    dec=None,
+    write_names=True,
+    write_uels=True,
+    distributed=False,
+    vcycles=0,
+):
+    start = perf_counter()
+    if isinstance(f_input, str):
+        f_input = Path(f_input)
+
+    if var_dense is not None and var_dense < 0:
+        var_dense = None
+
+    if equ_dense is not None and equ_dense < 0:
+        var_dense = None
+
+    if mpsreader is None:
+        lib_gurobi = importlib.util.find_spec("gurobipy")
+        if lib_gurobi is not None:
+            mpsreader = "gurobi"
+        else:
+            mpsreader = "highs"
+        print(f"Selected {mpsreader} as default mps reader.")
+
+    if k == 1 and method.lower() != "oneblock":
+        print('Specificed partitioning into a single block, switching method to "oneblock"')
+        method = "oneblock"
+
+    if regex_pattern is not None:
+        print('Specificed regex pattern, switching method to "regex"')
+        method = "regex"
+
+    if dec is not None:
+        print('Specificed .dec file, switching method to "dec"')
+        method = "dec"
+
+    # Check if valid hypergraph arguments are provided
+    if method == "hypergraph":
+        if hypergraph.lower() not in ["col", "row", "colrow", "rowcol"]:
+            raise IOError(
+                f"Hypergraph type {hypergraph} is not available. "
+                "Use either 'col', 'row', 'colrow', or 'rowcol'"
+            )
+        if hg_objective.lower() not in ["cut", "km1", "soed"]:
+            raise IOError(
+                f"Hypergraph objective {hg_objective} is not available. "
+                "Use either 'cut', 'km1', or 'soed'"
+            )
+
+    # Read in file and extract A, vars, cols
+    match f_input.suffix.lower():
+        case ".gdx":
+            A, cols, rows, objcol, objrow, objcoef, objjacval = read_gdx(f_input=f_input)
+        case ".parquet":
+            A, cols, rows, objcol, objrow, objcoef, objjacval = read_parquet(f_input=f_input)
+        case ".lp" | ".mps" | ".gz" | ".bz2":
+            compressed_input = f_input.suffix.lower() in [".gz", ".bz2"]
+
+            # File needs to be compresses lp or mps
+            if compressed_input:
+                suffix_uncomp = f_input.stem.lower()
+                if not suffix_uncomp.endswith(".lp") and not suffix_uncomp.endswith(".mps"):
+                    raise (
+                        IOError(
+                            "Can only read compressed files of type .lp and .mps, "
+                            f"not {f_input.suffix}"
+                        )
+                    )
+
+            # For highspy manually decompress file and delete later
+            remove_decompressed = False
+            if compressed_input and mpsreader == "highs":
+                print(f"Automatically decompressing problem file {f_input.as_posix()} for highs")
+                if f_input.with_suffix("").exists():
+                    raise (
+                        IOError(
+                            f"Cannot automatically decompress file {f_input.as_posix()}, since "
+                            "a file with the target name already exists!"
+                        )
+                    )
+                with gzip.open(f_input, "rb") as f_in:
+                    with open(f_input.with_suffix(""), "wb") as f_out:
+                        shutil.copyfileobj(f_in, f_out)
+                f_input = f_input.with_suffix("")
+                remove_decompressed = True
+
+            # Read in mps content
+            A, cols, rows, objcol, objrow, objcoef = read_mps(
+                f_input=f_input,
+                presolve=presolve,
+                read_names=write_names or method == "regex",
+                mpsreader=mpsreader,
+            )
+            objjacval = 1.0
+
+            # Remove temporary decompressed file again
+            if remove_decompressed:
+                print(f"Automatically removing decompressed file {f_input.as_posix()}")
+                f_input.unlink()
+        case _:
+            raise (IOError(f"No reader implemented for filetype {f_input.suffix}"))
+
+    # Check if kahypar is available
+    if method == "hypergraph":
+        lib_mkkahypar = importlib.util.find_spec("mtkahypar")
+        if lib_mkkahypar is None:
+            raise ImportError(
+                "Hypergraph partitioning specified, but library mtkahypar not available."
+            )
+
+    # Partition by regex, hypergraph or dense elements
+    cols, rows = get_partitions(
+        A=A,
+        cols=cols,
+        rows=rows,
+        method=method,
+        hypergraph=hypergraph,
+        hg_objective=hg_objective,
+        k=k,
+        var_dense=var_dense,
+        equ_dense=equ_dense,
+        regex_pattern=regex_pattern,
+        regex_order=regex_order,
+        regex_mapfile=regex_mapfile,
+        dec=dec,
+        vcycles=vcycles,
+        objcol=objcol,
+    )
+
+    # Derive stem for output
+    if f_output is None:
+        stem, suffix = get_stem(f_input)
+    else:
+        stem, suffix = get_stem(f_output)
+        if suffix is not None:
+            fileformat = suffix.lstrip(".")
+
+    f_output = Path(f"{stem}_{len(set(cols['partition'].unique()) - {1})}b")
+    f_output.parent.mkdir(parents=True, exist_ok=True)
+
+    # Get scoring for annotation
+    get_score(A, cols, rows)
+
+    # Write out annotated gdx file
+    match fileformat.lower():
+        case "gdx":
+            write_gdx(
+                f_input=f_input,
+                f_output=f_output,
+                A=A,
+                cols=cols,
+                rows=rows,
+                objcol=objcol,
+                objrow=objrow,
+                objcoef=objcoef,
+                objjacval=objjacval,
+                write_names=write_names,
+                write_uels=write_uels,
+                distributed=distributed,
+            )
+        case "parquet":
+            write_parquet(
+                f_input=f_input,
+                f_output=f_output,
+                A=A,
+                cols=cols,
+                rows=rows,
+                objcol=objcol,
+                objrow=objrow,
+                objcoef=objcoef,
+                objjacval=objjacval,
+            )
+        case _:
+            raise (IOError(f"No method implemented for output format {fileformat}"))
+
+    stop = perf_counter()
+    print(f"Annotated problem in {(stop - start):.2f} seconds")

pipstools/cli.py

@@ -0,0 +1,258 @@
+from pathlib import Path
+
+import typer
+from typing_extensions import Annotated
+
+
+def version_callback(value: bool):
+    if value:
+        import importlib.metadata
+
+        version = importlib.metadata.version("pipstools")
+        typer.echo(f"{version}")
+        raise typer.Exit()
+
+
+app = typer.Typer(
+    add_completion=False,
+)
+
+
+@app.callback()
+def common(
+    ctx: typer.Context,
+    version: bool = typer.Option(None, "--version", callback=version_callback),
+):
+    pass
+
+
+@app.command(
+    help="Annotate a lp/mps/gdx file using hypergraph partitioning or regular expressions."
+)
+def annotate(
+    # Options for file IO
+    inputfile: Annotated[
+        str,
+        typer.Argument(help="Path to the input file (.lp, .mps, .gdx, or .parquet)"),
+    ],
+    outputfile: Annotated[
+        str | None,
+        typer.Option(help="Path to the output file"),
+    ] = None,
+    fileformat: Annotated[
+        str,
+        typer.Option(help="File format the output file"),
+    ] = "gdx",
+    mpsreader: Annotated[
+        str | None,
+        typer.Option(help='Package to use as mps reader ("gurobi", "highs")'),
+    ] = None,
+    # Options for matrix structure
+    blocks: Annotated[
+        int,
+        typer.Option(help="Number of blocks to annotate"),
+    ] = 30,
+    densecol: Annotated[
+        int | None,
+        typer.Option(
+            help="Number of non-zero entries to consider a column as dense, "
+            "defaults to heuristic detection of dense columns"
+        ),
+    ] = None,
+    denserow: Annotated[
+        int | None,
+        typer.Option(
+            help="Number of non-zero entries to consider a row as dense, "
+            "defaults to no detection of dense rows"
+        ),
+    ] = None,
+    presolve: Annotated[
+        bool,
+        typer.Option(help="Presolve the problem before partitioning"),
+    ] = False,
+    # Options for methods
+    method: Annotated[
+        str,
+        typer.Option(help='Method to use for partitioning ("hypergraph", "oneblock", "regex")'),
+    ] = "hypergraph",
+    hypergraph: Annotated[
+        str,
+        typer.Option(help='Type of hypergraph to use for partitioning ("col", "row", "colrow")'),
+    ] = "col",
+    hg_objective: Annotated[
+        str,
+        typer.Option(
+            help='Objective metric to use for hypergraph partitioning ("cut", "km1", "soed")'
+        ),
+    ] = "soed",
+    regex_pattern: Annotated[
+        str | None,
+        typer.Option(help="Pattern to use for variable partitioning via regular expressions"),
+    ] = None,
+    regex_order: Annotated[
+        str | None,
+        typer.Option(help="Ordering of the capture groups for partitioning (e.g. [2,1])"),
+    ] = None,
+    regex_mapfile: Annotated[
+        str | None,
+        typer.Option(help="Mapping file from regex capture groups to blocks in csv format."),
+    ] = None,
+    dec: Annotated[
+        str | None,
+        typer.Option(help="Additional .dec file containing a predefined decomposition"),
+    ] = None,
+    # Options for performance
+    distributed: Annotated[
+        bool,
+        typer.Option(help="Write out distributed gdx files"),
+    ] = False,
+    names: Annotated[
+        bool,
+        typer.Option(help="Write variable and constraint names to output files"),
+    ] = True,
+    uels: Annotated[
+        bool,
+        typer.Option(help="Write unique element list to output files"),
+    ] = True,
+) -> None:
+    from pipstools import annotate_problem
+
+    f_input = Path(inputfile)
+    if not f_input.exists():
+        raise IOError(f"File {f_input.as_posix()} not found.")
+
+    annotate_problem(
+        f_input,
+        f_output=outputfile,
+        fileformat=fileformat,
+        k=int(blocks),
+        var_dense=densecol,
+        equ_dense=denserow,
+        presolve=presolve,
+        mpsreader=mpsreader,
+        method=method,
+        hypergraph=hypergraph,
+        hg_objective=hg_objective,
+        regex_pattern=regex_pattern,
+        regex_order=regex_order,
+        regex_mapfile=regex_mapfile,
+        dec=dec,
+        write_names=names,
+        write_uels=uels,
+        distributed=distributed,
+    )
+
+
+@app.command(help="Convert the problem structure between different file formats.")
+def convert(
+    # Options for file IO
+    inputfile: Annotated[
+        str,
+        typer.Argument(help="Path to the problem file (.gdx or .parquet)"),
+    ],
+    outputfile: Annotated[
+        str | None,
+        typer.Option(help="Path to the output file"),
+    ] = None,
+    fileformat: Annotated[
+        str,
+        typer.Option(help="File format the output file"),
+    ] = "gdx",
+    # Options for performance
+    distributed: Annotated[
+        bool,
+        typer.Option(help="Write out distributed gdx files"),
+    ] = False,
+    names: Annotated[
+        bool,
+        typer.Option(help="Write variable and constraint names to output files"),
+    ] = True,
+    uels: Annotated[
+        bool,
+        typer.Option(help="Write unique element list to output files"),
+    ] = True,
+) -> None:
+    from pipstools import convert_problem
+
+    f_input = Path(inputfile)
+    if not f_input.exists():
+        raise IOError(f"File {f_input.as_posix()} not found.")
+
+    convert_problem(
+        f_input,
+        f_output=outputfile,
+        fileformat=fileformat,
+        write_names=names,
+        write_uels=uels,
+        distributed=distributed,
+    )
+
+
+@app.command(help="Visualize the blocks structure of an annotated .gdx file.")
+def visualize(
+    inputfile: Annotated[
+        str,
+        typer.Argument(help="Path to the problem file (.gdx or .parquet)"),
+    ],
+    outputfile: Annotated[
+        str | None,
+        typer.Option(help="Redirect the figure to a file."),
+    ] = None,
+    annotation: Annotated[
+        bool,
+        typer.Option(help="Visualize the annotation"),
+    ] = True,
+    names: Annotated[
+        bool,
+        typer.Option(help="Try to use variable and constraint names"),
+    ] = True,
+    force: Annotated[
+        bool,
+        typer.Option(help="Try to use variable and constraint names"),
+    ] = False,
+) -> None:
+    from pipstools import visualize_problem
+
+    f_input = Path(inputfile)
+    if not f_input.exists():
+        raise IOError(f"File {f_input.as_posix()} not found.")
+
+    visualize_problem(
+        f_input,
+        output=outputfile,
+        read_names=names,
+        use_annotation=annotation,
+        force=force,
+    )
+
+
+@app.command(help="Score the blocks structure of an annotated problem.")
+def score(
+    inputfile: Annotated[
+        str,
+        typer.Argument(help="Path to the problem file (.gdx or .parquet)"),
+    ],
+) -> None:
+    from pipstools import score_problem
+
+    f_input = Path(inputfile)
+    if not f_input.exists():
+        raise IOError(f"File {f_input.as_posix()} not found.")
+
+    score_problem(f_input)
+
+
+@app.command(help="Validate the blocks structure of an annotated problem.")
+def validate(
+    inputfile: Annotated[
+        str,
+        typer.Argument(help="Path to the problem file (.gdx or .parquet)"),
+    ],
+) -> None:
+    from pipstools import validate_problem
+
+    f_input = Path(inputfile)
+    if not f_input.exists():
+        raise IOError(f"File {f_input.as_posix()} not found.")
+
+    validate_problem(f_input)

pipstools/conversion.py

@@ -0,0 +1,75 @@
+from pathlib import Path
+from time import perf_counter
+
+from pipstools.io import read_gdx, read_parquet, write_gdx, write_parquet
+from pipstools.utils import get_stem
+
+
+def convert_problem(
+    f_input,
+    f_output=None,
+    fileformat="gdx",
+    write_names=True,
+    write_uels=True,
+    distributed=False,
+):
+    if isinstance(f_input, str):
+        f_input = Path(f_input)
+
+    match f_input.suffix.lower():
+        case ".gdx":
+            A, cols, rows, objcol, objrow, objcoef, objjacval = read_gdx(f_input=f_input)
+        case ".parquet":
+            A, cols, rows, objcol, objrow, objcoef, objjacval = read_parquet(f_input=f_input)
+        case _:
+            raise IOError("Converting is only supported for .gdx files and .parquet collections.")
+
+    print(f"Converting a model with {len(rows)} rows, {len(cols)} columns, {len(A)} nonzeros")
+    start = perf_counter()
+
+    # Derive stem for output
+    if f_output is None:
+        stem, suffix = get_stem(f_input)
+    else:
+        stem, suffix = get_stem(f_output)
+        if suffix is not None:
+            print(f"Automatically derived fileformat {suffix} from output file")
+            fileformat = suffix.lstrip(".")
+
+    f_output = Path(f"{stem}_{len(set(cols['partition'].unique()) - {1})}b")
+    f_output.parent.mkdir(parents=True, exist_ok=True)
+
+    # Write out annotated gdx file
+    match fileformat.lower():
+        case "gdx":
+            write_gdx(
+                f_input=f_input,
+                f_output=f_output,
+                A=A,
+                cols=cols,
+                rows=rows,
+                objcol=objcol,
+                objrow=objrow,
+                objcoef=objcoef,
+                objjacval=objjacval,
+                write_names=write_names,
+                write_uels=write_uels,
+                distributed=distributed,
+            )
+        case "parquet":
+            write_parquet(
+                f_input=f_input,
+                f_output=f_output,
+                A=A,
+                cols=cols,
+                rows=rows,
+                objcol=objcol,
+                objrow=objrow,
+                objcoef=objcoef,
+                objjacval=objjacval,
+            )
+        case _:
+            raise (IOError(f"No method implemented for output format {fileformat}"))
+
+    stop = perf_counter()
+    print(f"Converted problem in {(stop - start):.2f} seconds")

pipstools/io/__init__.py

@@ -0,0 +1,5 @@
+from .gdx import read_gdx as read_gdx
+from .gdx import write_gdx as write_gdx
+from .mps import read_mps as read_mps
+from .parquet import read_parquet as read_parquet
+from .parquet import write_parquet as write_parquet