mdpsolver 0.10.0__cp313-cp313-win_amd64.whl

mdpsolver/__init__.py

@@ -0,0 +1,12 @@
+"""""" # start delvewheel patch
+def _delvewheel_patch_1_13_0():
+    import os
+    if os.path.isdir(libs_dir := os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'mdpsolver.libs'))):
+        os.add_dll_directory(libs_dir)
+
+
+_delvewheel_patch_1_13_0()
+del _delvewheel_patch_1_13_0
+# end delvewheel patch
+
+from mdpsolver.model import model

mdpsolver/model.py

@@ -0,0 +1,211 @@
+"""
+MDPSolver
+
+This file defines a `model` class that wraps around the `solvermodule` to create
+and solve Markov Decision Processes (MDPs). It provides methods to configure the model,
+solve it, and extract the resulting policy and value vectors.
+"""
+
+import sys
+from mdpsolver import solvermodule
+
+class model:
+    """
+    Creates an MDPSolver object.
+
+    This class provides methods to initialize, solve, and return results
+    from an MDP model.
+    """
+
+    def __init__(self):
+        """Initialize the model by creating a solver module object."""
+        self.initialize()
+
+    def initialize(self):
+        """Create and initialize the solver module object."""
+        self.mdl = solvermodule.Model()
+
+    def solve(
+        self,
+        algorithm: str = "mpi",
+        tolerance: float = 1e-3,
+        update: str = "standard",
+        criterion: str = "discounted",
+        parIterLim: int = 100,
+        SORrelaxation: float = 1.0,
+        initPolicy: list = list(),
+        initValueVector: list = list(),
+        verbose: bool = False,
+        postProcessing: bool = True,
+        makeFinalCheck: bool = True,
+        parallel: bool = True,
+    ) -> None:
+        """
+        Derive an epsilon-optimal policy for the selected MDP model.
+
+        Args:
+            algorithm (str): Algorithm to use.
+            tolerance (float): Convergence threshold for the algorithm.
+            update (str): The value-update method.
+            criterion (str): The optimality criterion.
+            parIterLim (int): The partial evaluation limit employed in the modified policy iteration algorithm.
+            SORrelaxation (float): Relaxation parameter for the Successive Over-Relaxation method.
+            initPolicy (list): A 1D-list defining the initial policy. This option can be used for "warm starting" the optimization procedure.
+            initValueVector (list): A 1D-list defining the initial value vector. This option can be used for "warm starting" the optimization procedure.
+            verbose (bool): If True, prints solver progress to console.
+            postProcessing (bool): If True, performs post-processing after solving.
+            makeFinalCheck (bool): If True, makes a final check of the value vector. This process checks if the resulting values are reasonable.
+            parallel (bool): If True, enables parallel computation for faster solving. This option is only available for the custom MDP model with standard updates.
+
+        Returns:
+            None
+        """
+        
+        if not (algorithm == "mpi" or algorithm == "pi" or algorithm == "vi"):
+            sys.exit("Error: Algorithm type not recognized. Select either: 'mpi', 'pi', or 'vi'.")
+        if tolerance <= 0.0:
+            sys.exit("Error: The tolerance needs to be a positive number.")
+        if not (update == "standard" or update == "gs" or update == "sor"):
+            sys.exit("Error: Update method not recognized. Select either: 'standard', 'gs', or 'sor'.")
+        if not (criterion == "discounted" or criterion == "average"):
+            sys.exit("Error: Optimality criterion not recognized. Select either: 'discounted' or 'average'.")
+        if parIterLim <= 0 and algorithm == "mpi":
+            sys.exit("Error: The partial evaluation limit needs to be a positive number.")
+        if (SORrelaxation <= 0.0 or SORrelaxation >= 2.0) and update == "sor":
+            sys.exit("Error: The relaxation parameter needs to be between 0.0 and 2.0 (i.e. 0 < SORrelaxation < 2).")
+        
+        self.mdl.solve(
+            algorithm=algorithm,
+            tolerance=tolerance,
+            update=update,
+            criterion=criterion,
+            parIterLim=parIterLim,
+            SORrelaxation=SORrelaxation,
+            initPolicy=initPolicy,
+            initValueVector=initValueVector,
+            verbose=verbose,
+            postProcessing=postProcessing,
+            makeFinalCheck=makeFinalCheck,
+            parallel=parallel,
+        )
+
+    def getRuntime(self) -> float:
+        """
+        Get the runtime of the last solver execution.
+
+        Returns:
+            float: Runtime in milliseconds.
+        """
+        return self.mdl.getRuntime()
+
+    def printPolicy(self) -> None:
+        """Print the entire policy to the terminal."""
+        self.mdl.printPolicy()
+
+    def printValueVector(self) -> None:
+        """Print the entire value vector to the terminal."""
+        self.mdl.printValueVector()
+
+    def getAction(self, stateIndex: int = 0) -> int:
+        """
+        Get the action from the optimized policy for a specific state.
+
+        Args:
+            stateIndex (int): Index of the state.
+
+        Returns:
+            int: Action for the given state.
+        """
+        return self.mdl.getAction(stateIndex=stateIndex)
+
+    def getValue(self, stateIndex: int = 0) -> float:
+        """
+        Get the value from the optimized value vector for a specific state.
+
+        Args:
+            stateIndex (int): Index of the state.
+
+        Returns:
+            float: Value for the given state.
+        """
+        return self.mdl.getValue(stateIndex=stateIndex)
+
+    def getPolicy(self) -> list:
+        """
+        Get the entire optimized policy.
+
+        Returns:
+            list: Optimized policy.
+        """
+        return self.mdl.getPolicy()
+
+    def getValueVector(self):
+        """
+        Get the entire optimized value vector.
+
+        Returns:
+            list: Optimized value vector.
+        """
+        return self.mdl.getValueVector()
+
+    def saveToFile(self, fileName: str = "result.csv", type: str = "policy") -> None:
+        """
+        Save the optimized policy or value vector to a file.
+
+        Args:
+            fileName (str): Name of the output file.
+            type (str): Type of result to save ('policy' or 'value').
+
+        Returns:
+            None
+        """
+        
+        if not (type == "policy" or type == "value"):
+            sys.exit("Error: Type not recognized. Select either: 'policy' or 'value'.")
+
+        return self.mdl.saveToFile(fileName=fileName, type=type)
+
+    def mdp(
+        self,
+        discount: float = 0.99,
+        rewards: list = list(),
+        rewardsElementwise: list = list(),
+        rewardsFromFile: str = "rewards.csv",
+        tranMatWithZeros: list = list(),
+        tranMatElementwise: list = list(),
+        tranMatProbs: list = list(),
+        tranMatColumns: list = list(),
+        tranMatFromFile: str = "transitions.csv",
+    ) -> None:
+        """
+        Define the generic MDP model.
+
+        Args:
+            discount (float): Discount factor.
+            rewards (list): A 2D-list containing the reward (float) of a particular action in a particular state.
+            rewardsElementwise (list): Alternative reward format. A 2D-list where each row corresponds to a combination of a state and an action.
+            rewardsFromFile (str): Load the rewards from a comma-separated (,) file.
+            tranMatWithZeros (list): A 3D-list containing the transition probabilities.
+            tranMatElementwise (list): Sparse transition probabilities (option 1). A 2D-list where each row corresponds to a combination of a current state, an action, and a next state.
+            tranMatProbs (list): Sparse transition probabilities (option 2, part 1). A 3D-list containing the non-zero transition probabilities.
+            tranMatColumns (list): Sparse transition probabilities (option 2, part 2). A 3D-list containing the columns of the non-zero transition probabilities.
+            tranMatFromFile (str): Load the transition probabilities from a comma-separated (,) file.
+
+        Returns:
+            None
+        """
+        
+        if discount <= 0.0 or discount >= 1.0:
+            sys.exit("Error: The discount needs to be in the interval between 0.0 and 1.0 (i.e. 0 < discount < 1).")
+        
+        self.mdl.mdp(
+            discount=discount,
+            rewards=rewards,
+            rewardsElementwise=rewardsElementwise,
+            rewardsFromFile=rewardsFromFile,
+            tranMatWithZeros=tranMatWithZeros,
+            tranMatElementwise=tranMatElementwise,
+            tranMatProbs=tranMatProbs,
+            tranMatColumns=tranMatColumns,
+            tranMatFromFile=tranMatFromFile,
+        )

mdpsolver-0.10.0.dist-info/DELVEWHEEL

@@ -0,0 +1,2 @@
+Version: 1.13.0
+Arguments: ['C:\\Users\\runneradmin\\AppData\\Local\\Temp\\cibw-run-_v3rp46l\\cp313-win_amd64\\build\\venv\\Scripts\\delvewheel', 'repair', '-w', 'C:\\Users\\runneradmin\\AppData\\Local\\Temp\\cibw-run-_v3rp46l\\cp313-win_amd64\\repaired_wheel', '-v', 'C:\\Users\\runneradmin\\AppData\\Local\\Temp\\cibw-run-_v3rp46l\\cp313-win_amd64\\built_wheel\\mdpsolver-0.10.0-cp313-cp313-win_amd64.whl']

mdpsolver-0.10.0.dist-info/METADATA

@@ -0,0 +1,151 @@
+Metadata-Version: 2.2
+Name: mdpsolver
+Version: 0.10.0
+Summary: A fast solver for Markov Decision Processes
+Author-Email: Anders Reenberg Andersen <andersra@live.dk>, Jesper Fink Andersen <jesperfinkandersen@gmail.com>
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: MacOS
+Project-URL: Homepage, https://github.com/areenberg/MDPSolver
+Project-URL: Issues, https://github.com/areenberg/MDPSolver/issues
+Requires-Python: >=3.10
+Provides-Extra: test
+Requires-Dist: numpy; extra == "test"
+Description-Content-Type: text/markdown
+
+# MDPSolver
+
+MDPSolver is a Python package for large Markov Decision Processes (MDPs) with infinite-horizons. 
+
+## Features
+
+* Fast solver: Our C++-based solver is substantially faster than other CPU-based MDP packages available for Python. See details in the documentation.
+* Two optimality criteria: *Discounted* and *Average* reward.
+* Three optimization algorithms: *Value iteration*, *Policy iteration*, and *Modified policy iteration*.
+* Three value-update methods: *Standard*, *Gauss–Seidel*, and *Successive over-relaxation*.
+* Supports sparse matrices.
+* Employs parallel computing.
+
+# Installation
+
+Prebuilt wheels are available for **Linux**, **Windows**, and **macOS (14+)** across Python 3.10–3.14. Install directly from PyPI with:
+
+```
+pip install mdpsolver
+```
+
+# Quick start guide
+
+The following shows how to get quickly started with `mdpsolver`.
+
+## Usage
+
+Start by specifying the reward function and transition probabilities as lists. The following is an example of a simple MDP containing **three states** and **two actions in each state**.
+
+```python
+#Import packages
+import mdpsolver
+
+#Rewards (3 states x 2 actions)
+#e.g. choosing second action in first state gives reward=-1
+rewards = [[5,-1],
+           [1,-2],
+           [50,0]]
+
+#Transition probabilities (3 from_states x 2 actions x 3 to_states)
+#e.g. choosing first action in third state gives a probability of 0.6 of staying in third state
+tranMatWithZeros = [[[0.9,0.1,0.0],[0.1,0.9,0.0]],
+                    [[0.4,0.5,0.1],[0.3,0.5,0.2]],
+                    [[0.2,0.2,0.6],[0.5,0.5,0.0]]]
+```
+
+Now, create the model object and insert the problem parameters.
+
+```python
+#Create model object
+mdl = mdpsolver.model()
+
+#Insert the problem parameters
+mdl.mdp(discount=0.8,
+        rewards=rewards,
+        tranMatWithZeros=tranMatWithZeros)
+```
+
+We can now optimize the policy.
+
+```python
+mdl.solve()
+```
+
+The optimized policy can be returned in a variety of ways. Here, we return the policy as a list and print directly in the terminal. 
+
+```python
+print(mdl.getPolicy())
+#[1, 1, 0]
+```
+
+## Sparse transition matrix?
+
+`mdpsolver` has three alternative formats for large and highly sparse transition probability matrices.
+
+**(1) Elementwise representation (excluding elements containing zeros):**
+
+```python
+#[from_state,action,to_state,probability]
+tranMatElementwise = [[0,0,0,0.9],
+                      [0,0,1,0.1],
+                      [0,1,0,0.1],
+                      [0,1,1,0.9],
+                      [1,0,0,0.4],
+                      [1,0,1,0.5],
+                      [1,0,2,0.1],
+                      [1,1,0,0.3],
+                      [1,1,1,0.5],
+                      [1,1,2,0.2],
+                      [2,0,0,0.2],
+                      [2,0,1,0.2],
+                      [2,0,2,0.6],
+                      [2,1,0,0.5],
+                      [2,1,1,0.5]]
+
+mdl.mdp(discount=0.8,
+        rewards=rewards,
+        tranMatElementwise=tranMatElementwise)
+```
+
+**(2) Probabilities and column (to_state) indices in separate lists:**
+
+```python
+tranMatProbs = [[[0.9,0.1],[0.1,0.9]],
+                [[0.4,0.5,0.1],[0.3,0.5,0.2]],
+                [[0.2,0.2,0.6],[0.5,0.5]]]
+
+tranMatColumns = [[[0,1],[0,1]],
+                [[0,1,2],[0,1,2]],
+                [[0,1,2],[0,1]]]
+
+mdl.mdp(discount=0.8,
+        rewards=rewards,
+        tranMatProbs=tranMatProbs,
+        tranMatColumns=tranMatColumns)
+
+```
+
+**(3) Load the elementwise representation from a file:**
+
+```python
+mdl.mdp(discount=0.8,
+        rewards=rewards,
+        tranMatFromFile="transitions.csv")
+```
+
+# Documentation
+
+The documentation can be found in the [wiki for MDPSolver](https://github.com/areenberg/MDPSolver/wiki).
+
+# How to cite
+
+[![DOI](https://zenodo.org/badge/294063917.svg)](https://zenodo.org/badge/latestdoi/294063917)

mdpsolver-0.10.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+mdpsolver/model.py,sha256=P2_KIKxmDlJIkJz-bKI8xWOyYvHOqGJkHy9GbszbMB8,8471
+mdpsolver/solvermodule.cp313-win_amd64.pyd,sha256=SleS_NHwqYEEZmTcONz9zBmecxgYrWsVQh2dseCXZFs,337408
+mdpsolver/__init__.py,sha256=xLEYt_z5jB31k1nXSlcpLamFj_IpuzEoHw7SqgtKbg8,366
+mdpsolver-0.10.0.dist-info/DELVEWHEEL,sha256=AqXqH2wGYfxAkA2Un_FZ0U_Dml-aSByrhJWKZWCRtZM,409
+mdpsolver-0.10.0.dist-info/METADATA,sha256=RZNDDc0XOIuYvyT5BFTIiHLVfAJ8nsqq7830zNLvRkQ,4546
+mdpsolver-0.10.0.dist-info/RECORD,,
+mdpsolver-0.10.0.dist-info/WHEEL,sha256=dXz53UX_wVrENU3pg7IjBHl1TfATF0jKSXtQDtrFnLw,105
+mdpsolver-0.10.0.dist-info/licenses/LICENSE,sha256=KOcIKv2SHW1fmMFokSbuqssz4Xg8mUnQ0kB8nOKIww4,1127
+mdpsolver.libs/libomp140.x86_64-4a20c1e5c115c29771a12324513eb109.dll,sha256=SiDB5cEVwpdxoSMkUT6xCbraxyGA95SBUnrXnZlv-zM,661856
+mdpsolver.libs/msvcp140-a4c2229bdc2a2a630acdc095b4d86008.dll,sha256=pMIim9wqKmMKzcCVtNhgCOXD47x3cxdDVPPaT1vrnN4,575056

mdpsolver-0.10.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: scikit-build-core 1.0.3
+Root-Is-Purelib: false
+Tag: cp313-cp313-win_amd64
+

mdpsolver-0.10.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Anders Reenberg Andersen and Jesper Fink Andersen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.