mdpsolver 0.9.6__tar.gz → 0.9.8__tar.gz

{mdpsolver-0.9.6 → mdpsolver-0.9.8}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Anders Reenberg Andersen and Jesper Fink Andersen
+Copyright (c) 2025 Anders Reenberg Andersen and Jesper Fink Andersen
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

{mdpsolver-0.9.6/src/mdpsolver.egg-info → mdpsolver-0.9.8}/PKG-INFO

@@ -1,8 +1,8 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: mdpsolver
-Version: 0.9.6
+Version: 0.9.8
 Summary: A fast solver for Markov Decision Processes
-Author-email: Anders Reenberg Andersen <andersra@live.dk>, Jesper Fink Andersen <jfan@dtu.dk>
+Author-email: Anders Reenberg Andersen <andersra@live.dk>, Jesper Fink Andersen <jesperfinkandersen@gmail.com>
 Project-URL: Homepage, https://github.com/areenberg/MDPSolver
 Project-URL: Issues, https://github.com/areenberg/MDPSolver/issues
 Classifier: Development Status :: 3 - Alpha
@@ -17,13 +17,14 @@ License-File: LICENSE
 
 <img src="https://github.com/areenberg/MDPSolver/blob/anders_development/Images/MDPSolver_logo_3.png" alt="Logo" width="180">
 
-MDPSolver is a Python package for Markov Decision Processes (MDPs) with discounted rewards and infinite-horizon. 
+MDPSolver is a Python package for large Markov Decision Processes (MDPs) with infinite-horizons. 
 
 ## Features
 
 * Fast solver: Our C++-based solver is substantially faster than other MDP packages available for Python. See details in the documentation.
+* Two optimality criteria: *Discounted* and *Average* reward.
 * Three optimization algorithms: *Value iteration*, *Policy iteration*, and *Modified policy iteration*.
-* Three value-update methods: *Standard*, *Gauss–Seidel*, *Successive over-relaxation*.
+* Three value-update methods: *Standard*, *Gauss–Seidel*, and *Successive over-relaxation*.
 * Supports sparse matrices.
 * Employs parallel computing.
 
@@ -37,7 +38,11 @@ Install directly from PyPI with:
 pip install mdpsolver
 ```
 
-After the installation, MDPSolver works *out of the box*.
+MDPSolver works *out of the box* on Ubuntu 22 and newer.
+
+### GLIBC not found
+
+Some users will encounter the `version 'GLIBC_2.32' not found` error when attempting to import MDPSolver in Python. In this case, it might help to manually compile and replace the SO-file for the optimization module in the MDPSolver package. See the steps on how to solve the issue in the [documentation](https://github.com/areenberg/MDPSolver/wiki#how-to-install).
 
 ## Windows
 
@@ -156,7 +161,7 @@ mdl.mdp(discount=0.8,
 
 # Documentation
 
-The documentation can be found in the wiki for MDPSolver (https://github.com/areenberg/MDPSolver/wiki).
+The documentation can be found in the [wiki for MDPSolver](https://github.com/areenberg/MDPSolver/wiki).
 
 # How to cite
 

{mdpsolver-0.9.6 → mdpsolver-0.9.8}/README.md

@@ -2,13 +2,14 @@
 
 <img src="https://github.com/areenberg/MDPSolver/blob/anders_development/Images/MDPSolver_logo_3.png" alt="Logo" width="180">
 
-MDPSolver is a Python package for Markov Decision Processes (MDPs) with discounted rewards and infinite-horizon. 
+MDPSolver is a Python package for large Markov Decision Processes (MDPs) with infinite-horizons. 
 
 ## Features
 
 * Fast solver: Our C++-based solver is substantially faster than other MDP packages available for Python. See details in the documentation.
+* Two optimality criteria: *Discounted* and *Average* reward.
 * Three optimization algorithms: *Value iteration*, *Policy iteration*, and *Modified policy iteration*.
-* Three value-update methods: *Standard*, *Gauss–Seidel*, *Successive over-relaxation*.
+* Three value-update methods: *Standard*, *Gauss–Seidel*, and *Successive over-relaxation*.
 * Supports sparse matrices.
 * Employs parallel computing.
 
@@ -22,7 +23,11 @@ Install directly from PyPI with:
 pip install mdpsolver
 ```
 
-After the installation, MDPSolver works *out of the box*.
+MDPSolver works *out of the box* on Ubuntu 22 and newer.
+
+### GLIBC not found
+
+Some users will encounter the `version 'GLIBC_2.32' not found` error when attempting to import MDPSolver in Python. In this case, it might help to manually compile and replace the SO-file for the optimization module in the MDPSolver package. See the steps on how to solve the issue in the [documentation](https://github.com/areenberg/MDPSolver/wiki#how-to-install).
 
 ## Windows
 
@@ -141,8 +146,8 @@ mdl.mdp(discount=0.8,
 
 # Documentation
 
-The documentation can be found in the wiki for MDPSolver (https://github.com/areenberg/MDPSolver/wiki).
+The documentation can be found in the [wiki for MDPSolver](https://github.com/areenberg/MDPSolver/wiki).
 
 # How to cite
 
-[![DOI](https://zenodo.org/badge/294063917.svg)](https://zenodo.org/badge/latestdoi/294063917)
+[![DOI](https://zenodo.org/badge/294063917.svg)](https://zenodo.org/badge/latestdoi/294063917)

{mdpsolver-0.9.6 → mdpsolver-0.9.8}/pyproject.toml

@@ -10,10 +10,10 @@ where = ["src"]
 
 [project]
 name = "mdpsolver"
-version = "0.9.6"
+version = "0.9.8"
 authors = [
   { name="Anders Reenberg Andersen", email="andersra@live.dk" },
-  { name="Jesper Fink Andersen", email="jfan@dtu.dk" },
+  { name="Jesper Fink Andersen", email="jesperfinkandersen@gmail.com" },
 ]
 description = "A fast solver for Markov Decision Processes"
 readme = "README.md"

{mdpsolver-0.9.6 → mdpsolver-0.9.8}/src/mdpsolver/model.py

@@ -17,6 +17,7 @@ class model:
               algorithm="mpi",
               tolerance=1e-3,
               update="standard",
+              criterion="discounted",
               parIterLim=100,
               SORrelaxation=1.0,
               initPolicy=list(),
@@ -29,6 +30,7 @@ class model:
         self.mdl.solve(algorithm=algorithm,
               tolerance=tolerance,
               update=update,
+              criterion=criterion,
               parIterLim=parIterLim,
               SORrelaxation=SORrelaxation,
               initPolicy=initPolicy,

{mdpsolver-0.9.6 → mdpsolver-0.9.8/src/mdpsolver.egg-info}/PKG-INFO

@@ -1,8 +1,8 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: mdpsolver
-Version: 0.9.6
+Version: 0.9.8
 Summary: A fast solver for Markov Decision Processes
-Author-email: Anders Reenberg Andersen <andersra@live.dk>, Jesper Fink Andersen <jfan@dtu.dk>
+Author-email: Anders Reenberg Andersen <andersra@live.dk>, Jesper Fink Andersen <jesperfinkandersen@gmail.com>
 Project-URL: Homepage, https://github.com/areenberg/MDPSolver
 Project-URL: Issues, https://github.com/areenberg/MDPSolver/issues
 Classifier: Development Status :: 3 - Alpha
@@ -17,13 +17,14 @@ License-File: LICENSE
 
 <img src="https://github.com/areenberg/MDPSolver/blob/anders_development/Images/MDPSolver_logo_3.png" alt="Logo" width="180">
 
-MDPSolver is a Python package for Markov Decision Processes (MDPs) with discounted rewards and infinite-horizon. 
+MDPSolver is a Python package for large Markov Decision Processes (MDPs) with infinite-horizons. 
 
 ## Features
 
 * Fast solver: Our C++-based solver is substantially faster than other MDP packages available for Python. See details in the documentation.
+* Two optimality criteria: *Discounted* and *Average* reward.
 * Three optimization algorithms: *Value iteration*, *Policy iteration*, and *Modified policy iteration*.
-* Three value-update methods: *Standard*, *Gauss–Seidel*, *Successive over-relaxation*.
+* Three value-update methods: *Standard*, *Gauss–Seidel*, and *Successive over-relaxation*.
 * Supports sparse matrices.
 * Employs parallel computing.
 
@@ -37,7 +38,11 @@ Install directly from PyPI with:
 pip install mdpsolver
 ```
 
-After the installation, MDPSolver works *out of the box*.
+MDPSolver works *out of the box* on Ubuntu 22 and newer.
+
+### GLIBC not found
+
+Some users will encounter the `version 'GLIBC_2.32' not found` error when attempting to import MDPSolver in Python. In this case, it might help to manually compile and replace the SO-file for the optimization module in the MDPSolver package. See the steps on how to solve the issue in the [documentation](https://github.com/areenberg/MDPSolver/wiki#how-to-install).
 
 ## Windows
 
@@ -156,7 +161,7 @@ mdl.mdp(discount=0.8,
 
 # Documentation
 
-The documentation can be found in the wiki for MDPSolver (https://github.com/areenberg/MDPSolver/wiki).
+The documentation can be found in the [wiki for MDPSolver](https://github.com/areenberg/MDPSolver/wiki).
 
 # How to cite
 

{mdpsolver-0.9.6 → mdpsolver-0.9.8}/src/mdpsolver.egg-info/SOURCES.txt

@@ -7,9 +7,11 @@ src/mdpsolver/model.py
 src/mdpsolver/solvermodule.cp310-win_amd64.pyd
 src/mdpsolver/solvermodule.cp311-win_amd64.pyd
 src/mdpsolver/solvermodule.cp312-win_amd64.pyd
+src/mdpsolver/solvermodule.cp313-win_amd64.pyd
 src/mdpsolver/solvermodule.cpython-310-x86_64-linux-gnu.so
 src/mdpsolver/solvermodule.cpython-311-x86_64-linux-gnu.so
 src/mdpsolver/solvermodule.cpython-312-x86_64-linux-gnu.so
+src/mdpsolver/solvermodule.cpython-313-x86_64-linux-gnu.so
 src/mdpsolver.egg-info/PKG-INFO
 src/mdpsolver.egg-info/SOURCES.txt
 src/mdpsolver.egg-info/dependency_links.txt
@@ -17,4 +19,6 @@ src/mdpsolver.egg-info/top_level.txt
 src/mdpsolver/problems/cbmaux.py
 src/mdpsolver/problems/tbmaux.py
 src/mdpsolver/problems/__pycache__/cbmaux.cpython-310.pyc
-src/mdpsolver/problems/__pycache__/tbmaux.cpython-310.pyc
+src/mdpsolver/problems/__pycache__/tbmaux.cpython-310.pyc
+tests/test1.py
+tests/test2.py

mdpsolver-0.9.8/tests/test1.py

@@ -0,0 +1,686 @@
+import mdpsolver
+import random
+import sys
+import numpy as np
+from random import randint
+
+#TEST 1
+#Simple MDP with 3 states and 2 actions in each state.
+
+#---------------------------------------
+# CONFIGURATION 1
+#---------------------------------------
+
+#rewards
+#1st index: from (current) states
+#2nd index: actions
+rewards = [[5,-1],
+           [1,-2],
+           [50,0]]
+
+#transition probabilities
+#1st index: from (current) states
+#2nd index: actions
+#3rd index: to (next) states 
+tranMatWithZeros = [[[0.9,0.1,0.0],[0.1,0.9,0.0]],
+                    [[0.4,0.5,0.1],[0.3,0.5,0.2]],
+                    [[0.2,0.2,0.6],[0.5,0.5,0.0]]]
+
+#initial policy
+random.seed(10)
+initPolicy = [randint(0, 1) for p in range(0, 3)]
+
+#Model 1a (discounted reward, parallel)
+mdl1a = mdpsolver.model()
+mdl1a.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatWithZeros=tranMatWithZeros)
+mdl1a.solve(algorithm="mpi",
+          update="standard",
+          parallel=True,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl1a.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 1a failed!")        
+if not np.array_equal(np.round(np.array(mdl1a.getValueVector()),6),np.round(np.array([200.00114718191236,212.86672887958622,298.7090458676583]),6)):
+        sys.exit("Model 1a failed!")
+
+#Model 2a (discounted reward, parallel)
+mdl2a = mdpsolver.model()
+mdl2a.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatWithZeros=tranMatWithZeros)
+mdl2a.solve(algorithm="pi",
+          update="standard",
+          parallel=True,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl2a.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 2a failed!")        
+if not np.array_equal(np.round(np.array(mdl2a.getValueVector()),6),np.round(np.array([200.00114718191236,212.86672887958622,298.7090458676583]),6)):
+        sys.exit("Model 2a failed!")
+
+#Model 3a (discounted reward, parallel)
+mdl3a = mdpsolver.model()
+mdl3a.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatWithZeros=tranMatWithZeros)
+mdl3a.solve(algorithm="vi",
+          update="standard",
+          parallel=True)
+if not np.array_equal(np.array(mdl3a.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 3a failed!")        
+if not np.array_equal(np.round(np.array(mdl3a.getValueVector()),6),np.round(np.array([200.0013199959708, 212.86689665815788, 298.709197760198]),6)):
+        sys.exit("Model 3a failed!")
+
+#Model 1b (discounted reward, unparallel)
+mdl1b = mdpsolver.model()
+mdl1b.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatWithZeros=tranMatWithZeros)
+mdl1b.solve(algorithm="mpi",
+          update="standard",
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl1b.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 1b failed!")        
+if not np.array_equal(np.round(np.array(mdl1b.getValueVector()),6),np.round(np.array([200.00114718191236, 212.86672887958622, 298.7090458676583]),6)):
+        sys.exit("Model 1b failed!")
+
+#Model 2b (discounted reward, unparallel)
+mdl2b = mdpsolver.model()
+mdl2b.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatWithZeros=tranMatWithZeros)
+mdl2b.solve(algorithm="pi",
+          update="standard",
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl2b.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 2b failed!")        
+if not np.array_equal(np.round(np.array(mdl2b.getValueVector()),6),np.round(np.array([200.00114718191236, 212.86672887958622, 298.7090458676583]),6)):
+        sys.exit("Model 2b failed!")
+
+#Model 3b (discounted reward, unparallel)
+mdl3b = mdpsolver.model()
+mdl3b.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatWithZeros=tranMatWithZeros)
+mdl3b.solve(algorithm="vi",
+          update="standard",
+          parallel=False)
+if not np.array_equal(np.array(mdl3b.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 3b failed!")        
+if not np.array_equal(np.round(np.array(mdl3b.getValueVector()),6),np.round(np.array([200.0013199959708, 212.86689665815788, 298.709197760198]),6)):
+        sys.exit("Model 3b failed!")
+
+#Model 1a (average reward, parallel)
+mdl1a = mdpsolver.model()
+mdl1a.mdp(rewards=rewards,
+        tranMatWithZeros=tranMatWithZeros)
+mdl1a.solve(algorithm="mpi",
+          update="standard",
+          criterion="average",
+          parallel=True,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl1a.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 1a failed!")        
+if not np.array_equal(np.round(np.array(mdl1a.getValueVector()),6),np.round(np.array([354.2101307157537, 368.21018917357264, 457.21037278800077]),6)):
+        sys.exit("Model 1a failed!")
+
+#Model 2a (average reward, parallel)
+mdl2a = mdpsolver.model()
+mdl2a.mdp(rewards=rewards,
+        tranMatWithZeros=tranMatWithZeros)
+mdl2a.solve(algorithm="pi",
+          update="standard",
+          criterion="average",
+          parallel=True,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl2a.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 2a failed!")        
+if not np.array_equal(np.round(np.array(mdl2a.getValueVector()),6),np.round(np.array([354.2101307157537, 368.21018917357264, 457.21037278800077]),6)):
+        sys.exit("Model 2a failed!")
+
+#Model 3a (average reward, parallel)
+mdl3a = mdpsolver.model()
+mdl3a.mdp(rewards=rewards,
+          tranMatWithZeros=tranMatWithZeros)
+mdl3a.solve(algorithm="vi",
+          update="standard",
+          criterion="average",
+          parallel=True)
+if not np.array_equal(np.array(mdl3a.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 3a failed!")        
+if not np.array_equal(np.round(np.array(mdl3a.getValueVector()),6),np.round(np.array([158.23345351808334, 172.23340169038516, 261.233237936489]),6)):
+        sys.exit("Model 3a failed!")
+
+#Model 1b (average reward, unparallel)
+mdl1b = mdpsolver.model()
+mdl1b.mdp(rewards=rewards,
+          tranMatWithZeros=tranMatWithZeros)
+mdl1b.solve(algorithm="mpi",
+          update="standard",
+          criterion="average",
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl1b.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 1b failed!")        
+if not np.array_equal(np.round(np.array(mdl1b.getValueVector()),6),np.round(np.array([354.2101307157537, 368.21018917357264, 457.21037278800077]),6)):
+        sys.exit("Model 1b failed!")
+
+#Model 2b (average reward, unparallel)
+mdl2b = mdpsolver.model()
+mdl2b.mdp(rewards=rewards,
+          tranMatWithZeros=tranMatWithZeros)
+mdl2b.solve(algorithm="pi",
+          update="standard",
+          criterion="average",
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl2b.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 2b failed!")        
+if not np.array_equal(np.round(np.array(mdl2b.getValueVector()),6),np.round(np.array([354.2101307157537, 368.21018917357264, 457.21037278800077]),6)):
+        sys.exit("Model 2b failed!")
+
+#Model 3b (average reward, unparallel)
+mdl3b = mdpsolver.model()
+mdl3b.mdp(rewards=rewards,
+          tranMatWithZeros=tranMatWithZeros)
+mdl3b.solve(algorithm="vi",
+          update="standard",
+          criterion="average",
+          parallel=False)
+if not np.array_equal(np.array(mdl3b.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 3b failed!")        
+if not np.array_equal(np.round(np.array(mdl3b.getValueVector()),6),np.round(np.array([158.23345351808334, 172.23340169038516, 261.233237936489]),6)):
+        sys.exit("Model 3b failed!")
+
+#Model 1c (discounted, Gauss-Seidel)
+mdl1c = mdpsolver.model()
+mdl1c.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatWithZeros=tranMatWithZeros)
+mdl1c.solve(algorithm="mpi",
+          update="gs",
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl1c.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 1c failed!")        
+if not np.array_equal(np.round(np.array(mdl1c.getValueVector()),6),np.round(np.array([200.0012771893156, 212.86686671568802, 298.7091798650016]),6)):
+        sys.exit("Model 1c failed!")
+
+#Model 2c (Gauss-Seidel)
+initPolicy=[1,1,1]
+mdl2c = mdpsolver.model()
+mdl2c.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatWithZeros=tranMatWithZeros)
+mdl2c.solve(algorithm="pi",
+            update="gs",
+            parallel=False,
+            initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl2c.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 2c failed!")        
+if not np.array_equal(np.round(np.array(mdl2c.getValueVector()),6),np.round(np.array([200.0012771893156, 212.86686671568802, 298.7091798650016]),6)):
+        sys.exit("Model 2c failed!")
+
+#Model 3c (Gauss-Seidel)
+mdl3c = mdpsolver.model()
+mdl3c.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatWithZeros=tranMatWithZeros)
+mdl3c.solve(algorithm="vi",
+          update="gs",
+          parallel=False)
+if not np.array_equal(np.array(mdl3c.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 3c failed!")        
+if not np.array_equal(np.round(np.array(mdl3c.getValueVector()),6),np.round(np.array([200.0010604606014, 212.86664512773723, 298.7089561808565]),6)):
+        sys.exit("Model 3c failed!")
+
+#Model 1d (SOR)
+mdl1d = mdpsolver.model()
+mdl1d.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatWithZeros=tranMatWithZeros)
+mdl1d.solve(algorithm="mpi",
+          update="sor",
+          SORrelaxation=1.01,
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl1d.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 1d failed!")        
+if not np.array_equal(np.round(np.array(mdl1d.getValueVector()),6),np.round(np.array([200.00127021272166, 212.86686059325598, 298.70917425463125]),6)):
+        sys.exit("Model 1d failed!")
+
+#Model 2d (SOR)
+mdl2d = mdpsolver.model()
+mdl2d.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatWithZeros=tranMatWithZeros)
+mdl2d.solve(algorithm="pi",
+          update="sor",
+          SORrelaxation=1.01,
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl2d.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 2d failed!")        
+if not np.array_equal(np.round(np.array(mdl2d.getValueVector()),6),np.round(np.array([200.00127021272166, 212.86686059325598, 298.70917425463125]),6)):
+        sys.exit("Model 2d failed!")
+
+#Model 3d (SOR)
+mdl3d = mdpsolver.model()
+mdl3d.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatWithZeros=tranMatWithZeros)
+mdl3d.solve(algorithm="vi",
+          update="sor",
+          SORrelaxation=1.01,
+          parallel=False)
+if not np.array_equal(np.array(mdl3d.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 3d failed!")        
+if not np.array_equal(np.round(np.array(mdl3d.getValueVector()),6),np.round(np.array([206.69740452375487, 222.30968454932525, 305.91165352015094]),6)):
+        sys.exit("Model 3d failed!")
+
+#---------------------------------------
+# CONFIGURATION 2
+#---------------------------------------
+
+#rewards
+#1st index: from (current) states
+#2nd index: actions
+rewards = [[5,-1],
+           [1,-2],
+           [50,0]]
+
+#transition probabilities
+#[from_state,action,to_state,probability]
+tranMatElementwise = [[0,0,0,0.9],
+                      [0,0,1,0.1],
+                      [0,1,0,0.1],
+                      [0,1,1,0.9],
+                      [1,0,0,0.4],
+                      [1,0,1,0.5],
+                      [1,0,2,0.1],
+                      [1,1,0,0.3],
+                      [1,1,1,0.5],
+                      [1,1,2,0.2],
+                      [2,0,0,0.2],
+                      [2,0,1,0.2],
+                      [2,0,2,0.6],
+                      [2,1,0,0.5],
+                      [2,1,1,0.5]]
+
+#initial policy
+random.seed(10)
+initPolicy = [randint(0, 1) for p in range(0, 3)]
+
+#Model 1a
+mdl1a = mdpsolver.model()
+mdl1a.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatElementwise=tranMatElementwise)
+mdl1a.solve(algorithm="mpi",
+          update="standard",
+          parallel=True,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl1a.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 1a failed!")        
+if not np.array_equal(np.round(np.array(mdl1a.getValueVector()),6),np.round(np.array([200.00114718191236, 212.86672887958622, 298.7090458676583]),6)):
+        sys.exit("Model 1a failed!")
+
+#Model 2a
+mdl2a = mdpsolver.model()
+mdl2a.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatElementwise=tranMatElementwise)
+mdl2a.solve(algorithm="pi",
+          update="standard",
+          parallel=True,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl2a.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 2a failed!")        
+if not np.array_equal(np.round(np.array(mdl2a.getValueVector()),6),np.round(np.array([200.00114718191236, 212.86672887958622, 298.7090458676583]),6)):
+        sys.exit("Model 2a failed!")
+
+#Model 3a
+mdl3a = mdpsolver.model()
+mdl3a.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatElementwise=tranMatElementwise)
+mdl3a.solve(algorithm="vi",
+          update="standard",
+          parallel=True)
+if not np.array_equal(np.array(mdl3a.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 3a failed!")        
+if not np.array_equal(np.round(np.array(mdl3a.getValueVector()),6),np.round(np.array([200.0013199959708, 212.86689665815788, 298.709197760198]),6)):
+        sys.exit("Model 3a failed!")
+
+#Model 1b
+mdl1b = mdpsolver.model()
+mdl1b.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatElementwise=tranMatElementwise)
+mdl1b.solve(algorithm="mpi",
+          update="standard",
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl1b.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 1b failed!")        
+if not np.array_equal(np.round(np.array(mdl1b.getValueVector()),6),np.round(np.array([200.00114718191236, 212.86672887958622, 298.7090458676583]),6)):
+        sys.exit("Model 1b failed!")
+
+#Model 2b
+mdl2b = mdpsolver.model()
+mdl2b.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatElementwise=tranMatElementwise)
+mdl2b.solve(algorithm="pi",
+          update="standard",
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl2b.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 2b failed!")        
+if not np.array_equal(np.round(np.array(mdl2b.getValueVector()),6),np.round(np.array([200.00114718191236, 212.86672887958622, 298.7090458676583]),6)):
+        sys.exit("Model 2b failed!")
+
+#Model 3b
+mdl3b = mdpsolver.model()
+mdl3b.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatElementwise=tranMatElementwise)
+mdl3b.solve(algorithm="vi",
+          update="standard",
+          parallel=False)
+if not np.array_equal(np.array(mdl3b.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 3b failed!")        
+if not np.array_equal(np.round(np.array(mdl3b.getValueVector()),6),np.round(np.array([200.0013199959708, 212.86689665815788, 298.709197760198]),6)):
+        sys.exit("Model 3b failed!")
+
+#Model 1c
+mdl1c = mdpsolver.model()
+mdl1c.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatElementwise=tranMatElementwise)
+mdl1c.solve(algorithm="mpi",
+          update="gs",
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl1c.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 1c failed!")        
+if not np.array_equal(np.round(np.array(mdl1c.getValueVector()),6),np.round(np.array([200.0012771893156, 212.86686671568802, 298.7091798650016]),6)):
+        sys.exit("Model 1c failed!")
+
+#Model 2c
+mdl2c = mdpsolver.model()
+mdl2c.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatElementwise=tranMatElementwise)
+mdl2c.solve(algorithm="pi",
+          update="gs",
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl2c.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 2c failed!")        
+if not np.array_equal(np.round(np.array(mdl2c.getValueVector()),6),np.round(np.array([200.0012771893156, 212.86686671568802, 298.7091798650016]),6)):
+        sys.exit("Model 2c failed!")
+
+#Model 3c
+mdl3c = mdpsolver.model()
+mdl3c.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatElementwise=tranMatElementwise)
+mdl3c.solve(algorithm="vi",
+          update="gs",
+          parallel=False)
+if not np.array_equal(np.array(mdl3c.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 3c failed!")        
+if not np.array_equal(np.round(np.array(mdl3c.getValueVector()),6),np.round(np.array([200.0010604606014, 212.86664512773723, 298.7089561808565]),6)):
+        sys.exit("Model 3c failed!")
+
+#Model 1d
+mdl1d = mdpsolver.model()
+mdl1d.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatElementwise=tranMatElementwise)
+mdl1d.solve(algorithm="mpi",
+          update="sor",
+          SORrelaxation=1.01,
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl1d.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 1d failed!")        
+if not np.array_equal(np.round(np.array(mdl1d.getValueVector()),6),np.round(np.array([200.00127021272166, 212.86686059325598, 298.70917425463125]),6)):
+        sys.exit("Model 1d failed!")
+
+#Model 2d
+mdl2d = mdpsolver.model()
+mdl2d.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatElementwise=tranMatElementwise)
+mdl2d.solve(algorithm="pi",
+          update="sor",
+          SORrelaxation=1.01,
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl2d.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 2d failed!")        
+if not np.array_equal(np.round(np.array(mdl2d.getValueVector()),6),np.round(np.array([200.00127021272166, 212.86686059325598, 298.70917425463125]),6)):
+        sys.exit("Model 2d failed!")
+
+#Model 3d
+mdl3d = mdpsolver.model()
+mdl3d.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatElementwise=tranMatElementwise)
+mdl3d.solve(algorithm="vi",
+          update="sor",
+          SORrelaxation=1.01,
+          parallel=False)
+if not np.array_equal(np.array(mdl3d.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 3d failed!")        
+if not np.array_equal(np.round(np.array(mdl3d.getValueVector()),6),np.round(np.array([206.69740452375487, 222.30968454932525, 305.91165352015094]),6)):
+        sys.exit("Model 3d failed!")
+
+#---------------------------------------
+# CONFIGURATION 3
+#---------------------------------------
+
+#rewards
+#1st index: from (current) states
+#2nd index: actions
+rewards = [[5,-1],
+           [1,-2],
+           [50,0]]
+
+#transition probabilities
+tranMatProbs = [[[0.9,0.1],[0.1,0.9]],
+                [[0.4,0.5,0.1],[0.3,0.5,0.2]],
+                [[0.2,0.2,0.6],[0.5,0.5]]]
+
+tranMatColumns = [[[0,1],[0,1]],
+                [[0,1,2],[0,1,2]],
+                [[0,1,2],[0,1]]]
+
+#initial policy
+random.seed(10)
+initPolicy = [randint(0, 1) for p in range(0, 3)]
+
+#Model 1a
+mdl1a = mdpsolver.model()
+mdl1a.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatProbs=tranMatProbs,
+        tranMatColumns=tranMatColumns)
+mdl1a.solve(algorithm="mpi",
+          update="standard",
+          parallel=True,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl1a.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 1a failed!")        
+if not np.array_equal(np.round(np.array(mdl1a.getValueVector()),6),np.round(np.array([200.00114718191236, 212.86672887958622, 298.7090458676583]),6)):
+        sys.exit("Model 1a failed!")
+
+#Model 2a
+mdl2a = mdpsolver.model()
+mdl2a.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatProbs=tranMatProbs,
+        tranMatColumns=tranMatColumns)
+mdl2a.solve(algorithm="pi",
+          update="standard",
+          parallel=True,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl2a.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 2a failed!")        
+if not np.array_equal(np.round(np.array(mdl2a.getValueVector()),6),np.round(np.array([200.00114718191236, 212.86672887958622, 298.7090458676583]),6)):
+        sys.exit("Model 2a failed!")
+
+#Model 3a
+mdl3a = mdpsolver.model()
+mdl3a.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatProbs=tranMatProbs,
+        tranMatColumns=tranMatColumns)
+mdl3a.solve(algorithm="vi",
+          update="standard",
+          parallel=True)
+if not np.array_equal(np.array(mdl3a.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 3a failed!")        
+if not np.array_equal(np.round(np.array(mdl3a.getValueVector()),6),np.round(np.array([200.0013199959708, 212.86689665815788, 298.709197760198]),6)):
+        sys.exit("Model 3a failed!")
+
+#Model 1b
+mdl1b = mdpsolver.model()
+mdl1b.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatProbs=tranMatProbs,
+        tranMatColumns=tranMatColumns)
+mdl1b.solve(algorithm="mpi",
+          update="standard",
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl1b.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 1b failed!")        
+if not np.array_equal(np.round(np.array(mdl1b.getValueVector()),6),np.round(np.array([200.00114718191236, 212.86672887958622, 298.7090458676583]),6)):
+        sys.exit("Model 1b failed!")
+
+#Model 2b
+mdl2b = mdpsolver.model()
+mdl2b.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatProbs=tranMatProbs,
+        tranMatColumns=tranMatColumns)
+mdl2b.solve(algorithm="pi",
+          update="standard",
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl2b.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 2b failed!")        
+if not np.array_equal(np.round(np.array(mdl2b.getValueVector()),6),np.round(np.array([200.00114718191236, 212.86672887958622, 298.7090458676583]),6)):
+        sys.exit("Model 2b failed!")
+
+#Model 3b
+mdl3b = mdpsolver.model()
+mdl3b.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatProbs=tranMatProbs,
+        tranMatColumns=tranMatColumns)
+mdl3b.solve(algorithm="vi",
+          update="standard",
+          parallel=False)
+if not np.array_equal(np.array(mdl3b.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 3b failed!")        
+if not np.array_equal(np.round(np.array(mdl3b.getValueVector()),6),np.round(np.array([200.0013199959708, 212.86689665815788, 298.709197760198]),6)):
+        sys.exit("Model 3b failed!")
+
+#Model 1c
+mdl1c = mdpsolver.model()
+mdl1c.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatProbs=tranMatProbs,
+        tranMatColumns=tranMatColumns)
+mdl1c.solve(algorithm="mpi",
+          update="gs",
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl1c.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 1c failed!")        
+if not np.array_equal(np.round(np.array(mdl1c.getValueVector()),6),np.round(np.array([200.0012771893156, 212.86686671568802, 298.7091798650016]),6)):
+        sys.exit("Model 1c failed!")
+
+#Model 2c
+mdl2c = mdpsolver.model()
+mdl2c.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatProbs=tranMatProbs,
+        tranMatColumns=tranMatColumns)
+mdl2c.solve(algorithm="pi",
+          update="gs",
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl2c.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 2c failed!")        
+if not np.array_equal(np.round(np.array(mdl2c.getValueVector()),6),np.round(np.array([200.0012771893156, 212.86686671568802, 298.7091798650016]),6)):
+        sys.exit("Model 2c failed!")
+
+#Model 3c
+mdl3c = mdpsolver.model()
+mdl3c.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatProbs=tranMatProbs,
+        tranMatColumns=tranMatColumns)
+mdl3c.solve(algorithm="vi",
+          update="gs",
+          parallel=False)
+if not np.array_equal(np.array(mdl3c.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 3c failed!")        
+if not np.array_equal(np.round(np.array(mdl3c.getValueVector()),6),np.round(np.array([200.0010604606014, 212.86664512773723, 298.7089561808565]),6)):
+        sys.exit("Model 3c failed!")
+
+#Model 1d
+mdl1d = mdpsolver.model()
+mdl1d.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatProbs=tranMatProbs,
+        tranMatColumns=tranMatColumns)
+mdl1d.solve(algorithm="mpi",
+          update="sor",
+          SORrelaxation=1.01,
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl1d.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 1d failed!")        
+if not np.array_equal(np.round(np.array(mdl1d.getValueVector()),6),np.round(np.array([200.00127021272166, 212.86686059325598, 298.70917425463125]),6)):
+        sys.exit("Model 1d failed!")
+
+#Model 2d
+mdl2d = mdpsolver.model()
+mdl2d.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatProbs=tranMatProbs,
+        tranMatColumns=tranMatColumns)
+mdl2d.solve(algorithm="pi",
+          update="sor",
+          SORrelaxation=1.01,
+          parallel=False,
+          initPolicy=initPolicy)
+if not np.array_equal(np.array(mdl2d.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 2d failed!")        
+if not np.array_equal(np.round(np.array(mdl2d.getValueVector()),6),np.round(np.array([200.00127021272166, 212.86686059325598, 298.70917425463125]),6)):
+        sys.exit("Model 2d failed!")
+
+#Model 3d
+mdl3d = mdpsolver.model()
+mdl3d.mdp(discount=0.95,
+        rewards=rewards,
+        tranMatProbs=tranMatProbs,
+        tranMatColumns=tranMatColumns)
+mdl3d.solve(algorithm="vi",
+          update="sor",
+          SORrelaxation=1.01,
+          parallel=False)
+if not np.array_equal(np.array(mdl3d.getPolicy()),np.array([1,1,0])):
+        sys.exit("Model 3d failed!")        
+if not np.array_equal(np.round(np.array(mdl3d.getValueVector()),6),np.round(np.array([206.69740452375487, 222.30968454932525, 305.91165352015094]),6)):
+        sys.exit("Model 3d failed!")
+
+print("Test 1 succesfully reproduced output!")

mdpsolver-0.9.8/tests/test2.py

@@ -0,0 +1,81 @@
+import mdpsolver
+import example_model
+import sys
+import numpy as np
+
+#TEST 2
+#In this test, we load the rewards and transition probabilities
+#from two CSV-files (rewards_example.csv and transitions.csv).
+#In addition, we test three optimization algorithms and compare
+#the runtimes.
+
+#PROBLEM DESCRIPTION: We derive the policy for a decision problem
+#involving a queueing system with finite capacity and two customer
+#types. The system is observed whenever a new customer arrives.
+#Customers are not allowed to wait and can only be accepted if there
+#are idle servers. Thus, with each arrival, we must choose whether
+#to accept or reject the customer. We receive a low reward for
+#accepting the first, more common type, and a high reward for
+#accepting the second, rarer type.
+
+#---------------------------------------
+# CREATE PARAMETERS
+#---------------------------------------
+
+#rewards
+capacity=100
+rewardCust1=1
+rewardCust2=1000
+example_model.rewards(capacity,rewardCust1,rewardCust2,"rewards_example2.csv")
+
+#transitions
+arrRate1=50
+arrRate2=10
+serRate=3/4
+example_model.tranMat(arrRate1,arrRate2,serRate,capacity,"transitions_example2.csv")
+
+#---------------------------------------
+# VALUE ITERATION
+#---------------------------------------
+
+mdl1 = mdpsolver.model()
+
+mdl1.mdp(discount=0.99,
+        rewardsFromFile="rewards_example2.csv",
+        tranMatFromFile="transitions_example2.csv")
+
+mdl1.solve(algorithm="vi")
+if not np.round(np.mean(mdl1.getValueVector()),6)==np.round(17074.81609832942,6):
+        sys.exit("Model 1 failed!")
+
+#---------------------------------------
+# POLICY ITERATION
+#---------------------------------------
+
+mdl2 = mdpsolver.model()
+
+mdl2.mdp(discount=0.99,
+        rewardsFromFile="rewards_example2.csv",
+        tranMatFromFile="transitions_example2.csv")
+
+mdl2.solve(algorithm="pi")
+
+if not np.round(np.mean(mdl2.getValueVector()),6)==np.round(17074.815578793514,6):
+        sys.exit("Model 2 failed!")
+
+#---------------------------------------
+# MODIFIED POLICY ITERATION
+#---------------------------------------
+
+mdl3 = mdpsolver.model()
+
+mdl3.mdp(discount=0.99,
+        rewardsFromFile="rewards_example2.csv",
+        tranMatFromFile="transitions_example2.csv")
+
+mdl3.solve(algorithm="mpi")
+
+if not np.round(np.mean(mdl3.getValueVector()),6)==np.round(17074.815563750446,6):
+        sys.exit("Model 3 failed!")
+        
+print("Test 2 succesfully reproduced output!")