mdpsolver 0.11.0__tar.gz → 0.12.0__tar.gz

{mdpsolver-0.11.0 → mdpsolver-0.12.0}/CPP_Source_Code/GeneralMDPmodel.cpp

@@ -23,6 +23,7 @@
 */
 
 #include "GeneralMDPmodel.h"
+#include <limits>
 
 GeneralMDPmodel::GeneralMDPmodel(Rewards * rw, TransitionMatrix * tm, double discount):
 rewards(rw),
@@ -40,48 +41,70 @@ GeneralMDPmodel::~GeneralMDPmodel() {
 
 
 void GeneralMDPmodel::initialize(){
-    numberOfStates=tranMat->numberOfRows();
+    tranMat->fillTables(csr);
+    numberOfStates=csr.numberOfStates;
     cidx=0;
+
+    //The tables need one reward per pair of the transition probabilities. In a well-formed
+    //model the rewards have the same numbers of actions, and so the same numbering, and
+    //their array is used as it is. Otherwise (which checkModel() reports) a copy numbered
+    //like the pairs is made, so that the solver reads no memory outside the rewards.
+    bool aligned = rewards->numberOfRows() >= numberOfStates;
+    for (int sidx=0; aligned && sidx<numberOfStates; sidx++){
+        aligned = rewards->numberOfActions(sidx) == tranMat->numberOfActions(sidx);
+    }
+    if (aligned){
+        csr.reward = rewards->data();
+    }else{
+        alignedRewards.assign(csr.stateStart[numberOfStates], numeric_limits<double>::quiet_NaN());
+        for (int sidx=0; sidx<numberOfStates && sidx<rewards->numberOfRows(); sidx++){
+            for (int aidx=0; aidx<tranMat->numberOfActions(sidx) && aidx<rewards->numberOfActions(sidx); aidx++){
+                alignedRewards[pair(sidx,aidx)] = rewards->getReward(sidx,aidx);
+            }
+        }
+        csr.reward = alignedRewards.data();
+    }
 }
-        
+
 double GeneralMDPmodel::reward(int &sidx, int &aidx){
     //calculates and returns the reward of taking action
     //aidx when in current state sidx
 
-    return rewards->getReward(sidx,aidx);
+    return csr.reward[pair(sidx,aidx)];
 }
 
 double GeneralMDPmodel::transProb(int &sidx, int &aidx, int &jidx){
-    //calculates the probability of jumping to state jidx from
-    //the current state sidx when taking action aidx.
-    //returns the calculated probability.
+    //returns the jidx-th non-zero transition probability of taking
+    //action aidx in state sidx (jidx counts the non-zero probabilities)
 
-    return tranMat->getProb(sidx,aidx,jidx);
+    return probability(entry(sidx,aidx,jidx));
 }
 
 int GeneralMDPmodel::getNumberOfJumps(int &sidx, int &aidx){
-    return tranMat->numberOfColumns(sidx,aidx);
+    int64_t i = pair(sidx,aidx);
+    return (int)(csr.pairStart[i+1] - csr.pairStart[i]);
 }
 
 void GeneralMDPmodel::updateNextState(int &sidx, int &aidx, int &jidx){
     //updates the next possible state, nextState, and the associated
-    //transition probability, psj.    
-    while (tranMat->getColumn(sidx,aidx,cidx)!=jidx){ //this will in many cases immediately evaluate to False
+    //transition probability, psj.
+    int nJumps = getNumberOfJumps(sidx,aidx);
+    while (csr.column[entry(sidx,aidx,cidx)]!=jidx){ //this will in many cases immediately evaluate to False
         cidx++;
-        if (cidx==tranMat->numberOfColumns(sidx,aidx)){ //to make sure cidx is always feasible
+        if (cidx==nJumps){ //to make sure cidx is always feasible
             cidx=0;
-        }    
+        }
     }
     cidx++; //move one additional step
-    if (cidx==tranMat->numberOfColumns(sidx,aidx)){
+    if (cidx==nJumps){
         cidx=0;
     }
-    nextState = tranMat->getColumn(sidx,aidx,cidx);
-    psj = tranMat->getProb(sidx,aidx,cidx);
-}    
+    nextState = csr.column[entry(sidx,aidx,cidx)];
+    psj = probability(entry(sidx,aidx,cidx));
+}
 
 int GeneralMDPmodel::getColumnIdx(int &sidx, int &aidx, int &cidx){
-    return tranMat->getColumn(sidx,aidx,cidx);
+    return csr.column[entry(sidx,aidx,cidx)];
 }
 
 int GeneralMDPmodel::postDecisionIdx(int &sidx, int &aidx){
@@ -90,7 +113,7 @@ int GeneralMDPmodel::postDecisionIdx(int &sidx, int &aidx){
     //both returns the value of the first new state, and
     //stores it in the variable nextState.
     cidx=0;
-    nextState = tranMat->getColumn(sidx,aidx,cidx);
+    nextState = csr.column[entry(sidx,aidx,cidx)];
     return nextState;
 }
 
@@ -103,7 +126,7 @@ int GeneralMDPmodel::getNumberOfStates(){
 }
 
 void GeneralMDPmodel::updateNumberOfActions(int &sidx){
-    numberOfActions=tranMat->numberOfActions(sidx);
+    numberOfActions=(int)(csr.stateStart[sidx+1] - csr.stateStart[sidx]);
 }
 
 int GeneralMDPmodel::getNumberOfActions(){
@@ -111,7 +134,7 @@ int GeneralMDPmodel::getNumberOfActions(){
 }
 
 int GeneralMDPmodel::getNumberOfActions(int &sidx){
-    return tranMat->numberOfActions(sidx);
+    return (int)(csr.stateStart[sidx+1] - csr.stateStart[sidx]);
 }
 
 int * GeneralMDPmodel::getNextState(){

{mdpsolver-0.11.0 → mdpsolver-0.12.0}/CPP_Source_Code/GeneralMDPmodel.h

@@ -34,9 +34,12 @@
 
 using namespace std;
 
+//The general MDP model: the transition probabilities and rewards the user gave to mdp(),
+//held as CSR tables (ModelType.h), which the solver reads directly. The methods of
+//ModelType read the same tables, for the code that goes through them.
 class GeneralMDPmodel : public ModelType{
 public:
-    
+
     //CONSTRUCTOR AND DESCTRUCTOR
     GeneralMDPmodel() {}; //dummy-constructor
     GeneralMDPmodel(Rewards * rw, TransitionMatrix * tm, double discount);
@@ -48,15 +51,18 @@ public:
 
     double discount;
 	int numberOfStates, numberOfActions;
-    
+
     //auxiliary variables
     int nextState; //a new state that the current state can jump to
     double psj; //transition probability from the current state to nextState
-	
+
     //METHODS
     void initialize();
-        
-    //GENERIC METHODS    
+
+    //the tables, which the solver's fast loops read
+    const CSRTables* tables() override { return &csr; }
+
+    //GENERIC METHODS
     double reward(int &sidx, int &aidx) override;
     double transProb(int &sidx, int &aidx, int &jidx) override;
     void updateNextState(int &sidx, int &aidx, int &jidx) override;
@@ -76,9 +82,19 @@ private:
     //VARIABLES
     Rewards * rewards;
     TransitionMatrix * tranMat;
+    CSRTables csr;
+    //the rewards numbered like the pairs of the transition probabilities, made only when the
+    //rewards have other numbers of actions or states (a malformed model); rewards the input
+    //lacks are NaN here, and those it has beyond the transition probabilities are left out
+    vector<double> alignedRewards;
     int cidx;
-    
+
+    //METHODS
+    int64_t pair(int sidx, int aidx) const { return csr.stateStart[sidx] + aidx; }
+    int64_t entry(int sidx, int aidx, int cidx) const { return csr.pairStart[pair(sidx,aidx)] + cidx; }
+    double probability(int64_t k) const { return csr.prob32 ? (double)csr.prob32[k] : csr.prob[k]; }
 
 };
 
-#endif /* TBMMODEL_H */
+#endif /* GENERALMDPMODEL_H */
+

mdpsolver-0.12.0/CPP_Source_Code/ModelType.h

@@ -0,0 +1,53 @@
+#ifndef MODELTYPE_H
+#define MODELTYPE_H
+
+#include <cstdint>
+
+//The transitions and rewards of a model, stored as tables in compressed sparse row (CSR)
+//form. The state-action pairs are numbered state by state, and within a state in the order
+//of its actions, so that pair stateStart[s] + a is action a of state s.
+struct CSRTables {
+    int numberOfStates = 0;
+    const int64_t* stateStart = nullptr; //numberOfStates+1 entries: the pairs of state s are stateStart[s], ..., stateStart[s+1]-1
+    const int64_t* pairStart = nullptr;  //one entry more than there are pairs: the non-zero transition probabilities of pair i are pairStart[i], ..., pairStart[i+1]-1
+    const int* column = nullptr;         //the next state of each non-zero transition probability
+    const double* prob = nullptr;        //and the probability itself, in double precision,
+    const float* prob32 = nullptr;       //or in single precision (float32): exactly one of the two is set
+    const double* reward = nullptr;      //the reward of each pair
+};
+
+//A model the solver can solve. There are two ways to supply one:
+// - Store the transitions and rewards as CSR tables and return them from tables(), as the
+//   general MDP model does. The solver then reads the arrays directly in its inner loops,
+//   which is much faster than calling the methods below for every transition.
+// - Compute the transitions on the fly in the methods below, as the built-in TBM and CBM
+//   models do, and leave tables() returning nullptr. The solver then runs its routines for
+//   such models, which call these methods. This suits models whose transitions follow a
+//   formula and would take too much memory to store.
+//(A special-case model of the second kind can also fill CSR tables once from its formulas
+//and return them, if they fit in memory, to be solved with the fast loops.)
+class ModelType {
+public:
+
+    virtual ~ModelType() {}
+
+    //the model's CSR tables, or nullptr if it computes its transitions on the fly
+    virtual const CSRTables* tables() { return nullptr; }
+
+    virtual double getDiscount() = 0;
+    virtual int getNumberOfStates() = 0;
+    virtual void updateNumberOfActions(int &sidx) = 0;
+    virtual int getNumberOfActions() = 0;
+    virtual int getNumberOfActions(int &sidx) = 0;
+    virtual int * getNextState() = 0;
+    virtual double getPsj() = 0;
+    virtual double reward(int &sidx, int &aidx) = 0;
+    virtual double transProb(int &sidx, int &aidx, int &jidx) = 0;
+    virtual void updateNextState(int &sidx, int &aidx, int &jidx) = 0;
+    virtual int postDecisionIdx(int &sidx, int &aidx) = 0;
+    virtual int getColumnIdx(int &sidx, int &aidx, int &cidx) = 0;
+    virtual int getNumberOfJumps(int &sidx, int &aidx) = 0;
+
+};
+
+#endif  // MODELTYPE_H