pyRDDLGym-jax 0.3__py3-none-any.whl → 0.5__py3-none-any.whl

pyRDDLGym_jax/core/simulator.py

@@ -1,7 +1,8 @@
-import jax
 import time
 from typing import Dict, Optional
 
+import jax
+
 from pyRDDLGym.core.compiler.model import RDDLLiftedModel
 from pyRDDLGym.core.debug.exception import (
     RDDLActionPreconditionNotSatisfiedError,

pyRDDLGym_jax/core/tuning.py

@@ -1,20 +1,18 @@
-from bayes_opt import BayesianOptimization
-from bayes_opt.util import UtilityFunction
 from copy import deepcopy
 import csv
 import datetime
-import jax
 from multiprocessing import get_context
-import numpy as np
 import os
 import time
 from typing import Any, Callable, Dict, Optional, Tuple
-
-Kwargs = Dict[str, Any]
-
 import warnings
 warnings.filterwarnings("ignore")
 
+from bayes_opt import BayesianOptimization
+from bayes_opt.acquisition import AcquisitionFunction, UpperConfidenceBound
+import jax
+import numpy as np
+
 from pyRDDLGym.core.debug.exception import raise_warning
 from pyRDDLGym.core.env import RDDLEnv
 
@@ -26,6 +24,7 @@ from pyRDDLGym_jax.core.planner import (
     JaxOnlineController
 )
 
+Kwargs = Dict[str, Any]
 
 # ===============================================================================
 # 
@@ -37,6 +36,9 @@ from pyRDDLGym_jax.core.planner import (
 # 3. deep reactive policies
 # 
 # ===============================================================================
+COLUMNS = ['pid', 'worker', 'iteration', 'target', 'best_target', 'acq_params']
+
+
 class JaxParameterTuning:
     '''A general-purpose class for tuning a Jax planner.'''
     
@@ -53,7 +55,7 @@ class JaxParameterTuning:
                  num_workers: int=1,
                  poll_frequency: float=0.2,
                  gp_iters: int=25,
-                 acquisition: Optional[UtilityFunction]=None,
+                 acquisition: Optional[AcquisitionFunction]=None,
                  gp_init_kwargs: Optional[Kwargs]=None,
                  gp_params: Optional[Kwargs]=None) -> None:
         '''Creates a new instance for tuning hyper-parameters for Jax planners
@@ -113,10 +115,9 @@ class JaxParameterTuning:
         self.gp_params = gp_params
         
         # create acquisition function
-        self.acq_args = None
         if acquisition is None:
             num_samples = self.gp_iters * self.num_workers
-            acquisition, self.acq_args = JaxParameterTuning._annealing_utility(num_samples)
+            acquisition = JaxParameterTuning._annealing_acquisition(num_samples)
         self.acquisition = acquisition
     
     def summarize_hyperparameters(self) -> None:
@@ -133,23 +134,15 @@ class JaxParameterTuning:
               f'    planning_trials_per_iter  ={self.eval_trials}\n'
               f'    planning_iters_per_trial  ={self.train_epochs}\n'
               f'    planning_timeout_per_trial={self.timeout_training}\n'
-              f'    acquisition_fn            ={type(self.acquisition).__name__}')
-        if self.acq_args is not None:
-            print(f'using default acquisition function:\n'
-                  f'    utility_kind ={self.acq_args[0]}\n'
-                  f'    initial_kappa={self.acq_args[1]}\n'
-                  f'    kappa_decay  ={self.acq_args[2]}')
+              f'    acquisition_fn            ={self.acquisition}')
         
     @staticmethod
-    def _annealing_utility(n_samples, n_delay_samples=0, kappa1=10.0, kappa2=1.0):
-        kappa_decay = (kappa2 / kappa1) ** (1.0 / (n_samples - n_delay_samples))
-        utility_fn = UtilityFunction(
-            kind='ucb',
+    def _annealing_acquisition(n_samples, n_delay_samples=0, kappa1=10.0, kappa2=1.0):
+        acq_fn = UpperConfidenceBound(
             kappa=kappa1,
-            kappa_decay=kappa_decay,
-            kappa_decay_delay=n_delay_samples)
-        utility_args = ['ucb', kappa1, kappa_decay]
-        return utility_fn, utility_args
+            exploration_decay=(kappa2 / kappa1) ** (1.0 / (n_samples - n_delay_samples)),
+            exploration_decay_delay=n_delay_samples)
+        return acq_fn
     
     def _pickleable_objective_with_kwargs(self):
         raise NotImplementedError
@@ -160,7 +153,7 @@ class JaxParameterTuning:
         pid = os.getpid()
         return index, pid, params, target
 
-    def tune(self, key: jax.random.PRNGKey, 
+    def tune(self, key: jax.random.PRNGKey,
              filename: str,
              save_plot: bool=False) -> Dict[str, Any]:
         '''Tunes the hyper-parameters for Jax planner, returns the best found.'''
@@ -178,32 +171,28 @@ class JaxParameterTuning:
             for (name, hparam) in self.hyperparams_dict.items()
         }
         optimizer = BayesianOptimization(
-            f=None,  # probe() is not called
+            f=None,
+            acquisition_function=self.acquisition,
             pbounds=hyperparams_bounds,
             allow_duplicate_points=True,  # to avoid crash
             random_state=np.random.RandomState(key),
             **self.gp_init_kwargs
         )
         optimizer.set_gp_params(**self.gp_params)
-        utility = self.acquisition
         
         # suggest initial parameters to evaluate
         num_workers = self.num_workers
-        suggested, kappas = [], []
+        suggested, acq_params = [], []
         for _ in range(num_workers):
-            utility.update_params()
-            probe = optimizer.suggest(utility)
-            suggested.append(probe)  
-            kappas.append(utility.kappa)
+            probe = optimizer.suggest()
+            suggested.append(probe) 
+            acq_params.append(vars(optimizer.acquisition_function))
         
         # clear and prepare output file
         filename = self._filename(filename, 'csv')
         with open(filename, 'w', newline='') as file:
             writer = csv.writer(file)
-            writer.writerow(
-                ['pid', 'worker', 'iteration', 'target', 'best_target', 'kappa'] + \
-                list(hyperparams_bounds.keys())
-            )
+            writer.writerow(COLUMNS + list(hyperparams_bounds.keys()))
                 
         # start multiprocess evaluation
         worker_ids = list(range(num_workers))
@@ -219,8 +208,8 @@ class JaxParameterTuning:
             
             # continue with next iteration
             print('\n' + '*' * 25 + 
-                  '\n' + f'[{datetime.timedelta(seconds=elapsed)}] ' + 
-                  f'starting iteration {it}' + 
+                  f'\n[{datetime.timedelta(seconds=elapsed)}] ' + 
+                  f'starting iteration {it + 1}' + 
                   '\n' + '*' * 25)
             key, *subkeys = jax.random.split(key, num=num_workers + 1)
             rows = [None] * num_workers
@@ -256,10 +245,9 @@ class JaxParameterTuning:
                         optimizer.register(params, target)
                         
                         # update acquisition function and suggest a new point
-                        utility.update_params()  
-                        suggested[index] = optimizer.suggest(utility)
-                        old_kappa = kappas[index]
-                        kappas[index] = utility.kappa
+                        suggested[index] = optimizer.suggest()
+                        old_acq_params = acq_params[index]
+                        acq_params[index] = vars(optimizer.acquisition_function)
                         
                         # transform suggestion back to natural space
                         rddl_params = {
@@ -272,8 +260,8 @@ class JaxParameterTuning:
                             best_params, best_target = rddl_params, target
                         
                         # write progress to file in real time
-                        rows[index] = [pid, index, it, target, best_target, old_kappa] + \
-                                      list(rddl_params.values())
+                        info_i = [pid, index, it, target, best_target, old_acq_params]
+                        rows[index] = info_i + list(rddl_params.values())
                         
             # write results of all processes in current iteration to file
             with open(filename, 'a', newline='') as file:
@@ -308,16 +296,20 @@ class JaxParameterTuning:
             raise_warning(f'failed to import packages matplotlib or sklearn, '
                           f'aborting plot of search space\n{e}', 'red')
         else:
-            data = np.loadtxt(filename, delimiter=',', dtype=object)
-            data, target = data[1:, 3:], data[1:, 2]
-            data = data.astype(np.float64)
-            target = target.astype(np.float64)
+            with open(filename, 'r') as file:
+                data_iter = csv.reader(file, delimiter=',')
+                data = [row for row in data_iter]
+            data = np.asarray(data, dtype=object)
+            hparam = data[1:, len(COLUMNS):].astype(np.float64)
+            target = data[1:, 3].astype(np.float64)
             target = (target - np.min(target)) / (np.max(target) - np.min(target))
             embedding = MDS(n_components=2, normalized_stress='auto')
-            data1 = embedding.fit_transform(data)
-            sc = plt.scatter(data1[:, 0], data1[:, 1], c=target, s=4.,
-                             cmap='seismic', edgecolor='gray',
-                             linewidth=0.01, alpha=0.4)
+            hparam_low = embedding.fit_transform(hparam)
+            sc = plt.scatter(hparam_low[:, 0], hparam_low[:, 1], c=target, s=5,
+                             cmap='seismic', edgecolor='gray', linewidth=0)
+            ax = plt.gca()
+            for i in range(len(target)):
+                ax.annotate(str(i), (hparam_low[i, 0], hparam_low[i, 1]), fontsize=3)         
             plt.colorbar(sc)
             plt.savefig(self._filename('gp_points', 'pdf'))
             plt.clf()
@@ -342,9 +334,11 @@ def objective_slp(params, kwargs, key, index):
         std, lr, w, wa = param_values
     else:
         std, lr, w = param_values
-        wa = None                      
+        wa = None         
+    key, subkey = jax.random.split(key)             
     if kwargs['verbose']:
-        print(f'[{index}] key={key}, std={std}, lr={lr}, w={w}, wa={wa}...', flush=True)
+        print(f'[{index}] key={subkey[0]}, '
+              f'std={std}, lr={lr}, w={w}, wa={wa}...', flush=True)
         
     # initialize planning algorithm
     planner = JaxBackpropPlanner(
@@ -358,7 +352,6 @@ def objective_slp(params, kwargs, key, index):
     model_params = {name: w for name in planner.compiled.model_params}
     
     # initialize policy
-    key, subkey = jax.random.split(key)
     policy = JaxOfflineController(
         planner=planner,
         key=subkey,
@@ -384,7 +377,7 @@ def objective_slp(params, kwargs, key, index):
         key, subkey = jax.random.split(key)
         total_reward = policy.evaluate(env, seed=np.array(subkey)[0])['mean']
         if kwargs['verbose']:
-            print(f'    [{index}] trial {trial + 1} key={subkey}, '
+            print(f'    [{index}] trial {trial + 1} key={subkey[0]}, '
                   f'reward={total_reward}', flush=True)
         average_reward += total_reward / kwargs['eval_trials']        
     if kwargs['verbose']:
@@ -474,8 +467,10 @@ def objective_replan(params, kwargs, key, index):
     else:
         std, lr, w, T = param_values
         wa = None        
+    key, subkey = jax.random.split(key)
     if kwargs['verbose']:
-        print(f'[{index}] key={key}, std={std}, lr={lr}, w={w}, wa={wa}, T={T}...', flush=True)
+        print(f'[{index}] key={subkey[0]}, '
+              f'std={std}, lr={lr}, w={w}, wa={wa}, T={T}...', flush=True)
 
     # initialize planning algorithm
     planner = JaxBackpropPlanner(
@@ -490,7 +485,6 @@ def objective_replan(params, kwargs, key, index):
     model_params = {name: w for name in planner.compiled.model_params}
     
     # initialize controller
-    key, subkey = jax.random.split(key)
     policy = JaxOnlineController(
         planner=planner,
         key=subkey,
@@ -516,7 +510,7 @@ def objective_replan(params, kwargs, key, index):
         key, subkey = jax.random.split(key)
         total_reward = policy.evaluate(env, seed=np.array(subkey)[0])['mean']
         if kwargs['verbose']:
-            print(f'    [{index}] trial {trial + 1} key={subkey}, '
+            print(f'    [{index}] trial {trial + 1} key={subkey[0]}, '
                   f'reward={total_reward}', flush=True)
         average_reward += total_reward / kwargs['eval_trials']        
     if kwargs['verbose']:
@@ -602,9 +596,11 @@ def objective_drp(params, kwargs, key, index):
     ]
     
     # unpack hyper-parameters
-    lr, w, layers, neurons = param_values                      
+    lr, w, layers, neurons = param_values   
+    key, subkey = jax.random.split(key)                   
     if kwargs['verbose']:
-        print(f'[{index}] key={key}, lr={lr}, w={w}, layers={layers}, neurons={neurons}...', flush=True)
+        print(f'[{index}] key={subkey[0]}, '
+              f'lr={lr}, w={w}, layers={layers}, neurons={neurons}...', flush=True)
            
     # initialize planning algorithm
     planner = JaxBackpropPlanner(
@@ -618,7 +614,6 @@ def objective_drp(params, kwargs, key, index):
     model_params = {name: w for name in planner.compiled.model_params}
     
     # initialize policy
-    key, subkey = jax.random.split(key)
     policy = JaxOfflineController(
         planner=planner,
         key=subkey,
@@ -644,7 +639,7 @@ def objective_drp(params, kwargs, key, index):
         key, subkey = jax.random.split(key)
         total_reward = policy.evaluate(env, seed=np.array(subkey)[0])['mean']
         if kwargs['verbose']:
-            print(f'    [{index}] trial {trial + 1} key={subkey}, '
+            print(f'    [{index}] trial {trial + 1} key={subkey[0]}, '
                   f'reward={total_reward}', flush=True)
         average_reward += total_reward / kwargs['eval_trials']
     if kwargs['verbose']:

pyRDDLGym_jax/examples/configs/Cartpole_Continuous_gym_replan.cfg

@@ -16,4 +16,5 @@ rollout_horizon=30
 [Training]
 key=42
 epochs=1000
-train_seconds=1
+train_seconds=1
+print_summary=False

pyRDDLGym_jax/examples/configs/PowerGen_Continuous_replan.cfg

@@ -16,4 +16,5 @@ rollout_horizon=5
 [Training]
 key=42
 epochs=2000
-train_seconds=1
+train_seconds=1
+print_summary=False

pyRDDLGym_jax/examples/configs/Reservoir_Continuous_replan.cfg

@@ -16,4 +16,5 @@ rollout_horizon=5
 [Training]
 key=42
 epochs=500
-train_seconds=1
+train_seconds=1
+print_summary=False

pyRDDLGym_jax/examples/configs/Wildfire_MDP_ippc2014_replan.cfg

@@ -6,9 +6,9 @@ tnorm_kwargs={}
 
 [Optimizer]
 method='JaxStraightLinePlan'
-method_kwargs={'initializer': 'normal', 'initializer_kwargs': {'stddev': 0.001}}
+method_kwargs={}
 optimizer='rmsprop'
-optimizer_kwargs={'learning_rate': 0.001}
+optimizer_kwargs={'learning_rate': 0.1}
 batch_size_train=32
 batch_size_test=32
 rollout_horizon=5
@@ -17,4 +17,5 @@ rollout_horizon=5
 key=42
 epochs=1000
 train_seconds=1
-policy_hyperparams={'cut-out': 10.0, 'put-out': 10.0}
+policy_hyperparams={'cut-out': 10.0, 'put-out': 10.0}
+print_summary=False

pyRDDLGym_jax/examples/configs/default_replan.cfg

@@ -17,4 +17,5 @@ rollout_horizon=5
 key=42
 epochs=2000
 train_seconds=1
-policy_hyperparams=2.0
+policy_hyperparams=2.0
+print_summary=False

pyRDDLGym_jax/examples/run_tune.py

@@ -59,9 +59,7 @@ def main(domain, instance, method, trials=5, iters=20, workers=4):
                           gp_iters=iters)
     
     # perform tuning and report best parameters
-    best = tuning.tune(key=train_args['key'], filename=f'gp_{method}', 
-                       save_plot=True)
-    print(f'best parameters found: {best}')
+    tuning.tune(key=train_args['key'], filename=f'gp_{method}', save_plot=True)
 
 
 if __name__ == "__main__":

pyRDDLGym_jax-0.5.dist-info/METADATA

@@ -0,0 +1,278 @@
+Metadata-Version: 2.1
+Name: pyRDDLGym-jax
+Version: 0.5
+Summary: pyRDDLGym-jax: automatic differentiation for solving sequential planning problems in JAX.
+Home-page: https://github.com/pyrddlgym-project/pyRDDLGym-jax
+Author: Michael Gimelfarb, Ayal Taitler, Scott Sanner
+Author-email: mike.gimelfarb@mail.utoronto.ca, ataitler@gmail.com, ssanner@mie.utoronto.ca
+License: MIT License
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyRDDLGym >=2.0
+Requires-Dist: tqdm >=4.66
+Requires-Dist: jax >=0.4.12
+Requires-Dist: optax >=0.1.9
+Requires-Dist: dm-haiku >=0.0.10
+Requires-Dist: tensorflow-probability >=0.21.0
+Provides-Extra: extra
+Requires-Dist: bayesian-optimization >=2.0.0 ; extra == 'extra'
+Requires-Dist: rddlrepository >=2.0 ; extra == 'extra'
+
+# pyRDDLGym-jax
+
+Author: [Mike Gimelfarb](https://mike-gimelfarb.github.io)
+
+This directory provides:
+1. automated translation and compilation of RDDL description files into [JAX](https://github.com/google/jax), converting any RDDL domain to a differentiable simulator!
+2. powerful, fast and scalable gradient-based planning algorithms, with extendible and flexible policy class representations, automatic model relaxations for working in discrete and hybrid domains, and much more!
+
+> [!NOTE]  
+> While Jax planners can support some discrete state/action problems through model relaxations, on some discrete problems it can perform poorly (though there is an ongoing effort to remedy this!).
+> If you find it is not making sufficient progress, check out the [PROST planner](https://github.com/pyrddlgym-project/pyRDDLGym-prost) (for discrete spaces) or the [deep reinforcement learning wrappers](https://github.com/pyrddlgym-project/pyRDDLGym-rl).
+
+## Contents
+
+- [Installation](#installation)
+- [Running from the Command Line](#running-from-the-command-line)
+- [Running from within Python](#running-from-within-python)
+- [Configuring the Planner](#configuring-the-planner)
+- [Simulation](#simulation)
+- [Manual Gradient Calculation](#manual-gradient-calculation)
+- [Citing pyRDDLGym-jax](#citing-pyrddlgym-jax)
+  
+## Installation
+
+To use the compiler or planner without the automated hyper-parameter tuning, you will need the following packages installed: 
+- ``pyRDDLGym>=2.0``
+- ``tqdm>=4.66``
+- ``jax>=0.4.12``
+- ``optax>=0.1.9``
+- ``dm-haiku>=0.0.10``
+- ``tensorflow-probability>=0.21.0``
+
+Additionally, if you wish to run the examples, you need ``rddlrepository>=2``. 
+To run the automated tuning optimization, you will also need ``bayesian-optimization>=2.0.0``.
+
+You can install pyRDDLGym-jax with all requirements using pip:
+
+```shell
+pip install pyRDDLGym-jax[extra]
+```
+
+## Running from the Command Line
+
+A basic run script is provided to run the Jax Planner on any domain in ``rddlrepository`` from the install directory of pyRDDLGym-jax:
+
+```shell
+python -m pyRDDLGym_jax.examples.run_plan <domain> <instance> <method> <episodes>
+```
+
+where:
+- ``domain`` is the domain identifier as specified in rddlrepository (i.e. Wildfire_MDP_ippc2014), or a path pointing to a valid ``domain.rddl`` file
+- ``instance`` is the instance identifier (i.e. 1, 2, ... 10), or a path pointing to a valid ``instance.rddl`` file
+- ``method`` is the planning method to use (i.e. drp, slp, replan)
+- ``episodes`` is the (optional) number of episodes to evaluate the learned policy.
+
+The ``method`` parameter supports three possible modes:
+- ``slp`` is the basic straight line planner described [in this paper](https://proceedings.neurips.cc/paper_files/paper/2017/file/98b17f068d5d9b7668e19fb8ae470841-Paper.pdf)
+- ``drp`` is the deep reactive policy network described [in this paper](https://ojs.aaai.org/index.php/AAAI/article/view/4744)
+- ``replan`` is the same as ``slp`` except the plan is recalculated at every decision time step.
+   
+A basic run script is also provided to run the automatic hyper-parameter tuning:
+
+```shell
+python -m pyRDDLGym_jax.examples.run_tune <domain> <instance> <method> <trials> <iters> <workers>
+```
+
+where:
+- ``domain`` is the domain identifier as specified in rddlrepository
+- ``instance`` is the instance identifier
+- ``method`` is the planning method to use (i.e. drp, slp, replan)
+- ``trials`` is the (optional) number of trials/episodes to average in evaluating each hyper-parameter setting
+- ``iters`` is the (optional) maximum number of iterations/evaluations of Bayesian optimization to perform
+- ``workers`` is the (optional) number of parallel evaluations to be done at each iteration, e.g. the total evaluations = ``iters * workers``.
+
+For example, the following will train the Jax Planner on the Quadcopter domain with 4 drones:
+
+```shell
+python -m pyRDDLGym_jax.examples.run_plan Quadcopter 1 slp
+```
+
+After several minutes of optimization, you should get a visualization as follows:
+
+<p align="center">
+<img src="Images/quadcopter.gif" width="400" height="400" margin=1/>
+</p>
+
+## Running from within Python
+
+To run the Jax planner from within a Python application, refer to the following example:
+
+```python
+import pyRDDLGym
+from pyRDDLGym_jax.core.planner import JaxBackpropPlanner, JaxOfflineController
+
+# set up the environment (note the vectorized option must be True)
+env = pyRDDLGym.make("domain", "instance", vectorized=True)
+
+# create the planning algorithm
+planner = JaxBackpropPlanner(rddl=env.model, **planner_args)
+controller = JaxOfflineController(planner, **train_args)
+
+# evaluate the planner
+controller.evaluate(env, episodes=1, verbose=True, render=True)
+env.close()
+```
+
+Here, we have used the straight-line controller, although you can configure the combination of planner and policy representation if you wish. 
+All controllers are instances of pyRDDLGym's ``BaseAgent`` class, so they provide the ``evaluate()`` function to streamline interaction with the environment.
+The ``**planner_args`` and ``**train_args`` are keyword argument parameters to pass during initialization, but we strongly recommend creating and loading a config file as discussed in the next section.
+
+## Configuring the Planner
+
+The simplest way to configure the planner is to write and pass a configuration file with the necessary [hyper-parameters](https://pyrddlgym.readthedocs.io/en/latest/jax.html#configuring-pyrddlgym-jax).
+The basic structure of a configuration file is provided below for a straight-line planner:
+
+```ini
+[Model]
+logic='FuzzyLogic'
+logic_kwargs={'weight': 20}
+tnorm='ProductTNorm'
+tnorm_kwargs={}
+
+[Optimizer]
+method='JaxStraightLinePlan'
+method_kwargs={}
+optimizer='rmsprop'
+optimizer_kwargs={'learning_rate': 0.001}
+batch_size_train=1
+batch_size_test=1
+
+[Training]
+key=42
+epochs=5000
+train_seconds=30
+```
+
+The configuration file contains three sections:
+- ``[Model]`` specifies the fuzzy logic operations used to relax discrete operations to differentiable approximations; the ``weight`` dictates the quality of the approximation,
+and ``tnorm`` specifies the type of [fuzzy logic](https://en.wikipedia.org/wiki/T-norm_fuzzy_logics) for relacing logical operations in RDDL (e.g. ``ProductTNorm``, ``GodelTNorm``, ``LukasiewiczTNorm``)
+- ``[Optimizer]`` generally specify the optimizer and plan settings; the ``method`` specifies the plan/policy representation (e.g. ``JaxStraightLinePlan``, ``JaxDeepReactivePolicy``), the gradient descent settings, learning rate, batch size, etc.
+- ``[Training]`` specifies computation limits, such as total training time and number of iterations, and options for printing or visualizing information from the planner.
+
+For a policy network approach, simply change the ``[Optimizer]`` settings like so:
+
+```ini
+...
+[Optimizer]
+method='JaxDeepReactivePolicy'
+method_kwargs={'topology': [128, 64], 'activation': 'tanh'}
+...
+```
+
+The configuration file must then be passed to the planner during initialization. 
+For example, the [previous script here](#running-from-within-python) can be modified to set parameters from a config file:
+
+```python
+from pyRDDLGym_jax.core.planner import load_config
+
+# load the config file with planner settings
+planner_args, _, train_args = load_config("/path/to/config.cfg")
+    
+# create the planning algorithm
+planner = JaxBackpropPlanner(rddl=env.model, **planner_args)
+controller = JaxOfflineController(planner, **train_args)
+...
+```
+
+## Simulation
+
+The JAX compiler can be used as a backend for simulating and evaluating RDDL environments:
+
+```python
+import pyRDDLGym
+from pyRDDLGym.core.policy import RandomAgent
+from pyRDDLGym_jax.core.simulator import JaxRDDLSimulator
+
+# create the environment
+env = pyRDDLGym.make("domain", "instance", backend=JaxRDDLSimulator)
+
+# evaluate the random policy
+agent = RandomAgent(action_space=env.action_space,
+                    num_actions=env.max_allowed_actions)
+agent.evaluate(env, verbose=True, render=True)
+```
+
+For some domains, the JAX backend could perform better than the numpy-based one, due to various compiler optimizations. 
+In any event, the simulation results using the JAX backend should (almost) always match the numpy backend.
+
+## Manual Gradient Calculation
+
+For custom applications, it is desirable to compute gradients of the model that can be optimized downstream. 
+Fortunately, we provide a very convenient function for compiling the transition/step function ``P(s, a, s')`` of the environment into JAX.
+
+```python
+import pyRDDLGym
+from pyRDDLGym_jax.core.planner import JaxRDDLCompilerWithGrad
+
+# set up the environment
+env = pyRDDLGym.make("domain", "instance", vectorized=True)
+
+# create the step function
+compiled = JaxRDDLCompilerWithGrad(rddl=env.model)
+compiled.compile()
+step_fn = compiled.compile_transition()
+```
+
+This will return a JAX compiled (pure) function requiring the following inputs:
+- ``key`` is the ``jax.random.PRNGKey`` key for reproducible randomness
+- ``actions`` is the dictionary of action fluent tensors
+- ``subs`` is the dictionary of state-fluent and non-fluent tensors
+- ``model_params`` are the parameters of the differentiable relaxations, such as ``weight``
+
+The function returns a dictionary containing a variety of variables, such as updated pvariables including next-state fluents (``pvar``), reward obtained (``reward``), error codes (``error``).
+It is thus possible to apply any JAX transformation to the output of the function, such as computing gradient using ``jax.grad()`` or batched simulation using ``jax.vmap()``.
+
+Compilation of entire rollouts is also possible by calling the ``compile_rollouts`` function.
+An [example is provided to illustrate how you can define your own policy class and compute the return gradient manually](https://github.com/pyrddlgym-project/pyRDDLGym-jax/blob/main/pyRDDLGym_jax/examples/run_gradient.py).
+
+## Citing pyRDDLGym-jax
+
+The [following citation](https://ojs.aaai.org/index.php/ICAPS/article/view/31480) describes the main ideas of the framework. Please cite it if you found it useful:
+
+```
+@inproceedings{gimelfarb2024jaxplan,
+    title={JaxPlan and GurobiPlan: Optimization Baselines for Replanning in Discrete and Mixed Discrete and Continuous Probabilistic Domains},
+    author={Michael Gimelfarb and Ayal Taitler and Scott Sanner},
+    booktitle={34th International Conference on Automated Planning and Scheduling},
+    year={2024},
+    url={https://openreview.net/forum?id=7IKtmUpLEH}
+}
+```
+
+The utility optimization is discussed in [this paper](https://ojs.aaai.org/index.php/AAAI/article/view/21226):
+
+```
+@inproceedings{patton2022distributional,
+    title={A distributional framework for risk-sensitive end-to-end planning in continuous mdps},
+    author={Patton, Noah and Jeong, Jihwan and Gimelfarb, Mike and Sanner, Scott},
+    booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
+    volume={36},
+    number={9},
+    pages={9894--9901},
+    year={2022}
+}
+```
+
+Some of the implementation details derive from the following literature, which you may wish to also cite in your research papers:
+- [Deep reactive policies for planning in stochastic nonlinear domains, AAAI 2019](https://ojs.aaai.org/index.php/AAAI/article/view/4744)
+- [Scalable planning with tensorflow for hybrid nonlinear domains, NeurIPS 2017](https://proceedings.neurips.cc/paper/2017/file/98b17f068d5d9b7668e19fb8ae470841-Paper.pdf)
+