swifttd 0.1.8__tar.gz

swifttd-0.1.8/.github/workflows/wheels.yml

@@ -0,0 +1,114 @@
+name: Build & Publish Wheels
+
+on:
+  push:
+    branches: [ main, master ]
+    tags: [ "v*" ]
+  workflow_dispatch: { }
+
+jobs:
+  build_wheels:
+    name: Build wheels (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-22.04
+          - os: ubuntu-24.04
+          - os: windows-latest
+          - os: macos-13      # Intel x86_64
+          - os: macos-14      # Apple Silicon arm64
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      # Enable QEMU so cibuildwheel can build aarch64 wheels on x86_64 Linux
+      - name: Setup QEMU (for Linux aarch64)
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
+
+      - name: Install build tooling
+        run: |
+          python -m pip install --upgrade pip
+          pip install cibuildwheel==2.* twine build
+
+      - name: Build wheels with cibuildwheel
+        env:
+          # Build only x86_64 and aarch64 on Linux (drops i686)
+          CIBW_ARCHS_LINUX: "x86_64 aarch64"
+          # Optional skip list: no PyPy, no musllinux
+          CIBW_SKIP: "pp* *-musllinux_*"
+          CIBW_BUILD_VERBOSITY: "1"
+          CIBW_TEST_COMMAND: "python -c \"import swifttd; print(swifttd.SwiftTD)\""
+          CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*"
+        run: |
+          python -m cibuildwheel --output-dir dist
+
+      - name: Upload wheel artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-wheels-${{ matrix.os }}
+          path: dist/*.whl
+          if-no-files-found: error
+
+  build_sdist:
+    name: Build sdist
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build sdist
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+          python -m build --sdist
+      - name: Upload sdist artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-sdist
+          path: dist/*.tar.gz
+          if-no-files-found: error
+
+  publish:
+    name: Publish to (Test)PyPI
+    needs: [ build_wheels, build_sdist ]
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: ./artifacts
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Combine dists
+        run: |
+          mkdir -p dist
+          find artifacts -type f -name "*.whl" -exec cp {} dist/ \;
+          find artifacts -type f -name "*.tar.gz" -exec cp {} dist/ \;
+          ls -la dist
+      - name: Install twine
+        run: |
+          python -m pip install --upgrade pip
+          pip install twine
+      - name: Upload to TestPyPI on branches
+        if: startsWith(github.ref, 'refs/heads/')
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
+        run: |
+          twine upload --repository-url https://test.pypi.org/legacy/ dist/*
+      - name: Upload to PyPI on tags
+        if: startsWith(github.ref, 'refs/tags/')
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          twine upload dist/*

swifttd-0.1.8/CMakeLists.txt

@@ -0,0 +1,38 @@
+cmake_minimum_required(VERSION 3.21)
+
+project(SwiftTD LANGUAGES CXX)
+
+# Build settings
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(CMAKE_CXX_VISIBILITY_PRESET hidden)
+set(CMAKE_VISIBILITY_INLINES_HIDDEN ON)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
+
+# pybind11 + Python (Development.Module ensures proper extension build flags)
+find_package(Python COMPONENTS Interpreter Development.Module REQUIRED)
+find_package(pybind11 CONFIG REQUIRED)
+
+# C++ core library (not installed; linked into the Python extension)
+add_library(SwiftTD STATIC
+        src/cpp/SwiftTD.cpp
+)
+target_include_directories(SwiftTD PUBLIC
+        ${CMAKE_CURRENT_SOURCE_DIR}/src/cpp
+)
+
+# Python extension module (top-level: import swift_td)
+pybind11_add_module(swift_td
+        src/cpp/pybind.cpp
+)
+
+target_link_libraries(swift_td PRIVATE SwiftTD)
+
+# Let scikit-build-core place the extension correctly inside the wheel
+# (use SKBUILD_PLATLIB_DIR instead of Python_SITEARCH)
+install(TARGETS swift_td
+        LIBRARY DESTINATION "${SKBUILD_PLATLIB_DIR}"
+        ARCHIVE DESTINATION "${SKBUILD_PLATLIB_DIR}"
+        RUNTIME DESTINATION "${SKBUILD_PLATLIB_DIR}"
+)

swifttd-0.1.8/PKG-INFO

@@ -0,0 +1,93 @@
+Metadata-Version: 2.2
+Name: swifttd
+Version: 0.1.8
+Summary: SwiftTD: Fast and Robust TD Learning
+Author: Khurram Javed
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: C++
+Classifier: Operating System :: OS Independent
+Project-URL: Homepage, https://github.com/khurramjaved96/SwiftTD
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+
+# SwiftTD: A Fast and Robust Algorithm for Temporal Difference Learning
+
+SwiftTD is an algorithm for learning value functions. It combines the ideas of step-size adaptation with the idea of a bound on the rate of learning. The implementations in this repository use linear function approximation. 
+
+## Installation
+
+```bash
+pip install SwiftTD
+```
+
+## Usage
+
+After installation, you can use the three implementations of SwiftTD in Python as: 
+
+```python
+import swifttd
+
+# Version of SwiftTD that expects the full feature vector as input. This should only be used if the feature representation is not sparse. Otherwise, the sparse versions are more efficient.
+td_dense = swifttd.SwiftTDNonSparse(
+    num_features=5,     # Number of input features
+    lambda_=0.95,        # Lambda parameter for eligibility traces
+    initial_alpha=1e-2,  # Initial learning rate
+    gamma=0.99,        # Discount factor
+    eps=1e-5,          # Small constant for numerical stability
+    max_step_size=0.1, # Maximum allowed step size
+    step_size_decay=0.999, # Step size decay rate
+    meta_step_size=1e-3,  # Meta learning rate
+    eta_min=1e-10 # Minimum value of the step-size parameter
+)
+
+# Feature vector
+features = [1.0, 0.0, 0.5, 0.2, 0.0] 
+reward = 1.0
+prediction = td_dense.step(features, reward)
+print("Dense prediction:", prediction)
+
+# Version of SwiftTD that expects the feature indices as input. This version assumes that the features are binary---0 or 1. For learning, the indices of the features that are 1 are provided. 
+td_sparse = swifttd.SwiftTDBinaryFeatures(
+    num_features=1000,     # Number of input features
+    lambda_=0.95,        # Lambda parameter for eligibility traces
+    initial_alpha=1e-2,  # Initial learning rate
+    gamma=0.99,        # Discount factor
+    eps=1e-5,          # Small constant for numerical stability
+    max_step_size=0.1, # Maximum allowed step size
+    step_size_decay=0.999, # Step size decay rate
+    meta_step_size=1e-3,  # Meta learning rate
+    eta_min=1e-10 # Minimum value of the step-size parameter
+)
+
+# Specify the indices of the features that are 1.
+active_features = [1, 42, 999]  # Indices of active features
+reward = 1.0
+prediction = td_sparse.step(active_features, reward)
+print("Sparse binary prediction:", prediction)
+
+# Version of SwiftTD that expects the feature indices and values as input. This version does not assume that the features are binary. For learning, it expects a list of (index, value) pairs. Only the indices of the features that are non-zero need to be provided. 
+
+td_sparse_nonbinary = swifttd.SwiftTD(
+    num_features=1000,     # Number of input features
+    lambda_=0.95,        # Lambda parameter for eligibility traces
+    initial_alpha=1e-2,  # Initial learning rate
+    gamma=0.99,        # Discount factor
+    eps=1e-5,          # Small constant for numerical stability
+    max_step_size=0.1, # Maximum allowed step size
+    step_size_decay=0.999, # Step size decay rate
+    meta_step_size=1e-3,  # Meta learning rate
+    eta_min=1e-10 # Minimum value of the step-size parameter
+)
+
+# Specify the indices and values of the features that are non-zero.
+feature_values = [(1, 0.8), (42, 0.3), (999, 1.2)]  # (index, value) pairs
+reward = 1.0
+prediction = td_sparse_nonbinary.step(feature_values, reward)
+print("Sparse non-binary prediction:", prediction)
+```
+
+## Resources
+- [Paper (PDF)](https://khurramjaved.com/swifttd.pdf)
+- [Interactive Demo](https://khurramjaved.com/swifttd.html)
+

swifttd-0.1.8/README.md

@@ -0,0 +1,80 @@
+# SwiftTD: A Fast and Robust Algorithm for Temporal Difference Learning
+
+SwiftTD is an algorithm for learning value functions. It combines the ideas of step-size adaptation with the idea of a bound on the rate of learning. The implementations in this repository use linear function approximation. 
+
+## Installation
+
+```bash
+pip install SwiftTD
+```
+
+## Usage
+
+After installation, you can use the three implementations of SwiftTD in Python as: 
+
+```python
+import swifttd
+
+# Version of SwiftTD that expects the full feature vector as input. This should only be used if the feature representation is not sparse. Otherwise, the sparse versions are more efficient.
+td_dense = swifttd.SwiftTDNonSparse(
+    num_features=5,     # Number of input features
+    lambda_=0.95,        # Lambda parameter for eligibility traces
+    initial_alpha=1e-2,  # Initial learning rate
+    gamma=0.99,        # Discount factor
+    eps=1e-5,          # Small constant for numerical stability
+    max_step_size=0.1, # Maximum allowed step size
+    step_size_decay=0.999, # Step size decay rate
+    meta_step_size=1e-3,  # Meta learning rate
+    eta_min=1e-10 # Minimum value of the step-size parameter
+)
+
+# Feature vector
+features = [1.0, 0.0, 0.5, 0.2, 0.0] 
+reward = 1.0
+prediction = td_dense.step(features, reward)
+print("Dense prediction:", prediction)
+
+# Version of SwiftTD that expects the feature indices as input. This version assumes that the features are binary---0 or 1. For learning, the indices of the features that are 1 are provided. 
+td_sparse = swifttd.SwiftTDBinaryFeatures(
+    num_features=1000,     # Number of input features
+    lambda_=0.95,        # Lambda parameter for eligibility traces
+    initial_alpha=1e-2,  # Initial learning rate
+    gamma=0.99,        # Discount factor
+    eps=1e-5,          # Small constant for numerical stability
+    max_step_size=0.1, # Maximum allowed step size
+    step_size_decay=0.999, # Step size decay rate
+    meta_step_size=1e-3,  # Meta learning rate
+    eta_min=1e-10 # Minimum value of the step-size parameter
+)
+
+# Specify the indices of the features that are 1.
+active_features = [1, 42, 999]  # Indices of active features
+reward = 1.0
+prediction = td_sparse.step(active_features, reward)
+print("Sparse binary prediction:", prediction)
+
+# Version of SwiftTD that expects the feature indices and values as input. This version does not assume that the features are binary. For learning, it expects a list of (index, value) pairs. Only the indices of the features that are non-zero need to be provided. 
+
+td_sparse_nonbinary = swifttd.SwiftTD(
+    num_features=1000,     # Number of input features
+    lambda_=0.95,        # Lambda parameter for eligibility traces
+    initial_alpha=1e-2,  # Initial learning rate
+    gamma=0.99,        # Discount factor
+    eps=1e-5,          # Small constant for numerical stability
+    max_step_size=0.1, # Maximum allowed step size
+    step_size_decay=0.999, # Step size decay rate
+    meta_step_size=1e-3,  # Meta learning rate
+    eta_min=1e-10 # Minimum value of the step-size parameter
+)
+
+# Specify the indices and values of the features that are non-zero.
+feature_values = [(1, 0.8), (42, 0.3), (999, 1.2)]  # (index, value) pairs
+reward = 1.0
+prediction = td_sparse_nonbinary.step(feature_values, reward)
+print("Sparse non-binary prediction:", prediction)
+```
+
+## Resources
+- [Paper (PDF)](https://khurramjaved.com/swifttd.pdf)
+- [Interactive Demo](https://khurramjaved.com/swifttd.html)
+

swifttd-0.1.8/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["scikit-build-core>=0.9", "pybind11>=2.12"]
+build-backend = "scikit_build_core.build"
+
+[project]
+name = "swifttd"
+version = "0.1.8"
+description = "SwiftTD: Fast and Robust TD Learning"
+readme = "README.md"
+requires-python = ">=3.7"
+license = {text = "MIT"}
+authors = [{name = "Khurram Javed"}]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Programming Language :: C++",
+  "Operating System :: OS Independent",
+]
+
+[project.urls]
+Homepage = "https://github.com/khurramjaved96/SwiftTD"
+
+[tool.scikit-build]
+# Where the pure-Python package lives (create these files below)
+wheel.packages = ["src/swifttd"]
+
+[tool.scikit-build.cmake]
+minimum-version = "3.21"

swifttd-0.1.8/src/cpp/SwiftTD.cpp

@@ -0,0 +1,372 @@
+//
+// Created by Khurram Javed on 2024-02-18.
+//
+
+#include "SwiftTD.h"
+#include <vector>
+#include <math.h>
+
+
+SwiftTDNonSparse::SwiftTDNonSparse(int number_of_features, float lambda_init, float alpha_init, float gamma_init,
+                                   float epsilon_init, float eta_init,
+                                   float decay_init, float meta_step_size_init, float eta_min_init)
+{
+    this->gamma = gamma_init;
+    this->w = std::vector<float>(number_of_features, 0.0f);
+    this->featureVector = std::vector<float>(number_of_features, 0);
+    this->z = std::vector<float>(number_of_features, 0);
+    this->z_delta = std::vector<float>(number_of_features, 0);
+    this->delta_w = std::vector<float>(number_of_features, 0);
+
+    this->h = std::vector<float>(number_of_features, 0);
+    this->h_old = std::vector<float>(number_of_features, 0);
+    this->h_temp = std::vector<float>(number_of_features, 0);
+    this->beta = std::vector<float>(number_of_features, log(alpha_init));
+    this->z_bar = std::vector<float>(number_of_features, 0);
+    this->p = std::vector<float>(number_of_features, 0);
+
+    this->v_old = 0;
+    this->lambda = lambda_init;
+    this->epsilon = epsilon_init;
+    this->v_delta = 0;
+    this->eta = eta_init;
+    this->eta_min = eta_min_init;
+    this->decay = decay_init;
+    this->meta_step_size = meta_step_size_init;
+}
+
+float Math::DotProduct(const std::vector<float>& a, const std::vector<float>& b)
+{
+    float result = 0;
+    for (int i = 0; i < a.size(); i++)
+    {
+        result += a[i] * b[i];
+    }
+    return result;
+}
+
+float SwiftTDNonSparse::Step(const std::vector<float>& features, float reward)
+{
+    float v = 0;
+    for (int i = 0; i < features.size(); i++)
+    {
+        v += this->w[i] * features[i];
+    }
+
+    float delta = reward + gamma * v - this->v_old;
+    for (int i = 0; i < features.size(); i++)
+    {
+        this->delta_w[i] = delta * this->z[i] - z_delta[i] * this->v_delta;
+        this->w[i] += this->delta_w[i];
+        this->beta[i] +=
+            this->meta_step_size / (exp(this->beta[i])) * (delta - v_delta) * this->p[i];
+        if (exp(this->beta[i]) > this->eta || isinf(exp(this->beta[i])))
+        {
+            this->beta[i] = log(this->eta);
+        }
+        if(exp(this->beta[i]) < log(this->eta_min))
+        {
+            this->beta[i] = log(this->eta_min);
+        }
+        this->h_old[i] = this->h[i];
+        this->h[i] = this->h_temp[i] +
+            delta * this->z_bar[i] - this->z_delta[i] * this->v_delta;
+        this->h_temp[i] = this->h[i];
+        z_delta[i] = 0;
+        this->z[i] *= gamma * this->lambda;
+        this->p[i] *= gamma * this->lambda;
+        this->z_bar[i] *= gamma * this->lambda;
+    }
+    this->v_delta = 0;
+    float tau = 0;
+    for (int i = 0; i < features.size(); i++)
+    {
+        tau += exp(this->beta[i]) * features[i] * features[i];
+    }
+    float b = 0;
+    for (int i = 0; i < features.size(); i++)
+    {
+        b += this->z[i] * features[i];
+    }
+
+    for (int i = 0; i < features.size(); i++)
+    {
+        this->v_delta += this->delta_w[i] * features[i];
+        float multiplier = 1;
+        if (eta / tau < 1)
+        {
+            multiplier = eta / tau;
+        }
+        this->z_delta[i] = multiplier * exp(this->beta[i]) * features[i];
+        this->z[i] += this->z_delta[i] * (1 - b);
+        this->p[i] += this->h_old[i] * features[i];
+        this->z_bar[i] += this->z_delta[i] * (1 - b - this->z_bar[i] * features[i]);
+        this->h_temp[i] = this->h[i] - this->h_old[i] * features[i] * (this->z[i] - this->z_delta[i]) -
+            this->h[i] * this->z_delta[i] * features[i];
+        if (tau > eta)
+        {
+            this->h_temp[i] = 0;
+            this->h[i] = 0;
+            this->h_old[i] = 0;
+            this->z_bar[i] = 0;
+            this->beta[i] += log(this->decay) * features[i] * features[i];
+        }
+    }
+    this->v_old = v;
+    return v;
+}
+
+SwiftTDBinaryFeatures::SwiftTDBinaryFeatures(int number_of_features, float lambda_init, float alpha_init,
+                                             float gamma_init,
+                                             float epsilon_init, float eta_init,
+                                             float decay_init, float meta_step_size_init,  float eta_min_init)
+{
+    this->gamma = gamma_init;
+    this->w = std::vector<float>(number_of_features, 0);
+    this->featureVector = std::vector<float>(number_of_features, 0);
+    this->z = std::vector<float>(number_of_features, 0);
+    this->z_delta = std::vector<float>(number_of_features, 0);
+    this->delta_w = std::vector<float>(number_of_features, 0);
+
+    this->h = std::vector<float>(number_of_features, 0);
+    this->h_old = std::vector<float>(number_of_features, 0);
+    this->h_temp = std::vector<float>(number_of_features, 0);
+    this->beta = std::vector<float>(number_of_features, log(alpha_init));
+    this->z_bar = std::vector<float>(number_of_features, 0);
+    this->p = std::vector<float>(number_of_features, 0);
+
+    this->last_alpha = std::vector<float>(number_of_features, 0);
+
+    this->v_old = 0;
+    this->lambda = lambda_init;
+    this->epsilon = epsilon_init;
+    this->v_delta = 0;
+    this->eta = eta_init;
+    this->eta_min = eta_min_init;
+    this->decay = decay_init;
+
+    this->meta_step_size = meta_step_size_init;
+}
+
+float SwiftTDBinaryFeatures::Step(const std::vector<int>& feature_indices, float reward)
+{
+    float v = 0;
+
+    for (auto& index : feature_indices)
+    {
+        v += this->w[index];
+    }
+
+    float delta = reward + gamma * v - this->v_old;
+    int position = 0;
+    while (position < this->setOfEligibleItems.size())
+    {
+        int index = this->setOfEligibleItems[position];
+        this->delta_w[index] = delta * this->z[index] - z_delta[index] * this->v_delta;
+        this->w[index] += this->delta_w[index];
+        this->beta[index] +=
+            this->meta_step_size / (exp(this->beta[index])) * (delta - v_delta) * this->p[index];
+        if (exp(this->beta[index]) > this->eta || isinf(exp(this->beta[index])))
+        {
+            this->beta[index] = log(this->eta);
+        }
+        if(exp(this->beta[index]) < log(this->eta_min))
+        {
+            this->beta[index] = log(this->eta_min);
+        }
+        this->h_old[index] = this->h[index];
+        this->h[index] = this->h_temp[index] +
+            delta * this->z_bar[index] - this->z_delta[index] * this->v_delta;
+        this->h_temp[index] = this->h[index];
+        z_delta[index] = 0;
+        this->z[index] = gamma * this->lambda * this->z[index];
+        this->p[index] = gamma * this->lambda * this->p[index];
+        this->z_bar[index] = gamma * this->lambda * this->z_bar[index];
+        if (this->z[index] <= this->last_alpha[index] * epsilon)
+        {
+            this->z[index] = 0;
+            this->p[index] = 0;
+            this->z_bar[index] = 0;
+            this->delta_w[index] = 0;
+            this->setOfEligibleItems[position] = this->setOfEligibleItems[this->setOfEligibleItems.size() - 1];
+            this->setOfEligibleItems.pop_back();
+        }
+        else
+        {
+            position++;
+        }
+    }
+    this->v_delta = 0;
+    float rate_of_learning = 0;
+
+    for (auto& index : feature_indices)
+    {
+        rate_of_learning += exp(this->beta[index]);
+    }
+    float E = this->eta;
+    if (rate_of_learning > this->eta)
+    {
+        E = rate_of_learning;
+    }
+
+
+    float t = 0;
+    for (auto& index : feature_indices)
+    {
+        t += this->z[index];
+    }
+
+    for (auto& index : feature_indices)
+    {
+        if (z[index] == 0)
+        {
+            this->setOfEligibleItems.push_back(index);
+        }
+        this->v_delta += this->delta_w[index];
+        this->z_delta[index] = (this->eta / E) * exp(this->beta[index]);
+        this->last_alpha[index] = this->z_delta[index];
+        if ((this->eta / E) < 1)
+        {
+            this->h_temp[index] = 0;
+            this->h[index] = 0;
+            this->h_old[index] = 0;
+            this->z_bar[index] = 0;
+            this->beta[index] += log(this->decay);
+        }
+        this->z[index] += this->z_delta[index] * (1 - t);
+        this->p[index] += this->h_old[index];
+        this->z_bar[index] += this->z_delta[index] * (1 - t - this->z_bar[index]);
+        this->h_temp[index] = this->h[index] - this->h_old[index] * (this->z[index] - this->z_delta[index]) -
+            this->h[index] * this->z_delta[index];
+    }
+    this->v_old = v;
+    return v;
+}
+
+
+SwiftTD::SwiftTD(int number_of_features, float lambda_init, float alpha_init, float gamma_init,
+                 float epsilon_init, float eta_init,
+                 float decay_init, float meta_step_size_init,  float eta_min_init)
+{
+    this->gamma = gamma_init;
+    this->w = std::vector<float>(number_of_features, 0);
+    this->featureVector = std::vector<float>(number_of_features, 0);
+    this->z = std::vector<float>(number_of_features, 0);
+    this->z_delta = std::vector<float>(number_of_features, 0);
+    this->delta_w = std::vector<float>(number_of_features, 0);
+
+    this->h = std::vector<float>(number_of_features, 0);
+    this->h_old = std::vector<float>(number_of_features, 0);
+    this->h_temp = std::vector<float>(number_of_features, 0);
+    this->beta = std::vector<float>(number_of_features, log(alpha_init));
+    this->z_bar = std::vector<float>(number_of_features, 0);
+    this->p = std::vector<float>(number_of_features, 0);
+
+    this->last_alpha = std::vector<float>(number_of_features, 0);
+
+    this->v_old = 0;
+    this->lambda = lambda_init;
+    this->epsilon = epsilon_init;
+    this->v_delta = 0;
+    this->eta = eta_init;
+    this->eta_min = eta_min_init;
+    this->decay = decay_init;
+
+    this->meta_step_size = meta_step_size_init;
+}
+
+float SwiftTD::Step(const std::vector<std::pair<int, float>>& feature_indices, float reward)
+{
+    float v = 0;
+
+    for (auto& index : feature_indices)
+    {
+        v += this->w[index.first] * index.second;
+    }
+
+    float delta = reward + gamma * v - this->v_old;
+    int position = 0;
+    while (position < this->setOfEligibleItems.size())
+    {
+        auto index = this->setOfEligibleItems[position];
+        this->delta_w[index.first] = delta * this->z[index.first] - z_delta[index.first] * this->v_delta;
+        this->w[index.first] += this->delta_w[index.first];
+        this->beta[index.first] +=
+            this->meta_step_size / (exp(this->beta[index.first])) * (delta - v_delta) * this->p[index.first];
+        if (exp(this->beta[index.first]) > this->eta || isinf(exp(this->beta[index.first])))
+        {
+            this->beta[index.first] = log(this->eta);
+        }
+        if(exp(this->beta[index.first]) < log(this->eta_min))
+        {
+            this->beta[index.first] = log(this->eta_min);
+        }
+        this->h_old[index.first] = this->h[index.first];
+        this->h[index.first] = this->h_temp[index.first] +
+            delta * this->z_bar[index.first] - this->z_delta[index.first] * this->v_delta;
+        this->h_temp[index.first] = this->h[index.first];
+        z_delta[index.first] = 0;
+        this->z[index.first] = gamma * this->lambda * this->z[index.first];
+        this->p[index.first] = gamma * this->lambda * this->p[index.first];
+        this->z_bar[index.first] = gamma * this->lambda * this->z_bar[index.first];
+        if (this->z[index.first] <= this->last_alpha[index.first] * epsilon)
+        {
+            this->z[index.first] = 0;
+            this->p[index.first] = 0;
+            this->z_bar[index.first] = 0;
+            this->delta_w[index.first] = 0;
+            this->setOfEligibleItems[position] = this->setOfEligibleItems[this->setOfEligibleItems.size() - 1];
+            this->setOfEligibleItems.pop_back();
+        }
+        else
+        {
+            position++;
+        }
+    }
+    this->v_delta = 0;
+    float rate_of_learning = 0;
+
+    for (auto& index : feature_indices)
+    {
+        rate_of_learning += exp(this->beta[index.first]) * index.second * index.second;
+    }
+    float E = this->eta;
+    if (rate_of_learning > this->eta)
+    {
+        E = rate_of_learning;
+    }
+
+    float t = 0;
+    for (auto& index : feature_indices)
+    {
+        t += this->z[index.first] * index.second;
+    }
+
+    for (auto& index : feature_indices)
+    {
+        if (z[index.first] == 0)
+        {
+            this->setOfEligibleItems.push_back(index);
+        }
+        this->v_delta += this->delta_w[index.first] * index.second;
+        this->z_delta[index.first] = (this->eta / E) * exp(this->beta[index.first]) * index.second;
+        this->last_alpha[index.first] = (this->eta / E) * exp(this->beta[index.first]);
+        if ((this->eta / E) < 1)
+        {
+            this->h_temp[index.first] = 0;
+            this->h[index.first] = 0;
+            this->h_old[index.first] = 0;
+            this->z_bar[index.first] = 0;
+            this->beta[index.first] += log(this->decay) * index.second * index.second;
+        }
+        this->z[index.first] += this->z_delta[index.first] * (1 - t);
+        this->p[index.first] += this->h_old[index.first] * index.second;
+        this->z_bar[index.first] += this->z_delta[index.first] * (1 - t - this->z_bar[index.first] * index.second);
+        this->h_temp[index.first] = this->h[index.first] - this->h_old[index.first] * index.second * (this->z[index.
+                    first] - this->
+                z_delta[index.first]) -
+            this->h[index.first] * this->z_delta[index.first] * index.second;
+    }
+    this->v_old = v;
+    return v;
+}

swifttd-0.1.8/src/cpp/SwiftTD.h

@@ -0,0 +1,132 @@
+
+//
+// Created by Khurram Javed on 2024-02-18.
+//
+
+#ifndef SWIFTTD_H
+#define SWIFTTD_H
+
+#include <vector>
+
+class Math
+{
+public:
+    static float DotProduct(const std::vector<float>& a, const std::vector<float>& b);
+};
+
+class SwiftTDNonSparse
+{
+private:
+    std::vector<float> w;
+    std::vector<float> z;
+    std::vector<float> z_delta;
+    std::vector<float> delta_w;
+
+    std::vector<float> featureVector;
+
+    std::vector<float> h;
+    std::vector<float> h_old;
+    std::vector<float> h_temp;
+    std::vector<float> beta;
+    std::vector<float> z_bar;
+    std::vector<float> p;
+
+    float v_delta;
+    float lambda;
+    float epsilon;
+    float v_old;
+    float meta_step_size;
+
+    float eta;
+    float eta_min;
+
+    float decay;
+    float gamma;
+
+public:
+    SwiftTDNonSparse(int number_of_features, float lambda_init, float alpha_init, float gamma_init, float epsilon_init,
+                     float eta_init,
+                     float decay_init, float meta_step_size_init, float eta_min = 1e-10);
+    float Step(const std::vector<float>& features, float reward);
+};
+
+class SwiftTDBinaryFeatures
+{
+    std::vector<int> setOfEligibleItems; // set of eligible items
+    std::vector<float> w;
+    std::vector<float> z;
+    std::vector<float> z_delta;
+    std::vector<float> delta_w;
+
+    std::vector<float> featureVector;
+
+    std::vector<float> h;
+    std::vector<float> h_old;
+    std::vector<float> h_temp;
+    std::vector<float> beta;
+    std::vector<float> z_bar;
+    std::vector<float> p;
+
+    std::vector<float> last_alpha;
+
+
+    float v_delta;
+    float lambda;
+    float epsilon;
+    float v_old;
+    float meta_step_size;
+
+    float eta;
+    float eta_min;
+
+    float decay;
+    float gamma;
+
+public:
+    SwiftTDBinaryFeatures(int number_of_features, float lambda_init, float alpha_init, float gamma_init,
+                          float epsilon_init, float eta_init,
+                          float decay_init, float meta_step_size_init, float eta_min = 1e-10);
+    float Step(const std::vector<int>& feature_indices, float reward);
+};
+
+
+class SwiftTD
+{
+    std::vector<std::pair<int, float>> setOfEligibleItems; // set of eligible items
+    std::vector<float> w;
+    std::vector<float> z;
+    std::vector<float> z_delta;
+    std::vector<float> delta_w;
+
+    std::vector<float> featureVector;
+
+    std::vector<float> h;
+    std::vector<float> h_old;
+    std::vector<float> h_temp;
+    std::vector<float> beta;
+    std::vector<float> z_bar;
+    std::vector<float> p;
+
+    std::vector<float> last_alpha;
+
+
+    float v_delta;
+    float lambda;
+    float epsilon;
+    float v_old;
+    float meta_step_size;
+
+    float eta;
+    float eta_min;
+
+    float decay;
+    float gamma;
+
+public:
+    SwiftTD(int number_of_features, float lambda_init, float alpha_init, float gamma_init,
+                          float epsilon_init, float eta_init,
+                          float decay_init, float meta_step_size_init, float eta_min = 1e-10);
+    float Step(const std::vector<std::pair<int, float>>& feature_indices, float reward);
+};
+
+#endif // SWIFTTD_H

swifttd-0.1.8/src/cpp/pybind.cpp

@@ -0,0 +1,62 @@
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include "SwiftTD.h"
+
+namespace py = pybind11;
+
+PYBIND11_MODULE(swift_td, m)
+{
+     m.doc() = "Python bindings for the SwiftTD reinforcement learning algorithm"; // Module docstring
+     py::class_<SwiftTDNonSparse>(m, "SwiftTDNonSparse")
+          .def(py::init<int, float, float, float, float, float, float, float, float>(),
+               "Initialize the SwiftTDNonSparse algorithm",
+               py::arg("num_of_features"),
+               py::arg("lambda"),
+               py::arg("alpha"),
+               py::arg("gamma"),
+               py::arg("epsilon"),
+               py::arg("eta"),
+               py::arg("decay"),
+               py::arg("meta_step_size"),
+               py::arg("eta_min"))
+          .def("step", &SwiftTDNonSparse::Step,
+               "Perform one step of learning",
+               py::arg("features"),
+               py::arg("reward"));
+
+     // Bind SwiftTDSparse class
+     py::class_<SwiftTDBinaryFeatures>(m, "SwiftTDBinaryFeatures")
+          .def(py::init<int, float, float, float, float, float, float, float, float>(),
+               "Initialize the SwiftTDBinaryFeatures algorithm",
+               py::arg("num_of_features"),
+               py::arg("lambda"),
+               py::arg("alpha"),
+               py::arg("gamma"),
+               py::arg("epsilon"),
+               py::arg("eta"),
+               py::arg("decay"),
+               py::arg("meta_step_size"),
+               py::arg("eta_min"))
+          .def("step", &SwiftTDBinaryFeatures::Step,
+               "Perform one step of learning with sparse features",
+               py::arg("features_indices"),
+               py::arg("reward"));
+
+     // Bind SwiftTDSparseAndNonBinaryFeatures class
+     py::class_<SwiftTD>(m, "SwiftTD")
+          .def(py::init<int, float, float, float, float, float, float, float, float>(),
+               "Initialize the SwiftTD algorithm",
+               py::arg("num_of_features"),
+               py::arg("lambda"),
+               py::arg("alpha"),
+               py::arg("gamma"),
+               py::arg("epsilon"),
+               py::arg("eta"),
+               py::arg("decay"),
+               py::arg("meta_step_size"),
+               py::arg("eta_min"))
+          .def("step", &SwiftTD::Step,
+               "Perform one step of learning with sparse non-binary features",
+               py::arg("feature_indices_values"),
+               py::arg("reward"));
+}

swifttd-0.1.8/src/swifttd/__init__.py

@@ -0,0 +1,5 @@
+from ._version import __version__
+# Import C++ bindings module built by pybind11_add_module (swift_td)
+from swift_td import SwiftTDNonSparse, SwiftTDBinaryFeatures, SwiftTD
+
+__all__ = ["SwiftTDNonSparse", "SwiftTDBinaryFeatures", "SwiftTD", "__version__"]

swifttd-0.1.8/src/swifttd/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.8"