symbiotic-learning 0.2.1__tar.gz

symbiotic_learning-0.2.1/.gitignore

@@ -0,0 +1,218 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+# Temporary file for partial code execution
+tempCodeRunnerFile.py
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml

symbiotic_learning-0.2.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 bsimon717
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

symbiotic_learning-0.2.1/PKG-INFO

@@ -0,0 +1,138 @@
+Metadata-Version: 2.5
+Name: symbiotic_learning
+Version: 0.2.1
+Summary: A tool for symbiotically training multiple ML models.
+Project-URL: Repository, https://github.com/bsimon717/symlearn
+Author-email: "Benjamin D. Simon" <bsimon71701@gmail.com>
+License-File: LICENSE
+Keywords: ensemble,machine learning,symbiotic
+Requires-Python: >=3.11
+Requires-Dist: matplotlib
+Requires-Dist: numpy
+Requires-Dist: scikit-learn
+Requires-Dist: torch>=2.9.0
+Requires-Dist: tqdm
+Description-Content-Type: text/markdown
+
+# Symbiotic-Learning
+
+The *symbiotic_learning* package functions as an implementation of "symbiotic learning": a paradigm for simultaneously training multiple machine-learning models at once, in which collaboration between models is **intrinsic** and **incentivized**. The goal for this method is to effectively leverage the collaboration of relatively small models to achieve performance comparable to that of large, computationally expensive models. 
+
+A key feature of this method is a minimally-sized attention block (termed the Readout) whose task is to aggregate the perspectives and decisions of the symbiotically trained upstream (pre-Readout) models, giving a final prediction. This attention block applies a linear transformation to the input logits before performing (multi-head) scaled dot-product attention. The output of this is then concatenated with a linear transformation of the input embeddings and passed through a user-specified number of fully connected layers, yielding the final prediction.
+
+The appending of this block to the overall system occurs after a pre-determined number of training epochs, this event being termed *uplift*. As such, the framework is separated into two phases: pre-uplift and post-uplift.
+
+A system of symbiotically trained ML models with a Readout block is termed a *Symbiotic Uplift Network*.
+
+---
+
+## Usage
+
+Currently, *symbiotic_learning* is only implemented for classification tasks.
+
+To install this package, run the following command:
+
+`pip install symbiotic-learning`
+
+To use this package, first include the following imports in your training script:
+
+```
+from symbiotic_learning.classify.readout import Readout
+import symbiotic_learning.classify.utils as classify
+```
+
+Then, include a block with a structure similar to the following:
+
+```
+save_path = ## Path to sym_logs folder ##
+save_end = ## Boolean for saving models at the end of training ##
+save_best = ## Boolean for saving models at epoch of highest validation accuracy ##
+
+data_loaders = [train_loader, valid_loader, test_loader]
+
+num_classes = ## Task Specific ##
+
+preR_dim = ## Dimension of pre-Readout embeddings ##
+
+readout_hidden_dim = ## Dimension of fully-connected hidden layers ##
+readout_num_hidden = ## Number of fully-connected hidden layers ##
+num_heads = ## Number of attention heads ##
+
+collab_params = [## List of collaboration parameters ##]
+temp = ## Temperature hyperparameter in Readout loss ##
+lamb = ## Responsibility hyperparameter ##
+eps = 1e-7 ## Small value to avoid divide-by-zero errors ##
+
+models = []
+opts = []
+scheds = []
+num_preR = 3
+
+for _ in range(num_preR):
+    models.append( ## Base Model Here ## )
+    opts.append( ## Optimizer Here ## )
+    scheds.append( ## LR Scheduler ## )
+
+readout = Readout(hidden_dim=readout_hidden_dim, num_hidden=readout_num_hidden, num_classes=num_classes, num_heads=num_heads, num_preR=num_preR, preR_dim=preR_dim)
+
+classify.train(epochs, models, opts, scheds, data_loaders, collab_params, temp, criterion, uplift=uplift, eps=eps, lamb=lamb, save_path=save_path, save_end=save_end, save_best=save_best)
+
+```
+
+---
+
+## Training
+$N$ pre-Readout models are initialized for the primary task, each having an "embedding block" and a "decision block": 
+
+- The exact architecture of the embedding block is task-dependent; for an image-classification task, for example, the embedding block could consist of convolutional layers. 
+
+- The only requirement of the decision block is that it must receive the concatenation of all $N$ embeddings as input to yield a task-specific prediction.
+
+
+### Pre-Uplift
+1. Each pre-Readout model performs its initial assessment of the input data using its embedding block.
+2. The $N$ embeddings are concatenated and used as input to each of the models' decision blocks, resulting in $N$ predictions.
+3. A pre-Readout model's total (symbiotic) loss is calculated using its own output as well as the outputs of its peers, with an additional term calculated from their initial embeddings to encourage diversity of perspectives. The weighting of each of these terms is determined by that model's *collaboration parameter*.
+
+### Post-Uplift
+1. Each pre-Readout model performs its initial assessment of the input data using its embedding block.
+2. The $N$ embeddings are concatenated and used as input to each of the models' decision blocks, resulting in $N$ predictions.
+3. The $N$ predictions are concatenated and passed to the Readout's attention layer. Additonally, the vector of pre-Readout embeddings is passed through a single fully-connected layer and concatenated with the attention layer's output. This vector is then passed through fully-connected layers, resulting in the final prediction.
+4. The Readout is then penalized on how strong its own prediction was compared to the strength of the pre-Readout predictions via a non-linearity.
+5. Each pre-Readout model's symbiotic loss then has a term added to it capturing that model's culpability for the Readout's mistakes. This term is called the model's "blame loss" and is scaled using a global hyperparameter (termed "responsibility").
+
+---
+
+## Definitions
+- Symbiotic Uplift Network: An aggregate network of machine-learning models trained using symbiotic learning.
+- Symbiotic Loss ($L_{sym,i}$): A pre-Readout model's multi-objective loss function. Collaboration parameters enable coupling of models' loss functions such that 1) an individual model's parameters will also be updated based on the other models' personal losses, and 2) diversity of perspective is encouraged via Embedding Loss. 
+
+$$ L_{sym,i} = (1-\alpha_i)L_i + \alpha_i(\sum_{j \neq i}{L_j}) + \alpha_{i}^{2}L_{embed,i} $$
+
+ (Note: The only learnable parameters affected by this coupling are those used in the initial embedding blocks.)
+ 
+- Collaboration Parameters ($\alpha_i$): Coupling constants (hyperparameters) in the symbiotic loss functions of pre-Readout models. Must be in the range $[0,1]$.
+- Personal Loss ($L_i$): A term in a pre-Readout model's symbiotic loss computed using only that model's prediction. Task-specific.
+- Embedding Loss ($L_{embed,i}$): A contrastive term in a pre-Readout model's symbiotic loss which encourages diverse initial assessments. *EmbedSim* is defined to be the cosine similarity function scaled to the range $[0,1]$, and $\delta$ is a temperature hyperparameter shared between all pre-Readout models.
+  
+$$ L_{embed,i} = \frac{1}{N-1}\sum_{j \neq i}[\exp{(EmbedSim(x_i, x_j)/\delta)-1}] $$
+
+- Blame Loss ($L_{blame, i}$): A term added to a pre-Readout model's symbiotic loss after uplift, capturing that model's contribution to the Readout's loss. $\lambda$ is termed a "responsibility" hyperparameter shared between all pre-Readout models
+
+$$ L_{blame, i} = \lambda(\frac{L_i}{\sum L_i})*L_F $$
+
+- Readout Loss ($L_{Readout}$): A loss function specific to the Readout block which penalizes it the lower the sum of pre-Readout personal losses is, where $L_F$ is its personal loss, and $\tau$ is a temperature hyperparameter.
+
+$$ L_{Readout} = L_F (1+\exp[-\tau(\sum L_i)]) $$
+
+## Example Symbiotic Uplift Network Architecture
+
+![Symbiotic Uplift Network Architecture](symlearn_arch.png)
+
+This figure shows the architecture for a Symbiotic Uplift Network with three pre-Readout models.
+
+## Readout Architecture
+
+![Readout Architecture](readout_arch.png)
+
+This figure shows the architecture of the Readout block. The attention mechanism used is (multi-head) scaled dot-product attention.

symbiotic_learning-0.2.1/README.md

@@ -0,0 +1,122 @@
+# Symbiotic-Learning
+
+The *symbiotic_learning* package functions as an implementation of "symbiotic learning": a paradigm for simultaneously training multiple machine-learning models at once, in which collaboration between models is **intrinsic** and **incentivized**. The goal for this method is to effectively leverage the collaboration of relatively small models to achieve performance comparable to that of large, computationally expensive models. 
+
+A key feature of this method is a minimally-sized attention block (termed the Readout) whose task is to aggregate the perspectives and decisions of the symbiotically trained upstream (pre-Readout) models, giving a final prediction. This attention block applies a linear transformation to the input logits before performing (multi-head) scaled dot-product attention. The output of this is then concatenated with a linear transformation of the input embeddings and passed through a user-specified number of fully connected layers, yielding the final prediction.
+
+The appending of this block to the overall system occurs after a pre-determined number of training epochs, this event being termed *uplift*. As such, the framework is separated into two phases: pre-uplift and post-uplift.
+
+A system of symbiotically trained ML models with a Readout block is termed a *Symbiotic Uplift Network*.
+
+---
+
+## Usage
+
+Currently, *symbiotic_learning* is only implemented for classification tasks.
+
+To install this package, run the following command:
+
+`pip install symbiotic-learning`
+
+To use this package, first include the following imports in your training script:
+
+```
+from symbiotic_learning.classify.readout import Readout
+import symbiotic_learning.classify.utils as classify
+```
+
+Then, include a block with a structure similar to the following:
+
+```
+save_path = ## Path to sym_logs folder ##
+save_end = ## Boolean for saving models at the end of training ##
+save_best = ## Boolean for saving models at epoch of highest validation accuracy ##
+
+data_loaders = [train_loader, valid_loader, test_loader]
+
+num_classes = ## Task Specific ##
+
+preR_dim = ## Dimension of pre-Readout embeddings ##
+
+readout_hidden_dim = ## Dimension of fully-connected hidden layers ##
+readout_num_hidden = ## Number of fully-connected hidden layers ##
+num_heads = ## Number of attention heads ##
+
+collab_params = [## List of collaboration parameters ##]
+temp = ## Temperature hyperparameter in Readout loss ##
+lamb = ## Responsibility hyperparameter ##
+eps = 1e-7 ## Small value to avoid divide-by-zero errors ##
+
+models = []
+opts = []
+scheds = []
+num_preR = 3
+
+for _ in range(num_preR):
+    models.append( ## Base Model Here ## )
+    opts.append( ## Optimizer Here ## )
+    scheds.append( ## LR Scheduler ## )
+
+readout = Readout(hidden_dim=readout_hidden_dim, num_hidden=readout_num_hidden, num_classes=num_classes, num_heads=num_heads, num_preR=num_preR, preR_dim=preR_dim)
+
+classify.train(epochs, models, opts, scheds, data_loaders, collab_params, temp, criterion, uplift=uplift, eps=eps, lamb=lamb, save_path=save_path, save_end=save_end, save_best=save_best)
+
+```
+
+---
+
+## Training
+$N$ pre-Readout models are initialized for the primary task, each having an "embedding block" and a "decision block": 
+
+- The exact architecture of the embedding block is task-dependent; for an image-classification task, for example, the embedding block could consist of convolutional layers. 
+
+- The only requirement of the decision block is that it must receive the concatenation of all $N$ embeddings as input to yield a task-specific prediction.
+
+
+### Pre-Uplift
+1. Each pre-Readout model performs its initial assessment of the input data using its embedding block.
+2. The $N$ embeddings are concatenated and used as input to each of the models' decision blocks, resulting in $N$ predictions.
+3. A pre-Readout model's total (symbiotic) loss is calculated using its own output as well as the outputs of its peers, with an additional term calculated from their initial embeddings to encourage diversity of perspectives. The weighting of each of these terms is determined by that model's *collaboration parameter*.
+
+### Post-Uplift
+1. Each pre-Readout model performs its initial assessment of the input data using its embedding block.
+2. The $N$ embeddings are concatenated and used as input to each of the models' decision blocks, resulting in $N$ predictions.
+3. The $N$ predictions are concatenated and passed to the Readout's attention layer. Additonally, the vector of pre-Readout embeddings is passed through a single fully-connected layer and concatenated with the attention layer's output. This vector is then passed through fully-connected layers, resulting in the final prediction.
+4. The Readout is then penalized on how strong its own prediction was compared to the strength of the pre-Readout predictions via a non-linearity.
+5. Each pre-Readout model's symbiotic loss then has a term added to it capturing that model's culpability for the Readout's mistakes. This term is called the model's "blame loss" and is scaled using a global hyperparameter (termed "responsibility").
+
+---
+
+## Definitions
+- Symbiotic Uplift Network: An aggregate network of machine-learning models trained using symbiotic learning.
+- Symbiotic Loss ($L_{sym,i}$): A pre-Readout model's multi-objective loss function. Collaboration parameters enable coupling of models' loss functions such that 1) an individual model's parameters will also be updated based on the other models' personal losses, and 2) diversity of perspective is encouraged via Embedding Loss. 
+
+$$ L_{sym,i} = (1-\alpha_i)L_i + \alpha_i(\sum_{j \neq i}{L_j}) + \alpha_{i}^{2}L_{embed,i} $$
+
+ (Note: The only learnable parameters affected by this coupling are those used in the initial embedding blocks.)
+ 
+- Collaboration Parameters ($\alpha_i$): Coupling constants (hyperparameters) in the symbiotic loss functions of pre-Readout models. Must be in the range $[0,1]$.
+- Personal Loss ($L_i$): A term in a pre-Readout model's symbiotic loss computed using only that model's prediction. Task-specific.
+- Embedding Loss ($L_{embed,i}$): A contrastive term in a pre-Readout model's symbiotic loss which encourages diverse initial assessments. *EmbedSim* is defined to be the cosine similarity function scaled to the range $[0,1]$, and $\delta$ is a temperature hyperparameter shared between all pre-Readout models.
+  
+$$ L_{embed,i} = \frac{1}{N-1}\sum_{j \neq i}[\exp{(EmbedSim(x_i, x_j)/\delta)-1}] $$
+
+- Blame Loss ($L_{blame, i}$): A term added to a pre-Readout model's symbiotic loss after uplift, capturing that model's contribution to the Readout's loss. $\lambda$ is termed a "responsibility" hyperparameter shared between all pre-Readout models
+
+$$ L_{blame, i} = \lambda(\frac{L_i}{\sum L_i})*L_F $$
+
+- Readout Loss ($L_{Readout}$): A loss function specific to the Readout block which penalizes it the lower the sum of pre-Readout personal losses is, where $L_F$ is its personal loss, and $\tau$ is a temperature hyperparameter.
+
+$$ L_{Readout} = L_F (1+\exp[-\tau(\sum L_i)]) $$
+
+## Example Symbiotic Uplift Network Architecture
+
+![Symbiotic Uplift Network Architecture](symlearn_arch.png)
+
+This figure shows the architecture for a Symbiotic Uplift Network with three pre-Readout models.
+
+## Readout Architecture
+
+![Readout Architecture](readout_arch.png)
+
+This figure shows the architecture of the Readout block. The attention mechanism used is (multi-head) scaled dot-product attention.

symbiotic_learning-0.2.1/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "symbiotic_learning"
+version = "0.2.1"
+requires-python = ">= 3.11"
+authors = [{name = "Benjamin D. Simon", email="bsimon71701@gmail.com"}]
+description = "A tool for symbiotically training multiple ML models."
+readme = "README.md"
+keywords = ["machine learning", "ensemble", "symbiotic"]
+dependencies = [
+    "torch>=2.9.0",
+    "numpy",
+    "matplotlib",
+    "scikit_learn",
+    "tqdm"
+]
+
+[project.urls]
+Repository = "https://github.com/bsimon717/symlearn"

symbiotic_learning-0.2.1/requirements.txt

@@ -0,0 +1,70 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --no-index _symlearn/pyproject.toml
+#
+cloudpickle==3.1.2
+    # via joblib
+colorama==0.4.6
+    # via tqdm
+contourpy==1.3.3
+    # via matplotlib
+cycler==0.12.1
+    # via matplotlib
+filelock==4.0.5
+    # via torch
+fonttools==4.66.0
+    # via matplotlib
+fsspec==2026.9.0
+    # via torch
+jinja2==3.1.6
+    # via torch
+joblib==1.6.0
+    # via scikit-learn
+kiwisolver==1.5.1
+    # via matplotlib
+markupsafe==3.0.3
+    # via jinja2
+matplotlib==3.11.2
+    # via symlearn (_symlearn/pyproject.toml)
+mpmath==1.3.0
+    # via sympy
+narwhals==2.26.0
+    # via scikit-learn
+networkx==3.6.1
+    # via torch
+numpy==2.4.6
+    # via
+    #   contourpy
+    #   matplotlib
+    #   scikit-learn
+    #   scipy
+    #   symlearn (_symlearn/pyproject.toml)
+packaging==26.3
+    # via matplotlib
+pillow==12.3.0
+    # via matplotlib
+pyparsing==3.3.3
+    # via matplotlib
+python-dateutil==2.9.0.post0
+    # via matplotlib
+scikit-learn==1.9.1
+    # via symlearn (_symlearn/pyproject.toml)
+scipy==1.17.1
+    # via scikit-learn
+six==1.17.0
+    # via python-dateutil
+sympy==1.14.0
+    # via torch
+threadpoolctl==3.7.0
+    # via scikit-learn
+torch==2.14.0
+    # via symlearn (_symlearn/pyproject.toml)
+tqdm==4.70.0
+    # via symlearn (_symlearn/pyproject.toml)
+typing-extensions==4.16.0
+    # via torch
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

symbiotic_learning-0.2.1/src/symbiotic_learning/__init__.py

@@ -0,0 +1,4 @@
+__all__ = ['loss', 'classify']
+
+from . import classify
+from . import loss

symbiotic_learning-0.2.1/src/symbiotic_learning/classify/readout.py

@@ -0,0 +1,111 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch import Tensor
+
+class Readout(nn.Module):
+    """
+    Attention block intended to aggregate the decisions of pre-Readout models.
+    """
+
+    def __init__(
+            self, 
+            hidden_dim: int = 32,
+            preR_dim: int = 32,
+            num_hidden: int = 1,
+            num_classes: int = 4, 
+            num_heads: int = 1, 
+            num_preR: int = 3, 
+            attn_dropout: float = 0.0,) -> None:
+        
+        super(Readout, self).__init__()
+
+        self.hidden_dim = hidden_dim
+        self.preR_dim = preR_dim
+        self.num_hidden = num_hidden
+        self.num_classes = num_classes
+        self.num_heads = num_heads
+        self.num_preR = num_preR
+        self.attn_dropout = attn_dropout
+        
+        if self.num_heads > 1:
+            self.multi_head = True
+        else:
+            self.multi_head = False
+        
+        self.batch_norm = nn.BatchNorm1d(self.preR_dim*self.num_preR, affine=False)  
+        self.fc_embeds = nn.Linear(self.preR_dim*self.num_preR, self.hidden_dim)
+        nn.init.kaiming_normal_(self.fc_embeds.weight, nonlinearity='linear')
+        nn.init.zeros_(self.fc_embeds.bias)
+
+        if self.num_hidden == 1:
+            layer = nn.Linear(self.num_classes*self.num_preR + self.hidden_dim, self.hidden_dim)
+            nn.init.kaiming_normal_(layer.weight, nonlinearity='leaky_relu')
+            nn.init.zeros_(layer.bias)
+        
+            self.linears = nn.ModuleList([layer])
+            
+        else:
+            first_layer = nn.Linear(self.num_classes*self.num_preR + self.hidden_dim, self.hidden_dim)
+            nn.init.kaiming_normal_(first_layer.weight, nonlinearity='leaky_relu')
+            nn.init.zeros_(first_layer.bias)
+            
+            self.linears = nn.ModuleList([first_layer])
+
+            for _ in range(self.num_hidden-1):
+                hidden_layer = nn.Linear(self.hidden_dim, self.hidden_dim)
+                nn.init.kaiming_normal_(hidden_layer.weight, nonlinearity='leaky_relu')
+                nn.init.zeros_(hidden_layer.bias)
+            
+                self.linears.append(hidden_layer)
+
+        self.out = nn.Linear(self.hidden_dim, self.num_classes)
+        nn.init.xavier_uniform_(self.out.weight)
+        nn.init.zeros_(self.out.bias)
+
+        if self.multi_head:
+            self.attn_embed = nn.Linear(self.num_classes*self.num_preR, self.hidden_dim*self.num_heads)
+            
+            self.multihead_attn = nn.MultiheadAttention(
+                self.num_heads*self.hidden_dim, 
+                self.num_heads, 
+                dropout=self.attn_dropout, 
+                batch_first=True
+                )
+            
+            self.attn_out = nn.Linear(self.num_heads*self.hidden_dim, self.num_classes*self.num_preR)
+        else:
+            self.attn_embed = nn.Linear(self.num_classes*self.num_preR, self.hidden_dim)
+            self.attn_out = nn.Linear(self.hidden_dim, self.num_classes*self.num_preR)
+            
+    def input(self, 
+              ind_embeds: Tensor) -> Tensor:
+        
+        embed = self.batch_norm(ind_embeds)
+        embed = self.fc_embeds(embed)
+        return embed
+        
+    def forward(self, 
+                logits: Tensor, 
+                ind_embeds: Tensor) -> Tensor:
+        
+        logits = self.attn_embed(logits)
+        q = logits
+        k = logits
+        v = logits
+
+        if not self.multi_head:
+            logits = F.scaled_dot_product_attention(q, k, v, dropout_p=self.attn_dropout)
+        else:
+            logits, _ = self.multihead_attn(q, k, v, need_weights=False)
+
+        logits = F.tanh(self.attn_out(logits))
+        embed = F.tanh(self.input(ind_embeds))
+        
+        x = torch.cat([logits,embed],dim=1)
+
+        for layer in self.linears:
+            x = layer(x)
+            x = F.leaky_relu(x)
+        
+        return self.out(x)

symbiotic_learning-0.2.1/src/symbiotic_learning/classify/utils.py

@@ -0,0 +1,562 @@
+import os
+from typing import List
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+from torch.nn import Module
+from torch.optim import Optimizer
+from torch.optim.lr_scheduler import LRScheduler
+from torch.utils.data import DataLoader
+
+from tqdm import tqdm
+from sklearn.metrics import accuracy_score
+from datetime import datetime
+import math
+import matplotlib.pyplot as plt
+
+from symbiotic_learning.loss import embed_sim, embed_summand, embed_loss
+
+def reports_summary(reports: List[dict], epoch: int, tags: List[str]) -> None:
+    """
+    Prints the contents of a list of training/validation/testing reports with labels. 
+    """
+
+    label_lookup = {}
+    label_lookup['personal'] = 'Average Personal Loss'
+    label_lookup['symbiotic'] = 'Average Symbiotic Loss'
+    label_lookup['accuracy'] = 'Accuracy'
+    label_lookup['embedding'] = 'Average Embedding Loss'
+    label_lookup['blame'] = 'Average Blame Loss'
+    
+    print(f'Summary:')
+    for i, report in enumerate(reports):
+        tag = tags[i]
+
+        print(f'\t- {tag}:')
+        for label in report.keys():
+            if report[label] == None:
+                continue
+            else:
+                if label != 'accuracy':
+                    print(f'\t\t-- {label_lookup[label]}: {report[label]:.4}')
+                else:
+                    print()
+                    print(f'\t\t-- {label_lookup[label]}: {report[label]:.4}')
+        print()
+
+    return
+
+def fill_lt_reports(lt_reports: List[dict], reports: List[dict], phase: str) -> None:
+    """
+    Initializes or appends report data to lifetime reports.
+    """
+    
+    for lt_report, report in zip(lt_reports, reports):
+            keys = list(report.keys())
+            for key in keys:
+                if key not in lt_report[phase].keys():
+                    lt_report[phase][key] = [report[key]]
+                else:
+                    lt_report[phase][key].append(report[key])
+                    
+    return
+    
+def train_one_epoch(
+        train_loader: DataLoader, 
+        models: List[Module], 
+        opts: List[Optimizer], 
+        scheds: List[LRScheduler], 
+        collab_params: List[float], 
+        temp: float,
+        epoch: int, 
+        criterion: Module, 
+        uplift: int = 10, 
+        eps: float = 1e-7, 
+        lamb: float = 1.0) -> List[dict]:
+
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+    reports = [{} for model in models]
+    personals = [[] for model in models]
+    embs = [[] for model in models[:-1]]
+    blames = [[] for model in models[:-1]]
+    syms = [[] for model in models]
+    
+    for model in models:
+        model.train().to(device)
+
+    ## TRAINING LOOP
+    pbar_train = tqdm(train_loader, total=len(train_loader))
+    pbar_train.set_description(f'Epoch {epoch}: Training')
+    for img, label in pbar_train:
+        for opt in opts:
+            opt.zero_grad()
+
+        img = img.to(device)
+
+        inds = [model.embed(img) for model in models[:-1]]
+        
+        ind_concat = torch.cat(inds, dim=1)
+        ind_stack = torch.stack(inds)
+    
+        logits_list = [model(ind_concat.clone()).clone() for model in models[:-1]]
+        
+        L_is = []
+        L_emb_is = []
+        for i, personal in enumerate(personals[:-1]):
+            logits = logits_list[i].clone().cpu()
+            L_i = criterion(logits, label)
+            L_is.append(L_i)
+            personal.append(float(L_i.clone().detach()))
+
+            src = ind_stack[i].clone()
+            auxs = [ind_stack[i].clone() for i in range(len(ind_stack))]
+            auxs.pop(i)
+            L_emb_i = embed_loss(src, auxs).cpu()
+            L_emb_is.append(L_emb_i)
+            embs[i].append(float(L_emb_i.clone().detach()))
+        
+        L_i_tensor = torch.stack(L_is)
+        L_emb_tensor = torch.stack(L_emb_is)
+        
+        L_syms = []
+        for i in range(len(L_is)):
+            param = collab_params[i]
+            aux_idxs = [j!=i for j in range(len(L_is))]
+            aux_L = L_i_tensor.clone()[aux_idxs]
+
+            L_sym_i = (1-param)*L_i_tensor[i] + param*torch.sum(aux_L) + (param**2)*L_emb_tensor[i]
+
+            L_syms.append(L_sym_i)
+
+        if epoch >= uplift:
+            upstream_input = torch.cat(logits_list, dim=1).to(device)
+
+            final_logits = models[-1](upstream_input, ind_concat.clone()).to('cpu')
+
+            L_F = criterion(final_logits, label)
+            personals[-1].append(float(L_F.clone().detach()))
+
+            L_up_sum = eps + torch.sum(L_i_tensor).detach()
+
+            L_sym_F = L_F*(1+torch.exp(eps-temp*L_up_sum))
+            L_syms.append(L_sym_F)
+
+            for i, L_i in enumerate(L_i_tensor.clone().detach()):
+                L_blame_i = lamb*(L_i/L_up_sum)*L_F.clone()
+                blames[i].append(float(L_blame_i.clone().detach()))
+                L_syms[i] = L_syms[i] + L_blame_i
+
+        for i, L_sym_i in enumerate(L_syms):
+            syms[i].append(float(L_sym_i.clone().detach()))
+            if i != len(L_syms):
+                L_sym_i.backward(retain_graph=True)
+            else:
+                L_sym_i.backward()
+            
+        for i in range(len(syms)-1):
+            opts[i].step()
+            scheds[i].step()
+
+        if epoch >= uplift:
+            opts[-1].step()
+            scheds[-1].step()
+        
+    ## FILL REPORTS
+    for i, report in enumerate(reports):
+        
+        if i != len(reports)-1:
+            report['personal'] = np.mean(personals[i])
+            report['embedding']= np.mean(embs[i])
+            if epoch >= uplift:
+                report['blame'] = np.mean(blames[i])
+            else:
+                report['blame'] = None
+                
+            report['symbiotic'] = np.mean(syms[i])
+        else:
+            if epoch >= uplift:
+                report['personal'] = np.mean(personals[i])
+                report['symbiotic'] = np.mean(syms[i])
+            else:
+                report['personal'] = None
+                report['symbiotic'] = None
+    
+    return reports
+            
+def eval_one_epoch(
+        eval_loader: DataLoader, 
+        models: List[Module], 
+        collab_params: List[float], 
+        temp: float, 
+        epoch: int, 
+        criterion: Module, 
+        uplift: int = 10, 
+        eps: float = 1e-7, 
+        lamb: float = 1.0, 
+        phase: str = 'Validation') -> List[dict]:
+
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+    reports = [{} for model in models]
+    personals = [[] for model in models]
+    embs = [[] for model in models[:-1]]
+    blames = [[] for model in models[:-1]]
+    syms = [[] for model in models]
+    preds = [[] for model in models]
+    
+    for model in models:
+        model.eval()
+        model.to(device)
+        
+    all_labels = []
+    
+    ## VALIDATION LOOP
+    pbar_eval = tqdm(eval_loader, total=len(eval_loader))
+    pbar_eval.set_description(f'Epoch {epoch}: {phase}')
+    with torch.no_grad():
+        for img, label in pbar_eval:
+    
+            img = img.to(device)
+            all_labels += label.tolist()
+
+            inds = [model.embed(img) for model in models[:-1]]
+            
+            ind_concat = torch.cat(inds, dim=1)
+            ind_stack = torch.stack(inds)
+            
+            logits_list = [model(ind_concat.clone()).clone() for model in models[:-1]]
+            
+            probs_list = [F.softmax(logits.clone(),dim=-1).to('cpu') for logits in logits_list]
+            preds_list = [torch.argmax(probs.clone(),dim=1).tolist() for probs in probs_list]
+            
+            L_is = []
+            L_emb_is = []
+            for i, personal in enumerate(personals[:-1]):
+                logits = logits_list[i].clone().cpu()
+                L_i = criterion(logits, label)
+                L_is.append(L_i)
+                personal.append(float(L_i.clone()))
+
+                preds[i] += preds_list[i]
+                
+                src = ind_stack[i].clone()
+                auxs = [ind_stack[i].clone() for i in range(len(ind_stack))]
+                auxs.pop(i)
+                L_emb_i = embed_loss(src, auxs).cpu()
+                L_emb_is.append(L_emb_i)
+                embs[i].append(float(L_emb_i.clone()))
+            
+            L_i_tensor = torch.stack(L_is)
+            L_emb_tensor = torch.stack(L_emb_is)
+            
+            L_syms = []
+            for i in range(len(L_is)):
+                param = collab_params[i]
+                aux_idxs = [j!=i for j in range(len(L_is))]
+                aux_L = L_i_tensor.clone()[aux_idxs]
+    
+                L_sym_i = (1-param)*L_i_tensor[i] + param*torch.sum(aux_L) + (param**2)*L_emb_tensor[i]
+    
+                L_syms.append(L_sym_i)
+    
+            if epoch >= uplift:
+                upstream_input = torch.cat(logits_list, dim=1).to(device)
+    
+                final_logits = models[-1](upstream_input, ind_concat.clone()).to('cpu')
+                final_probs = F.softmax(final_logits.clone(), dim=-1)
+                final_preds = torch.argmax(final_probs, dim=1).tolist()
+                preds[-1] += final_preds
+                
+                L_F = criterion(final_logits, label)
+                personals[-1].append(float(L_F.clone()))
+    
+                L_up_sum = eps + torch.sum(L_i_tensor)
+    
+                L_sym_F = L_F*(1+torch.exp(eps-temp*L_up_sum))
+                L_syms.append(L_sym_F)
+    
+                for i, L_i in enumerate(L_i_tensor.clone()):
+                    L_blame_i = lamb*(L_i/L_up_sum)*L_F.clone()
+                    blames[i].append(float(L_blame_i.clone()))
+                    L_syms[i] = L_syms[i] + L_blame_i
+    
+            for i, L_sym_i in enumerate(L_syms):
+                syms[i].append(float(L_sym_i.clone()))
+
+    ## FILL REPORTS
+    for i, report in enumerate(reports):
+        
+        if i != len(reports)-1:
+            report['personal'] = np.mean(personals[i])
+            report['embedding']= np.mean(embs[i])
+            if epoch >= uplift:
+                report['blame'] = np.mean(blames[i])
+            else:
+                report['blame'] = None
+                
+            report['symbiotic'] = np.mean(syms[i])
+            report['accuracy'] = accuracy_score(all_labels, preds[i])
+        else:
+            if epoch >= uplift:
+                report['personal'] = np.mean(personals[i])
+                report['symbiotic'] = np.mean(syms[i])
+                report['accuracy'] = accuracy_score(all_labels, preds[i])
+            else:
+                report['personal'] = None
+                report['symbiotic'] = None
+                report['accuracy'] = None
+    
+    return reports
+
+def train(
+        epochs: int, 
+        models: List[Module], 
+        opts: List[Optimizer],
+        scheds: List[LRScheduler], 
+        data_loaders: List[DataLoader], 
+        collab_params: List[float], 
+        temp: float, 
+        criterion: Module, 
+        tags: List[str] = False, 
+        uplift: int = 10, 
+        eps: float = 1e-7, 
+        lamb: float = 1.0, 
+        save_path: str = False, 
+        save_best: bool = False, 
+        save_end: bool = False, 
+        save_before_uplift: bool = False, 
+        load_at_uplift: bool = False) -> None:
+
+    if not os.path.exists(save_path):
+        os.mkdir(save_path)
+    
+    if tags==False:
+        tags = [f'Model_{i}' for i in range(len(models)-1)] + ['Readout']
+
+    for tag in tags:
+        os.mkdir(f'{save_path}/{tag}')
+        
+    train_loader, val_loader, test_loader = data_loaders
+    
+    len_train = len(train_loader)
+    len_valid = len(val_loader)
+    len_test = len(test_loader)
+
+    lt_reports = []
+
+    for i in range(len(models)):
+        lt_report_i = {
+            'Validation': {},
+            'Testing': {}
+        }
+
+        lt_reports.append(lt_report_i)
+
+    if load_at_uplift:
+        epoch_range = range(uplift, epochs)
+    else:
+        epoch_range = range(0, epochs)
+
+    best_epoch = 0.0
+    best_readout_acc = 0.0
+    for epoch in epoch_range:
+        
+        train_reports = train_one_epoch(
+            train_loader, 
+            models, 
+            opts, 
+            scheds, 
+            collab_params, 
+            temp, 
+            epoch, 
+            criterion, 
+            uplift=uplift, 
+            eps=eps, 
+            lamb=lamb
+        )
+        
+        reports_summary(train_reports, epoch, tags)
+        
+        valid_reports = eval_one_epoch(
+            val_loader, 
+            models, 
+            collab_params, 
+            temp, 
+            epoch, 
+            criterion, 
+            uplift=uplift, 
+            eps=eps, 
+            lamb=lamb, 
+            phase='Validation'
+        )
+
+        if epoch >= uplift and save_best==True:
+            readout_acc = valid_reports[-1]['accuracy']
+
+            if readout_acc > best_readout_acc:
+                best_readout_acc = readout_acc
+                best_epoch = epoch
+
+                best_checkpoints = []
+                for i, model in enumerate(models):
+                    tag = tags[i]
+                    opt = opts[i]
+                    sched = scheds[i]
+                    
+                    checkpoint = {
+                        'epoch': best_epoch,
+                        'model': model.state_dict(),
+                        'opt': opt.state_dict(),
+                        'sched': sched.state_dict(),
+                        'last_step': sched.last_epoch
+                    }
+
+                    best_checkpoints.append(checkpoint)
+        
+        reports_summary(valid_reports, epoch, tags)
+        fill_lt_reports(lt_reports, valid_reports, 'Validation')
+                            
+        if (epoch%5 == 0 or epoch == epochs-1) and epoch != 0:
+            test_reports = eval_one_epoch(
+                test_loader, 
+                models, 
+                collab_params, 
+                temp, 
+                epoch, 
+                criterion, 
+                uplift=uplift, 
+                eps=eps, 
+                lamb=lamb, 
+                phase='Testing'
+            )
+            
+            reports_summary(test_reports, epoch, tags)   
+            fill_lt_reports(lt_reports, test_reports, 'Testing')
+            
+        if epoch == uplift-1 and save_before_uplift == True:
+            for i, model in enumerate(models):
+                tag = tags[i]
+                opt = opts[i]
+                sched = scheds[i]
+                
+                checkpoint = {
+                    'epoch': epoch,
+                    'model': model.state_dict(),
+                    'opt': opt.state_dict(),
+                    'sched': sched.state_dict(),
+                    'last_step': sched.last_epoch
+                }
+                
+                torch.save(checkpoint, f'{save_path}/{tag}_pre-uplift.pt')
+            
+
+    for tag, lt_report in zip(tags, lt_reports):
+        for phase in lt_report.keys():
+            plot_phase(
+                lt_report, 
+                epochs, 
+                uplift, 
+                save_path, 
+                load_at_uplift=load_at_uplift, 
+                phase=phase, 
+                tag=tag
+            )
+
+    if save_end:
+        for i, model in enumerate(models):
+            tag = tags[i]
+            opt = opts[i]
+            sched = scheds[i]
+            
+            checkpoint = {
+                'epoch': epoch,
+                'model': model.state_dict(),
+                'opt': opt.state_dict(),
+                'sched': sched.state_dict(),
+                'last_step': sched.last_epoch
+            }
+            
+            torch.save(checkpoint, f'{save_path}/{tag}.pt')
+
+    if save_best:
+        print('Best Epoch: ', best_epoch)
+        print('Best Readout Accuracy: ', best_readout_acc)
+        for tag, checkpoint in zip(tags, best_checkpoints):
+            torch.save(checkpoint, f'{save_path}/{tag}_Best.pt')
+        
+    return
+
+def plot_phase(
+    lt_report: dict, 
+    epochs: int, 
+    uplift: int, 
+    save_path: str, 
+    load_at_uplift: bool = False, 
+    phase: str = 'Validation',
+    tag: str = 'Model_0') -> None:
+
+    label_lookup = {}
+    label_lookup['personal'] = 'Average Personal Loss'
+    label_lookup['symbiotic'] = 'Average Symbiotic Loss'
+    label_lookup['accuracy'] = 'Accuracy'
+    label_lookup['embedding'] = 'Average Embedding Loss'
+    label_lookup['blame'] = 'Average Blame Loss'
+    
+    if not load_at_uplift:
+        if phase == 'Validation':
+            full_axis = list(range(0, epochs))
+            post_uplift_axis = list(range(uplift, epochs))
+            
+        elif phase == 'Testing':
+            full_axis = list(np.arange(5,epochs,5)) + [epochs]
+            post_uplift_axis = list(np.arange( max([math.ceil(uplift/5)*5, 5]),epochs,5)) + [epochs]
+    else:
+        if phase == 'Validation':
+            full_axis = list(range(uplift, epochs))
+            post_uplift_axis = list(range(uplift, epochs))
+            
+        elif phase == 'Testing':
+            full_axis = list(np.arange(uplift,epochs,5)) + [epochs]
+            post_uplift_axis = list(np.arange(math.ceil(uplift/5)*5,epochs,5)) + [epochs]
+            
+    full_axis = np.array(full_axis)
+    post_uplift_axis = np.array(post_uplift_axis)
+    
+    for key in lt_report[phase].keys():
+        if tag == 'Readout' or key == 'blame':
+            x_axis = post_uplift_axis
+        else:
+            x_axis = full_axis
+
+        data_clean = [float(x) for x in lt_report[phase][key] if x is not None]
+        
+        if key == 'accuracy' and tag == 'Readout':
+            idx_best = np.argmax(data_clean)
+            best_epoch = x_axis[idx_best]
+            best_acc = data_clean[idx_best]
+            label = f'{tag}: Best Accuracy=\n{best_acc:.4f} at Epoch {best_epoch}'
+        else:
+            label = f'{tag}: {label_lookup[key]}'
+
+        try:
+            plt.plot(x_axis, np.array(data_clean), color='black', linestyle='-', label=label)
+        except:
+            print(f"Error Encountered Plotting {tag}'s {phase} {key.capitalize()} Report")
+            print('x axis: ', x_axis)
+            print('data: ', data_clean)
+            continue
+        
+        plt.title(f'{tag} {phase}: {label_lookup[key]} Per Epoch')
+        plt.xlabel('Epoch')
+        plt.ylabel(f'{label_lookup[key]}')
+        plt.grid()
+        if tag != 'Readout' and uplift != 0: plt.axvline(x=uplift, linestyle='dashed', color='red', label=f'Uplift: Epoch {uplift}')
+        plt.legend()
+        plt.savefig(f'{save_path}/{tag}/{phase}_{key}.png')
+        plt.clf()
+        
+    return

symbiotic_learning-0.2.1/src/symbiotic_learning/loss.py

@@ -0,0 +1,36 @@
+import torch
+from torch import Tensor
+import torch.nn as nn
+import torch.nn.functional as F
+
+def embed_sim(x1: Tensor, x2: Tensor) -> Tensor:
+    """
+    Computes the cosine similarity between two embeddings scaled to the range [0,1].
+    """
+    
+    cosine_sim = F.cosine_similarity(x1, x2, dim=0)
+    return (cosine_sim+1)/2
+
+def embed_summand(src: Tensor, aux: Tensor, delta: float = 0.5) -> Tensor:
+    """
+    Computes a sum-term in a pre-Readout model's Embedding Loss.
+    """
+    return torch.exp( (1/delta)*embed_sim(src, aux) ) - 1
+
+def embed_loss(src: Tensor, auxs: Tensor) -> Tensor:
+    """
+    Computes a pre-Readout model's Embedding Loss.
+    """
+
+    num_aux = len(auxs)
+    temp_func = lambda aux: embed_summand(src, aux)
+    temp_func = torch.vmap(temp_func)
+
+    auxs = torch.stack(auxs)
+    sum_terms = temp_func(auxs)
+    sum_terms = torch.sum(sum_terms, dim=0)
+    pre_factor = 1/num_aux
+    
+    L_embed = pre_factor*sum_terms
+
+    return L_embed.mean()