modelmark 0.1.0__py3-none-any.whl

modelmark/models/gru.py

@@ -0,0 +1,45 @@
+import torch
+import torch.nn as nn
+
+from modelmark.common.utils import load_config
+
+class GRUModel(nn.Module):
+	def __init__(self, input_dim : int, output_dim : int, context_size: int):
+		super().__init__()
+
+		# To avoid circular import, now it lives here
+		config = load_config()
+
+		self.input_size = input_dim
+		self.output_size = output_dim
+		self.context_size = context_size
+		
+		self.hidden_size = config.model_config["GRU"]["hidden_size"]
+		self.num_layers = config.model_config["GRU"]["num_layers"]
+
+		self.gru = nn.GRU(
+			input_size = self.input_size,
+			hidden_size = self.hidden_size,
+			num_layers = self.num_layers,
+			batch_first = True,
+		)
+
+		self.fc = nn.Linear(
+			self.hidden_size,
+			context_size * self.output_size,
+		)
+
+	def forward(self, x):
+		_, hn = self.gru(x)
+
+		# Last layer's hidden state
+		h = hn[-1]
+
+		out = self.fc(h)
+		out = out.view(
+			x.size(0),
+			self.context_size,
+			self.output_size,
+		)
+
+		return out

modelmark/models/linear.py

@@ -0,0 +1,65 @@
+""" 
+Description:
+	
+	This is an example of the Linear model definition.
+	You can write any other model, but the core components must remain intact:
+
+		1) The model __init__ core args must be included:
+			
+			(input_dim : int, output_dim : int, context_size: int)
+
+			For mode parameter tweaking use your config file, 
+			or define params right in the __init__() method (not recommended, config is more convenient).
+
+		2) The forward() method:
+
+			Input - [B, T, input_dim]
+			Output - [B, T, output_dim]
+
+			Currently only the continuous time-series data forecasting is available.
+
+		3) Differentiability:
+
+			The model should be differentiable, we will optimize it with Adam(). 
+
+"""
+
+import torch
+import torch.nn as nn
+
+# Switch to user's config when available
+from modelmark.common.utils import load_config
+
+class Linear(nn.Module):
+	def __init__(self, input_dim : int, output_dim : int, context_size: int):
+		super().__init__()
+
+		# To avoid circular import, now it lives here
+		config = load_config()
+
+		self.input_size = input_dim
+		self.output_size = output_dim
+		self.context_size = context_size
+		
+		self.hidden_size = config.model_config["Linear"]["hidden_size"]
+		self.num_layers = config.model_config["Linear"]["num_layers"]
+
+		layers = [
+			nn.Linear(self.input_size, self.hidden_size),
+			nn.GELU()
+		]
+
+		for _ in range(self.num_layers - 1):
+			layers.append(nn.Linear(self.hidden_size, self.hidden_size))
+			layers.append(nn.GELU())
+
+		self.net = nn.Sequential(*layers)
+
+		self.fc = nn.Linear(self.hidden_size, self.output_size)
+
+	def forward(self, x):
+
+		x = self.net(x)
+		x = self.fc(x)
+
+		return x

modelmark/models/lstm.py

@@ -0,0 +1,43 @@
+import torch
+import torch.nn as nn
+
+from modelmark.common.utils import load_config
+
+class LSTMModel(nn.Module):
+	def __init__(self, input_dim: int, output_dim: int, context_size: int):
+		super().__init__()
+
+		# To avoid circular import, now it lives here
+		config = load_config()
+
+		self.input_size = input_dim
+		self.output_size = output_dim
+		self.context_size = context_size
+		self.hidden_size = config.model_config["LSTM"]["hidden_size"]
+		self.num_layers = config.model_config["LSTM"]["num_layers"]
+		
+		# Change nn.GRU to nn.LSTM
+		self.lstm = nn.LSTM(
+			input_size=self.input_size,
+			hidden_size=self.hidden_size,
+			num_layers=self.num_layers,
+			batch_first=True,
+		)
+		self.fc = nn.Linear(
+			self.hidden_size,
+			context_size * self.output_size,
+		)
+
+	def forward(self, x):
+
+		_, (hn, cn) = self.lstm(x)
+		
+		# Take the last layer's hidden state
+		h = hn[-1]
+		out = self.fc(h)
+		out = out.view(
+			x.size(0),
+			self.context_size,
+			self.output_size,
+		)
+		return out

modelmark/models/transformer.py

@@ -0,0 +1,86 @@
+import torch
+import torch.nn as nn
+
+from modelmark.common.utils import load_config
+
+class TransformerModel(nn.Module):
+	def __init__(self, input_dim : int, output_dim : int, context_size: int):
+		super().__init__()
+
+		# To avoid circular import, now it lives here
+		config = load_config()
+	
+		self.input_size = input_dim
+		self.output_size = output_dim
+		self.context_size = context_size
+		self.max_seq_len = self.context_size
+		
+		self.hidden_size = config.model_config["Transformer"]["hidden_size"]
+		self.num_layers = config.model_config["Transformer"]["num_layers"]
+		self.num_heads = config.model_config["Transformer"]["num_heads"]
+		self.dropout = config.model_config["Transformer"]["dropout"]
+
+		# Input projection
+		self.input_proj = nn.Linear(self.input_size, self.hidden_size,)
+
+		# Learnable positional embedding
+		self.pos_embedding = nn.Parameter(torch.zeros(1, self.max_seq_len, self.hidden_size))
+
+		# Transformer decoder layer
+		decoder_layer = nn.TransformerDecoderLayer(
+			d_model=self.hidden_size,
+			nhead=self.num_heads,
+			dim_feedforward=4 * self.hidden_size,
+			dropout=self.dropout,
+			batch_first=True,
+		)
+
+		self.decoder = nn.TransformerDecoder(decoder_layer, num_layers=self.num_layers)
+
+		# Map hidden representation to outputs
+		self.fc = nn.Linear(self.hidden_size, self.output_size)
+
+	def forward(self, x):
+		# x:
+		# [batch, sequence_length, input_size]
+
+		batch_size, seq_len, _ = x.shape
+
+		if seq_len > self.pos_embedding.size(1):
+			raise ValueError(
+				f"Sequence length {seq_len} exceeds "
+				f"max_seq_len {self.pos_embedding.size(1)}"
+			)
+
+		# Project input to transformer dimension
+		x = self.input_proj(x)
+		# [batch, sequence_length, hidden_size]
+
+		# Add positional information
+		x = x + self.pos_embedding[:, :seq_len, :]
+
+		# Causal mask:
+		# position i can only attend to positions <= i
+		mask = nn.Transformer.generate_square_subsequent_mask(
+			seq_len,
+			device=x.device,
+		)
+
+		# Self-attention through decoder layers
+		x = self.decoder(
+			tgt=x,
+			memory=x,
+			tgt_mask=mask,
+			memory_mask=mask,
+		)
+		# [batch, sequence_length, hidden_size]
+
+		# Take the final context_size positions
+		x = x[:, -self.context_size:, :]
+		# [batch, context_size, hidden_size]
+
+		# Project to output
+		out = self.fc(x)
+		# [batch, context_size, output_size]
+
+		return out

modelmark-0.1.0.dist-info/METADATA

@@ -0,0 +1,109 @@
+Metadata-Version: 2.4
+Name: modelmark
+Version: 0.1.0
+Summary: Benchmark framework for neural network models
+Author: Andrew Larin
+License: MIT
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=2.4
+Requires-Dist: pandas>=3.0
+Requires-Dist: torch>=2.12
+Requires-Dist: tqdm>=4.67
+Requires-Dist: playwright>=1.62
+Requires-Dist: rich>=15.0
+Requires-Dist: thop>=0.1
+Dynamic: license-file
+
+# ModelMark
+This tool will help you to test NN models against each other, and form a detailed report that is easy to embed to a website.
+
+**Model evaluation report example:**
+
+<img src="result.png" alt="Evaluation result" width="600">
+
+## About:
+
+At each testing run iteration, modelmark:
+		
+	1) Selects next dataset, context, model and seed
+	2) Seeds the generators for reproducibility
+	3) Creates the loader, model and tester objects
+	4) Trains the model for E epochs, restores the state with the least validation loss
+	5) Tracks the GFLOPs, Memory, Time - all AVG over whole test
+	5) Evaluates the model on dataset with metrics from configuration file
+	6) Stores the mean result over S runs 
+
+	That way, the more seeds you run, the more "fair" the results are.
+	Finally, modelmark will form the report with all the testing results, training stats and your machine metadata.
+
+More on training stats:
+		
+	Time 		- average time per epoch 
+	Params 		- total number of model params
+	GFLOPs 		- average per batch
+	Peak Memory - max per training iteration
+
+Test consists of F * C * M * S runs, where:
+	
+	F - number of dataset files in the config (e.g. ["ETTh1" : ..., "Weather" : ...] - means F = 2)
+	C - number of context sizes (e.g. [32, 64, 128] - means C = 3)
+	M - number of models (e.g. ["Linear" : ..., "LSTM" : ...] - means M = 2)
+	S - number of seeds (e.g. [42, 43, 44] - means S = 2)
+
+# Requirements:
+	
+	OS: Windows or Linux
+	Python: 3.12+
+
+# Usage:
+
+	1) Install the package   
+		
+		pip install "modelmark @ git+https://github.com/gloptim77/ModelMark.git"
+
+	2) Run the initialization in an empty folder
+
+		modelmark -t init
+	
+	3.1) It will create two folders "modelmark_files" and "models"
+	   
+	    In modelmark_files/config.py there are 3 main configs:
+
+	   		model_config = {...} - Models hyperparameters (number of layers, hidden dim, kernel size, etc.)
+			data_config = {...} - Dataset parameters (path to file, input/output features, train/val ratios, etc.)
+			test_config = {...} - Testing options (optimizer, loss criterion, metrics, learning rate, etc.)
+
+	3.2) (Optional) download the ETT dataset files
+
+		modelmark -t load
+ 
+	4) When your config, model and data are ready, you can run the testing
+
+		modelmark -t run
+
+	5) When the test is over, report results will be in files "result.html" and "result.png"
+
+	Example of the model: "src/modelmark/models/linear.py"
+	Example of the config: "src/modelmark/config.py" 
+
+You will find the detailed config example with description at [src/modelmark/config.py](src/modelmark/config.py).
+
+Example Linear model with detailed description at [src/modelmark/models/linear.py](src/modelmark/models/linear.py).
+
+You can change the config and add your model files to suit your test requirements. 
+
+(But make sure that config and models are compatible)
+
+## Troubleshooting
+
+- **If something doesn't work**
+
+	- Please open an issue and attach your application logs (found at "modelmark_files/modelmark.log") so I can help you troubleshoot. 
+	- Try to restart
+		```python
+		modelmark -t restart
+  		```
+
+	- Try to manually delete "modelmark_files" and return to Usage->2)

modelmark-0.1.0.dist-info/RECORD

@@ -0,0 +1,24 @@
+modelmark/__init__.py,sha256=cJHYdio7T5sb5Y65QxncymbNMU4WfOOUqvPvtpDuIN8,56
+modelmark/__main__.py,sha256=_b2ICXLS4APDLGxjTRNhc17k36SGlpACeOsFW7nLGkk,81
+modelmark/config.py,sha256=OF4aS4QMx70fd57f31iteEVHo6N2TMSCyRBZPcv36u4,2508
+modelmark/constants.py,sha256=lSlR-TmxGhSFih3zJMZ6vCtYTUmVwFqFj1D-aM52K1Y,1096
+modelmark/modelmark.py,sha256=le6cVW2xGiCpWzBU6UmnuOOrlebh-oQn9MWGA2lu500,7714
+modelmark/common/dataset.py,sha256=Ahyux0rcWxgI3YHkPS-ApHlyBKZL5p81eXz1YkiV--o,2663
+modelmark/common/downloader.py,sha256=va2cqga4Tii6uqJgP79RRLhtDH3I2cNSlvjtBox-NiM,2666
+modelmark/common/loader.py,sha256=SYv8FDscfZbVQ4IaTxcwfsuFBL4t3JNxL3y_3KS-rEE,1769
+modelmark/common/logger.py,sha256=1cNMADpRxtVWMLdncJIfmrEzrBi4S17MLomeEeiFeH8,732
+modelmark/common/parser.py,sha256=T1OkpjAs1VYwmElZOPe1k4j5wvEdnP-YnCsN4v9NY10,3429
+modelmark/common/report.py,sha256=soOxwfk3VFjLwf730-1jsCMtbvxCDOsNIp7uEa1B7Co,19778
+modelmark/common/tester.py,sha256=mDPW7oIN9MnwchuBKwYTvMfAAOwmc7d6BALbCqe0jq8,4335
+modelmark/common/utils.py,sha256=O3HDrC47KnJVe8SsjlX1ynj499VDue613Jrcko4Nzbw,2526
+modelmark/models/conv.py,sha256=MoeaXcXFYjibR5SXVgMWJiXVWpl0v5x2p8SE01Uo5F4,1678
+modelmark/models/gru.py,sha256=tEo2goO1INQ8BObXsbSk3M8e9ZqwZJn70gCaA-oDiAM,943
+modelmark/models/linear.py,sha256=GHRpzhz-zJCQd6A2EimDoRzOrYLm8qoGPIXfUcxQVsg,1616
+modelmark/models/lstm.py,sha256=4FVOyxxXjNpde9d9YdqN_nE1d3DA2RRLE2G4uHwiM5o,982
+modelmark/models/transformer.py,sha256=EjVQgO7-AeHLK1mADscuN7YAaYvSvzGQiIx524wNIHQ,2315
+modelmark-0.1.0.dist-info/licenses/LICENSE,sha256=HtjFfP8fZ8LTdjFbSqhQKjt_4uZophTprjczqbTHxLo,1069
+modelmark-0.1.0.dist-info/METADATA,sha256=rnnz-MDRrfR4yNx3bsb65V9Kntrnc4HPGAYLkEvLMD0,3560
+modelmark-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+modelmark-0.1.0.dist-info/entry_points.txt,sha256=EpoYh6jh3uO9kKoDVkJJtsAtpF0EJYO8YRBUemwnzaE,54
+modelmark-0.1.0.dist-info/top_level.txt,sha256=l54tAXR7NL0-JVAci_RRPJOBtFNlRduLps5WR3Qg6g0,10
+modelmark-0.1.0.dist-info/RECORD,,

modelmark-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

modelmark-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+modelmark = modelmark.modelmark:run

modelmark-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Andrew Larin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

modelmark-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+modelmark