modelmark 0.1.0__py3-none-any.whl

modelmark/__init__.py

@@ -0,0 +1,3 @@
+"""Reusable testing framework."""
+
+__version__ = "0.1.0"

modelmark/__main__.py

@@ -0,0 +1,5 @@
+import sys
+from modelmark import run
+
+if __name__ == "__main__":
+	sys.exit(run())

modelmark/common/dataset.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import torch
+from torch.utils.data import Dataset
+
+import copy
+import numpy as np
+import pandas as pd
+
+from modelmark.common.utils import load_config
+config = load_config()
+
+class CausalDataset(Dataset):
+	"""
+	Sliding-window dataset for ETT.
+
+	Returns:
+		x: [seq_len, num_features]
+		y: [pred_len, 1]
+	"""
+
+	def __init__(
+		self,
+		data_config: dict,
+		seq_len: int,
+		pred_len: int,
+		split: str = "train",
+		mean: np.ndarray | None = None,
+		std: np.ndarray | None = None,
+	):
+		super().__init__()
+
+		self.seq_len = seq_len
+		self.pred_len = pred_len
+
+		# Extract config
+		data_path = config.data_path + data_config["path"]
+		features = copy.deepcopy(data_config["input_features"])
+		targets = data_config["output_features"]
+		features.extend([item for item in targets if item not in features])
+		train_ratio = data_config["train_ratio"]
+		val_ratio = data_config["val_ratio"]
+
+		# Load, extract, convert
+		df = pd.read_csv(data_path)
+		
+		df = df[features]
+		data = df.values.astype(np.float32)
+		self.input_indices = torch.tensor(df.columns.get_indexer(data_config["input_features"]), dtype=torch.int)
+		self.output_indices = torch.tensor(df.columns.get_indexer(data_config["output_features"]), dtype=torch.int)
+
+		n = len(data)
+
+		train_end = int(n * train_ratio)
+		val_end = int(n * (train_ratio + val_ratio))
+
+		if split == "train":
+			start = 0
+			end = train_end
+		elif split == "val":
+			start = train_end
+			end = val_end
+		elif split == "test":
+			start = val_end
+			end = n
+		else:
+			raise ValueError(f"Unknown split: {split}")
+
+		# Save scaling information
+		if split == "train":
+			self.mean = data[start:train_end].mean(axis=0)
+			self.std = data[start:train_end].std(axis=0)
+
+			# Prevent division by zero
+			self.std[self.std < 1e-8] = 1.0
+		else:
+			if mean is None or std is None:
+				raise ValueError(
+					"Validation/test datasets need training mean and std."
+				)
+
+			self.mean = mean
+			self.std = std
+
+		# Scale using TRAIN statistics
+		data = (data - self.mean) / self.std
+
+		# Keep only the split region
+		self.data = data[start:end]
+
+		# Number of valid windows
+		self.length = len(self.data) - seq_len - pred_len + 1
+
+		if self.length <= 0:
+			raise ValueError(
+				f"Split '{split}' is too short for "
+				f"seq_len={seq_len}, pred_len={pred_len}"
+			)
+
+	def __len__(self):
+		return self.length
+
+	def __getitem__(self, idx):
+		x_start = idx
+		x_end = x_start + self.seq_len
+
+		y_start = x_end
+		y_end = y_start + self.pred_len
+
+		x = self.data[x_start:x_end, self.input_indices]
+		y = self.data[y_start:y_end, self.output_indices]
+
+		return (
+			torch.from_numpy(x),
+			torch.from_numpy(y),
+		)

modelmark/common/downloader.py

@@ -0,0 +1,88 @@
+"""Download time-series datasets to a local directory.
+
+Supports:
+- ETT (ETTh1, ETTh2, ETTm1, ETTm2) — from the ETDataset GitHub repo
+- Air Quality (UCI Repository)
+- Weather (Max Planck Institute / Jena climate dataset)
+"""
+
+import argparse
+import shutil
+import urllib.request
+import zipfile
+from pathlib import Path
+
+import logging
+logger = logging.getLogger(__name__)
+from rich.console import Console
+console = Console()
+
+DATASETS = {
+	"ett": {
+		"dir": "data/ett",
+		"kind": "multi_csv",
+		"base_url": "https://raw.githubusercontent.com/zhouhaoyi/ETDataset/main/ETT-small",
+		"files": ["ETTh1", "ETTh2", "ETTm1", "ETTm2"],
+	},
+	"air_quality": {
+		"dir": "data/air_quality",
+		"kind": "zip",
+		"url": "https://archive.ics.uci.edu/static/public/360/air+quality.zip",
+		"zip_name": "AirQualityUCI.zip",
+	},
+	"weather": {
+		"dir": "data/weather",
+		"kind": "zip",
+		"url": "https://storage.googleapis.com/tensorflow/tf-keras-datasets/jena_climate_2009_2016.csv.zip",
+		"zip_name": "jena_climate_2009_2016.csv.zip",
+	},
+}
+
+
+def download_ett(cfg: dict, out_dir: Path) -> None:
+	out_dir.mkdir(parents=True, exist_ok=True)
+	for name in cfg["files"]:
+		url = f"{cfg['base_url']}/{name}.csv"
+		path = out_dir / f"{name}.csv"
+		if path.exists():
+			logger.info(f"{name}.csv already exists, skipping.")
+			continue
+		logger.info(f"Downloading {name}...")
+		console.print(f"Downloading {name}...")
+		urllib.request.urlretrieve(url, path)
+		logger.info(f"Saved to {path}")
+		console.print(f"Saved to {path}", style="green")
+
+
+def download_zip_dataset(cfg: dict, out_dir: Path) -> None:
+	out_dir.mkdir(parents=True, exist_ok=True)
+	zip_path = out_dir / cfg["zip_name"]
+
+	if not zip_path.exists():
+		logger.info(f"Downloading {cfg['zip_name']}...")
+		console.print(f"Downloading {cfg['zip_name']}...")
+		urllib.request.urlretrieve(cfg["url"], zip_path)
+		logger.info(f"Saved to {zip_path}")
+		console.print(f"Saved to {zip_path}", style="green")
+	else:
+		logger.info(f"{cfg['zip_name']} already exists, skipping download.")
+		console.print(f"{cfg['zip_name']} already exists, skipping download.", style="yellow")
+
+	logger.info(f"Extracting {zip_path.name}...")
+	console.print(f"Extracting {zip_path.name}...")
+	with zipfile.ZipFile(zip_path) as zf:
+		zf.extractall(out_dir)
+	console.print(f"Extracted to {out_dir}", color="green")
+
+def download_dataset(name: str) -> None:
+	if name not in DATASETS:
+		valid = ", ".join(DATASETS)
+		raise ValueError(f"Unknown dataset '{name}'. Choose from: {valid}")
+
+	cfg = DATASETS[name]
+	out_dir = Path(cfg["dir"])
+
+	if cfg["kind"] == "multi_csv":
+		download_ett(cfg, out_dir)
+	elif cfg["kind"] == "zip":
+		download_zip_dataset(cfg, out_dir)

modelmark/common/loader.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from modelmark.common.dataset import CausalDataset
+from torch.utils.data import DataLoader
+
+import logging
+logger = logging.getLogger(__name__)
+from modelmark.common.utils import load_config
+config = load_config()
+
+class Loader:
+
+	def __init__(self, file_config : dict, context_size : int):
+
+		self.get_dataloaders(file_config, context_size) 
+		logger.debug(f"Dataset len stats: train={len(self.train_loader)} val={len(self.val_loader)} test={len(self.test_loader)}")
+
+	def get_dataloaders(self, file_config : dict, context_size : int) -> tuple[DataLoader, DataLoader, DataLoader]:
+		"""Load the data, pack to datasets, create the loaders and return them"""
+	
+		train_ds = CausalDataset(data_config = file_config,
+								seq_len = context_size, 
+								pred_len = context_size, 
+								split = "train")
+
+		val_ds 	= CausalDataset(data_config = file_config, 
+								seq_len = context_size, 
+								pred_len = context_size, 
+								split = "val", 
+								mean = train_ds.mean, 
+								std = train_ds.std)
+
+		test_ds = CausalDataset(data_config = file_config, 
+								seq_len = context_size, 
+								pred_len = context_size, 
+								split = "test", 
+								mean = train_ds.mean, 
+								std = train_ds.std)
+		
+		self.train_loader = DataLoader(
+			train_ds,
+			batch_size = config.test_config["optim"]["batch_size"],
+			shuffle = True,
+			num_workers = 0,
+			pin_memory = True,
+		)
+
+		self.val_loader = DataLoader(
+			val_ds,
+			batch_size = config.test_config["optim"]["batch_size"],
+			shuffle = False,
+			num_workers = 0,
+			pin_memory = True,
+		)
+	
+		self.test_loader = DataLoader(
+			test_ds,
+			batch_size = config.test_config["optim"]["batch_size"],
+			shuffle = False,
+			num_workers = 0,
+			pin_memory = True,
+		)
+	

modelmark/common/logger.py

@@ -0,0 +1,34 @@
+import logging
+import sys
+from typing import Optional
+
+def setup_logging(log_format: Optional[str] = None) -> None:
+	"""
+	Configure logging for the entire application.
+
+	Args:
+		log_format: Optional custom format string. If None, a default format is used.
+	"""
+
+	if log_format is None:
+		log_format = "%(asctime)s [%(levelname)s] [%(name)s] - %(message)s"
+
+	logging.basicConfig(
+		level = logging.DEBUG,
+		filemode = 'w',
+		filename='modelmark_files/modelmark.log',
+		format = log_format,
+		datefmt = '%H:%M:%S',
+	)
+
+def get_logger(name: str) -> logging.Logger:
+	"""
+	Convenience function to get a logger for a module.
+
+	Args:
+		name: Usually __name__.
+
+	Returns:
+		A configured logger instance.
+	"""
+	return logging.getLogger(name)

modelmark/common/parser.py

@@ -0,0 +1,134 @@
+import argparse
+import sys
+import shutil
+import pandas as pd
+
+from modelmark.common.downloader import download_dataset
+import modelmark.constants as constants
+
+import logging
+logger = logging.getLogger(__name__)
+from rich.console import Console
+console = Console()
+
+class Parser:
+
+	def __init__(self):
+		"""Parse the arguments"""
+
+		# Create parser
+		self.parser = argparse.ArgumentParser(
+			formatter_class=argparse.RawTextHelpFormatter,
+			description = constants.PARSER_DESC
+		)
+	
+		# Add arguments
+		self.parser.add_argument(
+			"-t", "--task",
+			type = str,
+			default = "help",
+			help = constants.PARSER_HELP
+		)
+	
+		# Run without args / help request - print help
+		if len(sys.argv) == 1:
+			self.parser.print_help()
+			sys.exit(1)
+		
+		# Parse argument values
+		self.args = self.parser.parse_args()
+
+		# Setup other
+		pd.set_option('display.colheader_justify', 'center')	
+
+	def run(self):
+		"""Select the task and run"""
+
+		# Config initialization #
+		if self.args.task == "init":
+			self.init_config()
+			return 0
+
+		# Dataset download #
+		if self.args.task == "load":
+			download_dataset("ett")
+			return 0
+
+		# Run the test #
+		if self.args.task == "run":
+			return None
+
+		# Clean the config folder #
+		if self.args.task == "clear":
+			self.clear_config()
+			return 0
+
+		# Reset the modelmark config #
+		if self.args.task == "reset":
+			self.reset_config()
+			return 0
+
+		# Print the modelmark usage #
+		if self.args.task == "help":
+			self.parser.print_help()
+			return 0
+		
+		# Check if task is correct #
+		if self.args.task != "run":
+			logger.error(f"Unknown task: {self.args.task}, run 'modelmark' for more info")
+			console.print(f"Unknown task: {self.args.task}, run 'modelmark' for more info", style="red")
+			return 1
+
+		# Otherwise return error
+		return 1
+
+	def init_config(self):
+		"Copy the config example file to the user dir"
+
+		user_config_dir = constants.USER_CONFIG_DIR
+		user_models_dir = constants.USER_MODELS_DIR
+
+		package_config = constants.PACKAGE_CONFIG_PATH
+		user_config = constants.USER_CONFIG_PATH
+		package_model = constants.PACKAGE_MODEL_PATH
+		user_model = constants.USER_MODEL_PATH
+
+		# Make the dirs
+		user_config_dir.mkdir(parents=True, exist_ok=True)
+		user_models_dir.mkdir(parents=True, exist_ok=True)
+
+		if not user_config.exists():
+			shutil.copy(package_config, user_config)
+
+			logger.info(f"Customizable config created at: {user_config}")
+			console.print(f"Customizable config created at: {user_config}")			
+
+			shutil.copy(package_model, user_model)
+
+			logger.info(f"Model example file created at: {user_model}")
+			console.print(f"Model example file created at: {user_model}")
+						
+			console.print(f"Initialization complete.", style="green")
+		else:
+
+			console.print(f"config.py already exists", style="yellow")
+			self.parser.print_help()
+
+	def clear_config(self):
+		"""Remove the user config dir"""
+		user_config_path = constants.USER_CONFIG_DIR
+		user_example_model_path = constants.USER_MODEL_PATH
+
+		if user_config_path.exists():
+			shutil.rmtree(user_config_path)
+			logger.info(f"Folder {user_config_path} removed.")
+			console.print(f"Folder {user_config_path} removed.")
+		if user_example_model_path.exists():
+			user_example_model_path.unlink(missing_ok=True)
+			logger.info(f"File {user_example_model_path} removed.")
+			console.print(f"File {user_example_model_path} removed.")
+				
+	def reset_config(self):
+		self.clear_config()
+		self.init_config()
+