dsfolder 0.0.0.1__py3-none-any.whl

dsfolder-0.0.0.1.dist-info/METADATA

@@ -0,0 +1,49 @@
+Metadata-Version: 2.1
+Name: dsfolder
+Version: 0.0.0.1
+Summary: Add your description here
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: path>=17.1.1
+Requires-Dist: python-dotenv>=1.2.1
+Requires-Dist: sqlalchemy>=2.0.48
+Requires-Dist: toml>=0.10.2
+
+## Data Science Folder Structure Setup
+
+1. This was developed for creating a Python folder to start a data science project.
+
+2. It includes preconfigured connections to different systems:
+   - uv  
+   - git  
+   - GitHub workflow  
+   - .gitignore  
+
+3. It also provides small code snippets for connecting to other systems.
+
+4. Below are some of the systems included, along with additional setup steps to get them working:
+   - logging  
+   - SQL  
+   - src  
+   - config  
+   - pre-commit hook  
+   - .gitignore  
+   - git  
+   - GitHub workflow  
+   - uv  
+   - env  
+   - validation  
+   - testing  
+   - requirements.txt  
+     - Install dependencies with: `uv install requirements.txt`
+
+5. Install the package:
+   ```bash
+   pip install datascience-folder-structure
+6. Create a folder structure:
+   ```bash
+   createdatasciencefolder NameOfProject
+
+   or
+   ```bash
+   dsf NameOfProject 

dsfolder-0.0.0.1.dist-info/RECORD

@@ -0,0 +1,39 @@
+dspackagefolder/__init__.py,sha256=T4SQ9iIEoHqgs3vcujfIj14638recwUibHxZJMp_x0g,36
+dspackagefolder/install_precommit_and_create_env.py,sha256=WBHSDf6V4Dinelrx4zPOnKn1txgJxj13GmBOcXT850E,5899
+dspackagefolder/main.py,sha256=2uT3xXbruN9RKaCW4AZU3TSYMtsYeolJjqH2O3cThoo,1651
+dspackagefolder/template/.gitignore,sha256=z31xAJAzlQXZa7OPSmKu-AvzRLiy8x2l5z1hmuGFzg4,198
+dspackagefolder/template/.pre-commit-config.yaml,sha256=KqIkDWcZz_1SyNx2SbGpMjN81kpLlHUgTU_dqY_Mkt4,288
+dspackagefolder/template/.python-version,sha256=p7naXjVc4upFCMxTdWhJ9EyWDYejjlp20Mc011EsU3E,6
+dspackagefolder/template/Dockerfile,sha256=kOWmv8tJOcWrMTYAu2O3sB-lSSH-APlSRDk_qNC0CzQ,867
+dspackagefolder/template/README.md,sha256=PhYLaYwDOaxnCPcGC8IprTABLM00z3I-fUeby0OsXBg,376
+dspackagefolder/template/main.py,sha256=bPmQKOveew9aL9nolZgxic4RbwJII-oekxugIa5DVrU,1613
+dspackagefolder/template/main_dashboard.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dspackagefolder/template/requirements.txt,sha256=TuUEcQyEvKvPEmmZNW351SJsVtWjE4KTvLZ2HCK1PBQ,74
+dspackagefolder/template/template_notebook.ipynb,sha256=Cw_zuCJ0UGpqkCAqU0EtSf3wd5if9-8zIiz8Qha3d2s,8474
+dspackagefolder/template/.config/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dspackagefolder/template/.config/example.toml,sha256=O1CFW5XwAaj9TltHIyOVLGrA0_V97YsIu60m41mtqKM,10
+dspackagefolder/template/EXAMPLE/example.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dspackagefolder/template/checklist/.gitignore,sha256=L4FYmilJi5_FIJCHOdbeBC35uyrz6Lb0QpIYTHDrd3Y,3
+dspackagefolder/template/checklist/GENERIC_CHECKLIST.xlsx,sha256=JZsJaItwx6YgCkDsmn01kmOid19LhKSLzT4Kl0SBzrI,10080
+dspackagefolder/template/checklist/ML_CHECKLIST.xlsx,sha256=ezIJjW6xSojmFL-PcY7xvyHgtqinqK0GtXxtBvwkvMQ,11157
+dspackagefolder/template/checklist/STATISTICS_CHECKLIST.xlsx,sha256=_oVHef7dDJoNs21Vic1ez5T4_pmEckGpB0kgAaes3sk,9557
+dspackagefolder/template/data/.gitkeep,sha256=L4FYmilJi5_FIJCHOdbeBC35uyrz6Lb0QpIYTHDrd3Y,3
+dspackagefolder/template/data/INPUT/.gitkeep,sha256=L4FYmilJi5_FIJCHOdbeBC35uyrz6Lb0QpIYTHDrd3Y,3
+dspackagefolder/template/data/OUTPUT/.gitkeep,sha256=L4FYmilJi5_FIJCHOdbeBC35uyrz6Lb0QpIYTHDrd3Y,3
+dspackagefolder/template/logs/.gitkeep,sha256=yJX96YmS0LvV0np6RZeW2d1q8B9Do1MqExKH5iIhUAY,10
+dspackagefolder/template/models/.gitignore,sha256=L4FYmilJi5_FIJCHOdbeBC35uyrz6Lb0QpIYTHDrd3Y,3
+dspackagefolder/template/sql/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dspackagefolder/template/sql/query.sql,sha256=frcCV1k9oG9oKj3dpUqdJg1PxRT2RSN_XKdLCPjaYaY,2
+dspackagefolder/template/src/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dspackagefolder/template/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dspackagefolder/template/src/sqlcrud.py,sha256=UZfWfdCwK1nnu1WJShMSWDYOvWStq0iHlx6z8c24h5Y,2858
+dspackagefolder/template/src/utils/config_loader.py,sha256=rjUFGcdbEIDxelUpIHp1YX6HMGjJCcUkWOzAsHQYayw,629
+dspackagefolder/template/src/utils/env_loader.py,sha256=0ajuMmv3BLhsRgIt5qOLlJaZxNXZBZ3NGcfJaZf5bkM,589
+dspackagefolder/template/src/utils/logging_config.py,sha256=hhCcIAHjc5E_QIyYixewI-jQgAJwpJiJRaM7wv9ARvM,1394
+dspackagefolder/template/unit_testing/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dspackagefolder/template/validation/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dsfolder-0.0.0.1.dist-info/METADATA,sha256=jwA8w01F49D0eKOr-Nce-mikg_DfQNS1K_1wyvIqzs8,1222
+dsfolder-0.0.0.1.dist-info/WHEEL,sha256=EWGgdj_eH-g9dwsWZOTG0kmGucEuKPxGzOKD-jAVUHs,92
+dsfolder-0.0.0.1.dist-info/entry_points.txt,sha256=kHH2Hc8lEfWczBRr8UXUWid_KHU6MHsikh3RWHTHgy4,134
+dsfolder-0.0.0.1.dist-info/top_level.txt,sha256=XKesjMHDkbB-GfN3xELx0YajPXIKPGEJVVZ1c42KL_Q,16
+dsfolder-0.0.0.1.dist-info/RECORD,,

dsfolder-0.0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.48.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dsfolder-0.0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+createdatasciencefolder = dspackagefolder.main:create_ds_folder_cli
+dsf = dspackagefolder.main:create_ds_folder_cli

dsfolder-0.0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+dspackagefolder

dspackagefolder/__init__.py

@@ -0,0 +1 @@
+from .main import create_ds_folder

dspackagefolder/install_precommit_and_create_env.py

@@ -0,0 +1,184 @@
+## install uv
+## uv add requirements.txt
+## install pre-commit
+## pre-commit install
+## creeate env file into the new folder
+
+
+import subprocess
+import sys
+from pathlib import Path
+
+
+def setup_project_environment(project_dir: str | Path = "."):
+    project_dir = Path(project_dir).resolve()
+
+    try:
+        if not project_dir.exists():
+            raise FileNotFoundError(f"{project_dir} does not exist")
+    except Exception as e:
+        print(f"❌ Invalid project directory: {e}")
+        return
+
+    def run(cmd: list[str]):
+        try:
+            subprocess.check_call(cmd, cwd=project_dir)
+        except subprocess.CalledProcessError as e:
+            print(f"❌ Command failed: {' '.join(cmd)}\n{e}")
+        except FileNotFoundError as e:
+            print(f"❌ Command not found: {cmd[0]}\n{e}")
+        except Exception as e:
+            print(f"❌ Unexpected error running {' '.join(cmd)}: {e}")
+
+    # 1️⃣ Install uv if missing
+    try:
+        subprocess.check_call([sys.executable, "-m", "uv", "--version"])
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        try:
+            print("Installing uv...")
+            subprocess.check_call([sys.executable, "-m", "pip", "install", "uv"])
+        except Exception as e:
+            print(f"❌ Failed to install uv: {e}")
+
+    # 2️⃣ Initialise uv project if pyproject.toml is missing
+    try:
+        pyproject = project_dir / "pyproject.toml"
+        if not pyproject.exists():
+            print("Initialising uv project...")
+            run(["uv", "init"])
+    except Exception as e:
+        print(f"❌ Error during uv init: {e}")
+
+    # 3️⃣ Add requirements.txt using uv
+    try:
+        requirements = project_dir / "requirements.txt"
+        if requirements.exists():
+            run(["uv", "add", "-r", "requirements.txt"])
+        else:
+            print("requirements.txt not found — skipping uv add")
+    except Exception as e:
+        print(f"❌ Error adding requirements: {e}")
+
+    # 4️⃣ Install pre-commit if missing
+    try:
+        subprocess.check_call(["pre-commit", "--version"])
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        try:
+            print("Installing pre-commit...")
+            subprocess.check_call(
+                [sys.executable, "-m", "pip", "install", "pre-commit"]
+            )
+        except Exception as e:
+            print(f"❌ Failed to install pre-commit: {e}")
+
+    # 5️⃣ Install pre-commit hooks
+    try:
+        run(["git", "init"])
+    except FileNotFoundError:
+        print("❌ Git is not installed or is not in your PATH.")
+        print("Please install Git from https://git-scm.com/downloads")
+        print("After installing Git, restart your terminal and try again.")
+    except Exception as e:
+        print(f"❌ Error initializing git repository: {e}")
+    try:
+        run(["pre-commit", "install"])
+    except Exception as e:
+        print(f"❌ Error setting up pre-commit hooks: {e}")
+
+    # 6️⃣ Create ENVIRONMENT folder and .env file
+    try:
+        env_folder = project_dir / "ENVIRONMENT"
+        env_folder.mkdir(parents=True, exist_ok=True)
+
+        env_file = env_folder / ".env"
+        if not env_file.exists():
+            env_file.write_text("# Environment variables\n", encoding="utf-8")
+            print(f"Created {env_file}")
+        else:
+            print(f"{env_file} already exists")
+
+        gitignore_file = env_folder / ".gitignore"
+        if not gitignore_file.exists():
+            gitignore_file.write_text("*\n", encoding="utf-8")
+            print(f"Created {gitignore_file}")
+        else:
+            print(f"{gitignore_file} already exists")
+
+    except Exception as e:
+        print(f"❌ Error creating environment files: {e}")
+
+    # 7️⃣ Create GitHub Actions workflow
+    try:
+        github_workflows = project_dir / ".github" / "workflows"
+        github_workflows.mkdir(parents=True, exist_ok=True)
+
+        precommit_workflow = github_workflows / "pre-commit.yml"
+
+        if not precommit_workflow.exists():
+            precommit_workflow.write_text(
+                """name: Pre-commit
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          pip install pre-commit
+
+      - name: Run pre-commit
+        run: |
+          pre-commit run --all-files
+""",
+                encoding="utf-8",
+            )
+            print(f"Created GitHub Actions workflow at {precommit_workflow}")
+        else:
+            print(f"{precommit_workflow} already exists")
+
+    except Exception as e:
+        print(f"❌ Error creating GitHub workflow: {e}")
+
+    # 7️⃣ Create GitHub Actions workflow
+    try:
+        precommit_workflow = project_dir / ".pre-commit-config.yaml"
+
+        if not precommit_workflow.exists():
+            precommit_workflow.write_text(
+                """repos:
+ - repo: https://github.com/kynan/nbstripout
+   rev: 0.6.1
+   hooks:
+     - id: nbstripout
+ - repo: https://github.com/Yelp/detect-secrets
+   rev: v1.5.0
+   hooks:
+     - id: detect-secrets
+ - repo: https://github.com/psf/black
+   rev: 24.4.2
+   hooks:
+     - id: black
+""",
+                encoding="utf-8",
+            )
+            print(f"Created pre-commit configuration at {precommit_workflow}")
+        else:
+            print(f"{precommit_workflow} already exists")
+
+    except Exception as e:
+        print(f"❌ Error creating pre-commit configuration: {e}")
+
+    print("✅ Project environment setup complete")

dspackagefolder/main.py

@@ -0,0 +1,46 @@
+from pathlib import Path
+import shutil
+import importlib.resources as resources
+import sys
+from dspackagefolder.install_precommit_and_create_env import setup_project_environment
+
+
+def create_ds_folder(folder_name: str = "dsframework"):
+    # 1️⃣ Create destination folder
+    # 2️⃣ Locate the template folder INSIDE the package
+    with resources.path("dspackagefolder", "template") as template_path:
+        target = Path(folder_name)
+
+        if target.exists():
+            print(f"Folder already exists: {target}")
+            return
+
+        # 3️⃣ Copy template into destination
+        shutil.copytree(template_path, target)
+
+    print(f"Project created at: {target.resolve()}")
+    # 4️⃣ Setup project environment
+    setup_project_environment(target)
+    print("🎉 Setup complete! Happy coding!")
+
+
+def create_ds_folder_cli():
+    # 1️⃣ Create destination folder
+    # dest_root = Path("dspackagefolder")
+    # dest_root.mkdir(exist_ok=True)
+    project_name = sys.argv[1] if len(sys.argv) > 1 else "data_science_project"
+    # 2️⃣ Locate the template folder INSIDE the package
+    with resources.path("dspackagefolder", "template") as template_path:
+        target = Path(project_name)
+
+        if target.exists():
+            print(f"Folder already exists: {target}")
+            return
+
+        # 3️⃣ Copy template into destination
+        shutil.copytree(template_path, target)
+
+    print(f"Project created at: {target.resolve()}")
+    # 4️⃣ Setup project environment
+    setup_project_environment(project_name)
+    print("🎉 Setup complete! Happy coding!")

dspackagefolder/template/.config/example.toml

@@ -0,0 +1 @@
+"data" = 3

dspackagefolder/template/.gitignore

@@ -0,0 +1,18 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+
+
+# JUPYTER NOTEBOOKS
+.ipynb_checkpoints/
+
+
+# Environment variables
+.env

dspackagefolder/template/.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+ - repo: https://github.com/kynan/nbstripout
+   rev: 0.6.1
+   hooks:
+     - id: nbstripout
+ - repo: https://github.com/Yelp/detect-secrets
+   rev: v1.5.0
+   hooks:
+     - id: detect-secrets
+ - repo: https://github.com/psf/black
+   rev: 24.4.2
+   hooks:
+     - id: black

dspackagefolder/template/.python-version

@@ -0,0 +1 @@
+3.13

dspackagefolder/template/Dockerfile

@@ -0,0 +1,33 @@
+# Dockerfile
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Install uv and Jupyter
+RUN pip install --no-cache-dir uv jupyterlab notebook
+
+CMD ["bash"]
+
+
+## Build the Docker image
+# Run the following command from the folder containing this Dockerfile:
+# docker build -t my-dev-env .
+
+## Run the container (mount your local project directory)
+# Linux/macOS:
+# docker run -it -p 8888:8888 -v $(pwd):/app my-dev-env bash
+#
+# Windows (Command Prompt):
+# docker run -it -p 8888:8888 -v %cd%:/app my-dev-env cmd
+
+## Start Jupyter Notebook inside the container
+# jupyter notebook --ip=0.0.0.0 --port=8888 --no-browser --allow-root
+#
+# Then open your browser and go to:
+# http://localhost:8888
+
+## Restart an existing container
+# docker restart <container_name_or_id>
+
+## Reconnect to a running container
+# docker exec -it <container_name_or_id> bash

dspackagefolder/template/README.md

@@ -0,0 +1,13 @@
+## TO USE THIS FOLDER:
+run the install_precommit_and_create_env.py first
+run uv add requirements.txt
+
+
+
+## main_dashboard use and running
+main_dashboard.py is used for visualizing the dashboard of the data. 
+you run it by uv run streamlit run main_dashboard.py
+
+# WHAT TO DO NEXT
+I need to show examples of how to connect the data/sql/validation/logs/config/src
+

dspackagefolder/template/checklist/.gitignore

@@ -0,0 +1 @@
+!*/

dspackagefolder/template/data/.gitkeep

@@ -0,0 +1 @@
+!*/

dspackagefolder/template/data/INPUT/.gitkeep

@@ -0,0 +1 @@
+!*/

dspackagefolder/template/data/OUTPUT/.gitkeep

@@ -0,0 +1 @@
+!*/

dspackagefolder/template/logs/.gitkeep

@@ -0,0 +1,2 @@
+!*/
+*.log

dspackagefolder/template/main.py

@@ -0,0 +1,60 @@
+##### This is a template for making src available in any folder.
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+from src.utils.logging_config import setup_logger
+from src.utils.config_loader import load_config
+
+###### Set up logger
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+from src.utils.logging_config import setup_logger
+
+logger = setup_logger(__name__, base_filename="pipeline")
+
+logger.info("Started run")
+logger.error("Failure occurred")
+
+# logging = setup_logging()
+
+
+### Load config
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+from src.utils.config_loader import load_config
+
+config = load_config()
+print(config)
+
+
+### Load env variables
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+from src.utils.env_loader import load_env_from_root
+
+env_path = load_env_from_root()
+print(f"Loaded env variables from: {env_path}")
+##import os
+##os.getenv("YOUR_ENV_VAR")
+
+### Example usage of SQLCRUD
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+from src.sqlcrud import SQLServerCRUD
+
+connection_string = "DRIVER={ODBC Driver 17 for SQL Server};SERVER=your_server;DATABASE=your_db;UID=your_user;PWD=your_password"
+sql_crud = SQLServerCRUD(connection_string)
+sql_crud.connect()
+sql_crud.build_query("sql/query.sql", params={"param1": "value1"})
+df = sql_crud.read_data()
+print(df)
+sql_crud.close_connection()

dspackagefolder/template/models/.gitignore

@@ -0,0 +1 @@
+!*/

dspackagefolder/template/requirements.txt

@@ -0,0 +1,6 @@
+# toml
+# streamlit
+# pydantic
+# python-dotenv
+# sqlalchemy
+# pandas

dspackagefolder/template/sql/query.sql

@@ -0,0 +1 @@
+

dspackagefolder/template/src/sqlcrud.py

@@ -0,0 +1,114 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Dict, Any
+
+
+class SQLCRUD(ABC):
+    """
+    Abstract base class for SQL CRUD operations.
+    """
+
+    def _init_(self, connection_params: dict):
+        self.connection_params = connection_params
+        self.connection = None
+        self.query = None
+
+    @abstractmethod
+    def connect(self):
+        """
+        Establish a database connection.
+        """
+        pass
+
+    @abstractmethod
+    def build_query(self, sql_file_path: str, params: Optional[Dict[str, Any]] = None):
+        """
+        Build SQL query from a .sql file and optional parameters.
+        """
+        pass
+
+    @abstractmethod
+    def read_data(self):
+        """
+        Execute the built query and return the results.
+        """
+        pass
+
+    def close_connection(self):
+        """
+        Close the database connection if open.
+        """
+        if self.connection:
+            self.connection.close()
+            self.connection = None
+
+
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import URL
+from typing import Optional, Dict, Any
+
+
+class SQLServerCRUD(SQLCRUD):
+    """
+    Concrete implementation of SQLCRUD for SQL Server using SQLAlchemy.
+    Designed for NMTR training and imputation queries.
+    """
+
+    def _init_(self, connection_string: str):
+        super()._init_({"connection_string": connection_string})
+        self.engine = None
+        self.params: Dict[str, Any] = {}
+
+
+def connect(self):
+    connection_url = URL.create(
+        "mssql+pyodbc",
+        query={"odbc_connect": self.connection_params["connection_string"]},
+    )
+    self.engine = create_engine(connection_url)
+
+
+def build_query(
+    self,
+    sql: Optional[str] = None,
+    sql_file_path: Optional[str] = None,
+    params: Optional[Dict[str, Any]] = None,
+):
+    """
+    Build query from either:
+    - raw SQL string
+    - .sql file path
+    """
+
+    if sql_file_path:
+        with open(sql_file_path, "r") as file:
+            sql = file.read()
+
+    if not sql:
+        raise ValueError("Either sql or sql_file_path must be provided")
+
+    self.query = sql
+    self.params = params or {}
+
+
+def read_data(self):
+    import pandas as pd
+
+    """
+ Executes the query. For SELECT returns results,
+ for DDL (like SELECT INTO) just executes.
+ """
+    if not self.engine:
+        raise ValueError("Connection not established")
+
+    if not self.query:
+        raise ValueError("Query not built")
+
+    with self.engine.begin() as conn:
+        result = conn.execute(text(self.query), self.params)
+
+    # Try fetching (will fail for DDL, so handle gracefully)
+    try:
+        df = pd.DataFrame(result.fetchall(), columns=result.keys())
+        return df
+    except:
+        return None

dspackagefolder/template/src/utils/config_loader.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+import tomllib  # Python 3.11+
+
+# use: import tomli as tomllib   # if Python <= 3.10
+
+
+def load_config(config_name: str = ".config/example.toml") -> dict:
+    """
+    Load config.toml from project root regardless of where this is imported from.
+    """
+
+    current = Path(__file__).resolve()
+
+    for parent in [current] + list(current.parents):
+        candidate = parent / config_name
+        if candidate.exists():
+            with open(candidate, "rb") as f:
+                return tomllib.load(f)
+
+    raise FileNotFoundError(f"{config_name} not found in any parent directory")

dspackagefolder/template/src/utils/env_loader.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+from dotenv import load_dotenv
+
+
+def load_env_from_root(env_subpath: str = "ENVIRONMENT/.env") -> Path:
+    """
+    Search upward from this file to find ENVIRONMENT/.env and load it.
+
+    Returns the path of the loaded env file.
+    """
+
+    current = Path(__file__).resolve()
+
+    for parent in [current] + list(current.parents):
+        env_path = parent / env_subpath
+        if env_path.exists():
+            load_dotenv(env_path)
+            return env_path
+
+    raise FileNotFoundError(f"{env_subpath} not found in any parent directory")

dspackagefolder/template/src/utils/logging_config.py

@@ -0,0 +1,48 @@
+import logging
+from pathlib import Path
+from datetime import datetime
+
+
+def setup_logger(
+    name: str = "app_logger",
+    log_dir: str = "logs",
+    base_filename: str = "app",
+    level: int = logging.INFO,
+) -> logging.Logger:
+    """
+    Configure logger with timestamped log filename and datetime in messages.
+    """
+
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+
+    # Prevent duplicate handlers if called multiple times
+    if logger.handlers:
+        return logger
+
+    # ---- create timestamped filename ----
+    timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    project_root = Path(__file__).resolve().parents[2]
+    log_path = project_root / "logs"
+    print(f"Log file will be saved to: {log_path / f'{base_filename}_{timestamp}.log'}")
+    log_path.mkdir(parents=True, exist_ok=True)
+
+    log_file = log_path / f"{base_filename}_{timestamp}.log"
+
+    # ---- formatter with datetime ----
+    formatter = logging.Formatter(
+        fmt="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    # ---- console handler ----
+    ch = logging.StreamHandler()
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+
+    # ---- file handler ----
+    fh = logging.FileHandler(log_file)
+    fh.setFormatter(formatter)
+    logger.addHandler(fh)
+
+    return logger

dspackagefolder/template/template_notebook.ipynb

@@ -0,0 +1,384 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "530631d3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#uv add ipykernel\n",
+    "#uv run -m ipykernel install --user --name=.venv --display-name \"Python (.venv)\"\n",
+    "\n",
+    "## The code above is used to add the current virtual environment as a kernel in Jupyter Notebook.\n",
+    "## Run the first command in a terminal or command prompt to add ipykernel to your virtual environment, and then run the second command to install the kernel with a specific name and display name. This allows you to select this virtual environment as a kernel when working in Jupyter Notebook."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a585775f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "### the above code is used to automatically reload modules in Jupyter Notebook when they are modified. This is useful for development and testing, as it allows you to see changes in your code without having to restart the kernel or manually reload the modules. The `%load_ext autoreload` command loads the autoreload extension, and `%autoreload 2` sets the autoreload mode to automatically reload all modules (except those excluded by `%aimport`) every time before executing a new line of code."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "abd53d26-89fa-4cfc-94fd-a93f30d73151",
+   "metadata": {},
+   "source": [
+    "# INTRODUCTION ANALYSIS OF DATA"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5fedeefb-c662-42a3-8a5f-9e29376685ba",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "58680534-ea5d-4ebb-803d-1ee09c6a0e1b",
+   "metadata": {},
+   "source": [
+    "# IMPORT LIBRARIES AND CODES FROM SRC"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "883ba69b-2973-48ea-b7e2-70b342e2c42d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1a58ac2e-aa62-4963-a695-a0fa5302d3bd",
+   "metadata": {
+    "jp-MarkdownHeadingCollapsed": true
+   },
+   "source": [
+    "# IMPORT THE DATA"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1a95314a-23ed-4663-8a6a-f8cbe46e47d6",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fcfbfeb5-cd8e-4be9-aa24-5155a5a5ebeb",
+   "metadata": {},
+   "source": [
+    "# VALIDATE/EDA OF THE DATA"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9c453db0-fc2c-485c-a448-79bcc35620ce",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "899ef0f2-18ce-4fdc-8e38-0a12c8b22464",
+   "metadata": {},
+   "source": [
+    "# CLEANING"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "369813bf-5d91-4ed2-ad02-a00a9ffdfa68",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "73d24e6d-503c-4cfd-8686-7548bbcd0401",
+   "metadata": {},
+   "source": [
+    "# VISUALIZATION"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6e7d2c3d-de56-4f28-aeb6-93d489051fe9",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0c082268-f0b8-496c-85f0-1b0c4d06f8c8",
+   "metadata": {
+    "jp-MarkdownHeadingCollapsed": true
+   },
+   "source": [
+    "## STATISTICAL ANALYSIS"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e689d695-eaa0-4697-b278-be8f9a250120",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aa4f7188-1903-4276-ae43-36fb65b911bd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "35e9808b-0fdb-45f6-a3df-45778f684a91",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "be07857e-8425-4bdb-80d8-b6d53e23c2cb",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1c4d04c1-e619-464a-8781-c764a403e5e5",
+   "metadata": {},
+   "source": [
+    "## MODELLING AND/OR FORECASTING"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "06415f8a-e1ad-4faf-ab24-1708915d7cd5",
+   "metadata": {},
+   "source": [
+    "### PREPROCESSING"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "db6408f5-e3db-4441-a5bd-5256c07fdf48",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8ce3a1ba-98e2-4957-9689-83cfa627b599",
+   "metadata": {},
+   "source": [
+    "### FEATURE ENGINEERING"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cb311c49-2c67-47ce-9146-89d84cd569c2",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1495385c-9fc1-4175-8006-d58189db5db2",
+   "metadata": {},
+   "source": [
+    "### MODEL TRAINING"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "de9b8db8-57b9-4370-bb25-943221485c6c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "50f3cb28-d654-48ac-b529-f7e41e37bd73",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b9e435c9-42a3-4f50-ab59-09b99b6cf759",
+   "metadata": {},
+   "source": [
+    "### PERFORMANCE EVALUATION"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1c16a623-ac1b-454b-95b5-258e93f1c860",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "579b2748-0cfb-4eec-8ce0-9ef3771f2c74",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e806bf62-16d5-498e-b2b1-ddf85abc233c",
+   "metadata": {},
+   "source": [
+    "### PREDICTION"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3a855cee-98ab-4b7a-9b75-7f5819e06bb1",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cd0badb5-511f-4864-8f09-191e20aeac51",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "97f94129-2514-443f-bc0b-4f0e0c39f8db",
+   "metadata": {},
+   "source": [
+    "### SAVE THE MODEL"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5ef3ab9e-2d76-41d4-9243-e32aa8d97a61",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bed9b348-1185-4be7-a51d-b2f100ca0bf4",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "87c3ab8d-be4d-454b-ae0b-0856dec7b327",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1bed1e08-6a8e-4bcc-a333-af7aa6e909df",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f3d2aea7-5e4e-4c6d-8a71-e11c454e25c3",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a9810bee-c9af-4de9-a617-a73ad651c170",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "02a8d509-6651-4fbc-b75f-83c3588c9493",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "67e5117d-e4c8-4b70-b4b6-902bbae5bc45",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "92dc84de-7353-4e62-9a86-69e553f52fe0",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}