dsfolder 0.0.0.1__tar.gz

dsfolder-0.0.0.1/MANIFEST.in

@@ -0,0 +1,2 @@
+recursive-include dspackagefolder/template *
+recursive-include dspackagefolder/template .*

dsfolder-0.0.0.1/PKG-INFO

@@ -0,0 +1,45 @@
+Metadata-Version: 2.1
+Name: dsfolder
+Version: 0.0.0.1
+Summary: Add your description here
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+## Data Science Folder Structure Setup
+
+1. This was developed for creating a Python folder to start a data science project.
+
+2. It includes preconfigured connections to different systems:
+   - uv  
+   - git  
+   - GitHub workflow  
+   - .gitignore  
+
+3. It also provides small code snippets for connecting to other systems.
+
+4. Below are some of the systems included, along with additional setup steps to get them working:
+   - logging  
+   - SQL  
+   - src  
+   - config  
+   - pre-commit hook  
+   - .gitignore  
+   - git  
+   - GitHub workflow  
+   - uv  
+   - env  
+   - validation  
+   - testing  
+   - requirements.txt  
+     - Install dependencies with: `uv install requirements.txt`
+
+5. Install the package:
+   ```bash
+   pip install datascience-folder-structure
+6. Create a folder structure:
+   ```bash
+   createdatasciencefolder NameOfProject
+
+   or
+   ```bash
+   dsf NameOfProject 

dsfolder-0.0.0.1/README.md

@@ -0,0 +1,38 @@
+## Data Science Folder Structure Setup
+
+1. This was developed for creating a Python folder to start a data science project.
+
+2. It includes preconfigured connections to different systems:
+   - uv  
+   - git  
+   - GitHub workflow  
+   - .gitignore  
+
+3. It also provides small code snippets for connecting to other systems.
+
+4. Below are some of the systems included, along with additional setup steps to get them working:
+   - logging  
+   - SQL  
+   - src  
+   - config  
+   - pre-commit hook  
+   - .gitignore  
+   - git  
+   - GitHub workflow  
+   - uv  
+   - env  
+   - validation  
+   - testing  
+   - requirements.txt  
+     - Install dependencies with: `uv install requirements.txt`
+
+5. Install the package:
+   ```bash
+   pip install datascience-folder-structure
+6. Create a folder structure:
+   ```bash
+   createdatasciencefolder NameOfProject
+
+   or
+   ```bash
+   dsf NameOfProject 

dsfolder-0.0.0.1/dsfolder.egg-info/PKG-INFO

@@ -0,0 +1,45 @@
+Metadata-Version: 2.1
+Name: dsfolder
+Version: 0.0.0.1
+Summary: Add your description here
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+## Data Science Folder Structure Setup
+
+1. This was developed for creating a Python folder to start a data science project.
+
+2. It includes preconfigured connections to different systems:
+   - uv  
+   - git  
+   - GitHub workflow  
+   - .gitignore  
+
+3. It also provides small code snippets for connecting to other systems.
+
+4. Below are some of the systems included, along with additional setup steps to get them working:
+   - logging  
+   - SQL  
+   - src  
+   - config  
+   - pre-commit hook  
+   - .gitignore  
+   - git  
+   - GitHub workflow  
+   - uv  
+   - env  
+   - validation  
+   - testing  
+   - requirements.txt  
+     - Install dependencies with: `uv install requirements.txt`
+
+5. Install the package:
+   ```bash
+   pip install datascience-folder-structure
+6. Create a folder structure:
+   ```bash
+   createdatasciencefolder NameOfProject
+
+   or
+   ```bash
+   dsf NameOfProject 

dsfolder-0.0.0.1/dsfolder.egg-info/SOURCES.txt

@@ -0,0 +1,75 @@
+MANIFEST.in
+README.md
+pyproject.toml
+setup.py
+./dspackagefolder/__init__.py
+./dspackagefolder/install_precommit_and_create_env.py
+./dspackagefolder/main.py
+./dspackagefolder/template/.gitignore
+./dspackagefolder/template/.pre-commit-config.yaml
+./dspackagefolder/template/.python-version
+./dspackagefolder/template/Dockerfile
+./dspackagefolder/template/README.md
+./dspackagefolder/template/main.py
+./dspackagefolder/template/main_dashboard.py
+./dspackagefolder/template/requirements.txt
+./dspackagefolder/template/template_notebook.ipynb
+./dspackagefolder/template/.config/.gitkeep
+./dspackagefolder/template/.config/example.toml
+./dspackagefolder/template/EXAMPLE/example.py
+./dspackagefolder/template/checklist/.gitignore
+./dspackagefolder/template/checklist/GENERIC_CHECKLIST.xlsx
+./dspackagefolder/template/checklist/ML_CHECKLIST.xlsx
+./dspackagefolder/template/checklist/STATISTICS_CHECKLIST.xlsx
+./dspackagefolder/template/data/.gitkeep
+./dspackagefolder/template/data/INPUT/.gitkeep
+./dspackagefolder/template/data/OUTPUT/.gitkeep
+./dspackagefolder/template/logs/.gitkeep
+./dspackagefolder/template/models/.gitignore
+./dspackagefolder/template/sql/.gitkeep
+./dspackagefolder/template/sql/query.sql
+./dspackagefolder/template/src/.gitkeep
+./dspackagefolder/template/src/__init__.py
+./dspackagefolder/template/src/sqlcrud.py
+./dspackagefolder/template/src/utils/config_loader.py
+./dspackagefolder/template/src/utils/env_loader.py
+./dspackagefolder/template/src/utils/logging_config.py
+./dspackagefolder/template/unit_testing/.gitkeep
+./dspackagefolder/template/validation/.gitkeep
+dsfolder.egg-info/PKG-INFO
+dsfolder.egg-info/SOURCES.txt
+dsfolder.egg-info/dependency_links.txt
+dsfolder.egg-info/entry_points.txt
+dsfolder.egg-info/requires.txt
+dsfolder.egg-info/top_level.txt
+dspackagefolder/template/.gitignore
+dspackagefolder/template/.pre-commit-config.yaml
+dspackagefolder/template/.python-version
+dspackagefolder/template/Dockerfile
+dspackagefolder/template/README.md
+dspackagefolder/template/main.py
+dspackagefolder/template/main_dashboard.py
+dspackagefolder/template/requirements.txt
+dspackagefolder/template/template_notebook.ipynb
+dspackagefolder/template/.config/.gitkeep
+dspackagefolder/template/.config/example.toml
+dspackagefolder/template/EXAMPLE/example.py
+dspackagefolder/template/checklist/.gitignore
+dspackagefolder/template/checklist/GENERIC_CHECKLIST.xlsx
+dspackagefolder/template/checklist/ML_CHECKLIST.xlsx
+dspackagefolder/template/checklist/STATISTICS_CHECKLIST.xlsx
+dspackagefolder/template/data/.gitkeep
+dspackagefolder/template/data/INPUT/.gitkeep
+dspackagefolder/template/data/OUTPUT/.gitkeep
+dspackagefolder/template/logs/.gitkeep
+dspackagefolder/template/models/.gitignore
+dspackagefolder/template/sql/.gitkeep
+dspackagefolder/template/sql/query.sql
+dspackagefolder/template/src/.gitkeep
+dspackagefolder/template/src/__init__.py
+dspackagefolder/template/src/sqlcrud.py
+dspackagefolder/template/src/utils/config_loader.py
+dspackagefolder/template/src/utils/env_loader.py
+dspackagefolder/template/src/utils/logging_config.py
+dspackagefolder/template/unit_testing/.gitkeep
+dspackagefolder/template/validation/.gitkeep

dsfolder-0.0.0.1/dsfolder.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

dsfolder-0.0.0.1/dsfolder.egg-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+createdatasciencefolder = dspackagefolder.main:create_ds_folder_cli
+dsf = dspackagefolder.main:create_ds_folder_cli

dsfolder-0.0.0.1/dsfolder.egg-info/requires.txt

@@ -0,0 +1,4 @@
+path>=17.1.1
+python-dotenv>=1.2.1
+sqlalchemy>=2.0.48
+toml>=0.10.2

dsfolder-0.0.0.1/dsfolder.egg-info/top_level.txt

@@ -0,0 +1 @@
+dspackagefolder

dsfolder-0.0.0.1/dspackagefolder/__init__.py

@@ -0,0 +1 @@
+from .main import create_ds_folder

dsfolder-0.0.0.1/dspackagefolder/install_precommit_and_create_env.py

@@ -0,0 +1,184 @@
+## install uv
+## uv add requirements.txt
+## install pre-commit
+## pre-commit install
+## creeate env file into the new folder
+
+
+import subprocess
+import sys
+from pathlib import Path
+
+
+def setup_project_environment(project_dir: str | Path = "."):
+    project_dir = Path(project_dir).resolve()
+
+    try:
+        if not project_dir.exists():
+            raise FileNotFoundError(f"{project_dir} does not exist")
+    except Exception as e:
+        print(f"❌ Invalid project directory: {e}")
+        return
+
+    def run(cmd: list[str]):
+        try:
+            subprocess.check_call(cmd, cwd=project_dir)
+        except subprocess.CalledProcessError as e:
+            print(f"❌ Command failed: {' '.join(cmd)}\n{e}")
+        except FileNotFoundError as e:
+            print(f"❌ Command not found: {cmd[0]}\n{e}")
+        except Exception as e:
+            print(f"❌ Unexpected error running {' '.join(cmd)}: {e}")
+
+    # 1️⃣ Install uv if missing
+    try:
+        subprocess.check_call([sys.executable, "-m", "uv", "--version"])
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        try:
+            print("Installing uv...")
+            subprocess.check_call([sys.executable, "-m", "pip", "install", "uv"])
+        except Exception as e:
+            print(f"❌ Failed to install uv: {e}")
+
+    # 2️⃣ Initialise uv project if pyproject.toml is missing
+    try:
+        pyproject = project_dir / "pyproject.toml"
+        if not pyproject.exists():
+            print("Initialising uv project...")
+            run(["uv", "init"])
+    except Exception as e:
+        print(f"❌ Error during uv init: {e}")
+
+    # 3️⃣ Add requirements.txt using uv
+    try:
+        requirements = project_dir / "requirements.txt"
+        if requirements.exists():
+            run(["uv", "add", "-r", "requirements.txt"])
+        else:
+            print("requirements.txt not found — skipping uv add")
+    except Exception as e:
+        print(f"❌ Error adding requirements: {e}")
+
+    # 4️⃣ Install pre-commit if missing
+    try:
+        subprocess.check_call(["pre-commit", "--version"])
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        try:
+            print("Installing pre-commit...")
+            subprocess.check_call(
+                [sys.executable, "-m", "pip", "install", "pre-commit"]
+            )
+        except Exception as e:
+            print(f"❌ Failed to install pre-commit: {e}")
+
+    # 5️⃣ Install pre-commit hooks
+    try:
+        run(["git", "init"])
+    except FileNotFoundError:
+        print("❌ Git is not installed or is not in your PATH.")
+        print("Please install Git from https://git-scm.com/downloads")
+        print("After installing Git, restart your terminal and try again.")
+    except Exception as e:
+        print(f"❌ Error initializing git repository: {e}")
+    try:
+        run(["pre-commit", "install"])
+    except Exception as e:
+        print(f"❌ Error setting up pre-commit hooks: {e}")
+
+    # 6️⃣ Create ENVIRONMENT folder and .env file
+    try:
+        env_folder = project_dir / "ENVIRONMENT"
+        env_folder.mkdir(parents=True, exist_ok=True)
+
+        env_file = env_folder / ".env"
+        if not env_file.exists():
+            env_file.write_text("# Environment variables\n", encoding="utf-8")
+            print(f"Created {env_file}")
+        else:
+            print(f"{env_file} already exists")
+
+        gitignore_file = env_folder / ".gitignore"
+        if not gitignore_file.exists():
+            gitignore_file.write_text("*\n", encoding="utf-8")
+            print(f"Created {gitignore_file}")
+        else:
+            print(f"{gitignore_file} already exists")
+
+    except Exception as e:
+        print(f"❌ Error creating environment files: {e}")
+
+    # 7️⃣ Create GitHub Actions workflow
+    try:
+        github_workflows = project_dir / ".github" / "workflows"
+        github_workflows.mkdir(parents=True, exist_ok=True)
+
+        precommit_workflow = github_workflows / "pre-commit.yml"
+
+        if not precommit_workflow.exists():
+            precommit_workflow.write_text(
+                """name: Pre-commit
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          pip install pre-commit
+
+      - name: Run pre-commit
+        run: |
+          pre-commit run --all-files
+""",
+                encoding="utf-8",
+            )
+            print(f"Created GitHub Actions workflow at {precommit_workflow}")
+        else:
+            print(f"{precommit_workflow} already exists")
+
+    except Exception as e:
+        print(f"❌ Error creating GitHub workflow: {e}")
+
+    # 7️⃣ Create GitHub Actions workflow
+    try:
+        precommit_workflow = project_dir / ".pre-commit-config.yaml"
+
+        if not precommit_workflow.exists():
+            precommit_workflow.write_text(
+                """repos:
+ - repo: https://github.com/kynan/nbstripout
+   rev: 0.6.1
+   hooks:
+     - id: nbstripout
+ - repo: https://github.com/Yelp/detect-secrets
+   rev: v1.5.0
+   hooks:
+     - id: detect-secrets
+ - repo: https://github.com/psf/black
+   rev: 24.4.2
+   hooks:
+     - id: black
+""",
+                encoding="utf-8",
+            )
+            print(f"Created pre-commit configuration at {precommit_workflow}")
+        else:
+            print(f"{precommit_workflow} already exists")
+
+    except Exception as e:
+        print(f"❌ Error creating pre-commit configuration: {e}")
+
+    print("✅ Project environment setup complete")

dsfolder-0.0.0.1/dspackagefolder/main.py

@@ -0,0 +1,46 @@
+from pathlib import Path
+import shutil
+import importlib.resources as resources
+import sys
+from dspackagefolder.install_precommit_and_create_env import setup_project_environment
+
+
+def create_ds_folder(folder_name: str = "dsframework"):
+    # 1️⃣ Create destination folder
+    # 2️⃣ Locate the template folder INSIDE the package
+    with resources.path("dspackagefolder", "template") as template_path:
+        target = Path(folder_name)
+
+        if target.exists():
+            print(f"Folder already exists: {target}")
+            return
+
+        # 3️⃣ Copy template into destination
+        shutil.copytree(template_path, target)
+
+    print(f"Project created at: {target.resolve()}")
+    # 4️⃣ Setup project environment
+    setup_project_environment(target)
+    print("🎉 Setup complete! Happy coding!")
+
+
+def create_ds_folder_cli():
+    # 1️⃣ Create destination folder
+    # dest_root = Path("dspackagefolder")
+    # dest_root.mkdir(exist_ok=True)
+    project_name = sys.argv[1] if len(sys.argv) > 1 else "data_science_project"
+    # 2️⃣ Locate the template folder INSIDE the package
+    with resources.path("dspackagefolder", "template") as template_path:
+        target = Path(project_name)
+
+        if target.exists():
+            print(f"Folder already exists: {target}")
+            return
+
+        # 3️⃣ Copy template into destination
+        shutil.copytree(template_path, target)
+
+    print(f"Project created at: {target.resolve()}")
+    # 4️⃣ Setup project environment
+    setup_project_environment(project_name)
+    print("🎉 Setup complete! Happy coding!")

dsfolder-0.0.0.1/dspackagefolder/template/.config/example.toml

@@ -0,0 +1 @@
+"data" = 3

dsfolder-0.0.0.1/dspackagefolder/template/.gitignore

@@ -0,0 +1,18 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+
+
+# JUPYTER NOTEBOOKS
+.ipynb_checkpoints/
+
+
+# Environment variables
+.env

dsfolder-0.0.0.1/dspackagefolder/template/.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+ - repo: https://github.com/kynan/nbstripout
+   rev: 0.6.1
+   hooks:
+     - id: nbstripout
+ - repo: https://github.com/Yelp/detect-secrets
+   rev: v1.5.0
+   hooks:
+     - id: detect-secrets
+ - repo: https://github.com/psf/black
+   rev: 24.4.2
+   hooks:
+     - id: black

dsfolder-0.0.0.1/dspackagefolder/template/.python-version

@@ -0,0 +1 @@
+3.13

dsfolder-0.0.0.1/dspackagefolder/template/Dockerfile

@@ -0,0 +1,33 @@
+# Dockerfile
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Install uv and Jupyter
+RUN pip install --no-cache-dir uv jupyterlab notebook
+
+CMD ["bash"]
+
+
+## Build the Docker image
+# Run the following command from the folder containing this Dockerfile:
+# docker build -t my-dev-env .
+
+## Run the container (mount your local project directory)
+# Linux/macOS:
+# docker run -it -p 8888:8888 -v $(pwd):/app my-dev-env bash
+#
+# Windows (Command Prompt):
+# docker run -it -p 8888:8888 -v %cd%:/app my-dev-env cmd
+
+## Start Jupyter Notebook inside the container
+# jupyter notebook --ip=0.0.0.0 --port=8888 --no-browser --allow-root
+#
+# Then open your browser and go to:
+# http://localhost:8888
+
+## Restart an existing container
+# docker restart <container_name_or_id>
+
+## Reconnect to a running container
+# docker exec -it <container_name_or_id> bash

dsfolder-0.0.0.1/dspackagefolder/template/README.md

@@ -0,0 +1,13 @@
+## TO USE THIS FOLDER:
+run the install_precommit_and_create_env.py first
+run uv add requirements.txt
+
+
+
+## main_dashboard use and running
+main_dashboard.py is used for visualizing the dashboard of the data. 
+you run it by uv run streamlit run main_dashboard.py
+
+# WHAT TO DO NEXT
+I need to show examples of how to connect the data/sql/validation/logs/config/src
+

dsfolder-0.0.0.1/dspackagefolder/template/checklist/.gitignore

@@ -0,0 +1 @@
+!*/

dsfolder-0.0.0.1/dspackagefolder/template/data/.gitkeep

@@ -0,0 +1 @@
+!*/

dsfolder-0.0.0.1/dspackagefolder/template/data/INPUT/.gitkeep

@@ -0,0 +1 @@
+!*/

dsfolder-0.0.0.1/dspackagefolder/template/data/OUTPUT/.gitkeep

@@ -0,0 +1 @@
+!*/

dsfolder-0.0.0.1/dspackagefolder/template/logs/.gitkeep

@@ -0,0 +1,2 @@
+!*/
+*.log

dsfolder-0.0.0.1/dspackagefolder/template/main.py

@@ -0,0 +1,60 @@
+##### This is a template for making src available in any folder.
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+from src.utils.logging_config import setup_logger
+from src.utils.config_loader import load_config
+
+###### Set up logger
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+from src.utils.logging_config import setup_logger
+
+logger = setup_logger(__name__, base_filename="pipeline")
+
+logger.info("Started run")
+logger.error("Failure occurred")
+
+# logging = setup_logging()
+
+
+### Load config
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+from src.utils.config_loader import load_config
+
+config = load_config()
+print(config)
+
+
+### Load env variables
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+from src.utils.env_loader import load_env_from_root
+
+env_path = load_env_from_root()
+print(f"Loaded env variables from: {env_path}")
+##import os
+##os.getenv("YOUR_ENV_VAR")
+
+### Example usage of SQLCRUD
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).resolve().parents[1]))
+from src.sqlcrud import SQLServerCRUD
+
+connection_string = "DRIVER={ODBC Driver 17 for SQL Server};SERVER=your_server;DATABASE=your_db;UID=your_user;PWD=your_password"
+sql_crud = SQLServerCRUD(connection_string)
+sql_crud.connect()
+sql_crud.build_query("sql/query.sql", params={"param1": "value1"})
+df = sql_crud.read_data()
+print(df)
+sql_crud.close_connection()

dsfolder-0.0.0.1/dspackagefolder/template/models/.gitignore

@@ -0,0 +1 @@
+!*/

dsfolder-0.0.0.1/dspackagefolder/template/requirements.txt

@@ -0,0 +1,6 @@
+# toml
+# streamlit
+# pydantic
+# python-dotenv
+# sqlalchemy
+# pandas

dsfolder-0.0.0.1/dspackagefolder/template/sql/query.sql

@@ -0,0 +1 @@
+