dataset-holder 0.1.0__tar.gz

dataset_holder-0.1.0/PKG-INFO

@@ -0,0 +1,30 @@
+Metadata-Version: 2.4
+Name: dataset_holder
+Version: 0.1.0
+Summary: A central repository to hold, manage, monitor, scan, and ingest Python datasets.
+Author-email: Your Name <your.email@example.com>
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: pandas
+Requires-Dist: openpyxl
+Requires-Dist: pyarrow
+
+# Dataset Holder
+
+A lightweight Python library to manage, monitor, scan, and ingest data sets easily.
+
+## Installation
+```bash
+pip install dataset_holder
+```
+
+## Quick Start
+```python
+from dataset_holder import DatasetHolder
+holder = DatasetHolder()
+holder.load_file("my_data.csv")
+print(holder.get_memory_report())
+```

dataset_holder-0.1.0/README.md

@@ -0,0 +1,16 @@
+# Dataset Holder
+
+A lightweight Python library to manage, monitor, scan, and ingest data sets easily.
+
+## Installation
+```bash
+pip install dataset_holder
+```
+
+## Quick Start
+```python
+from dataset_holder import DatasetHolder
+holder = DatasetHolder()
+holder.load_file("my_data.csv")
+print(holder.get_memory_report())
+```

dataset_holder-0.1.0/dataset_holder/__init__.py

@@ -0,0 +1 @@
+from .manager import DatasetHolder

dataset_holder-0.1.0/dataset_holder/dataset_holder.py

@@ -0,0 +1,153 @@
+import sys
+import pickle
+import json
+from pathlib import Path
+import pandas as pd
+
+class DatasetHolder:
+    """A central repository to hold, manage, monitor, scan, and ingest Python datasets."""
+    
+    def __init__(self):
+        self._store = {}
+        # Supported extensions and their human-readable formats
+        self._dataset_extensions = {
+            '.csv': 'Comma-Separated Values (CSV)',
+            '.xlsx': 'Excel Spreadsheet (XLSX)',
+            '.xls': 'Excel Spreadsheet (XLS)',
+            '.json': 'JSON Data File',
+            '.parquet': 'Apache Parquet',
+            '.pkl': 'Python Pickle File',
+            '.pickle': 'Python Pickle File'
+        }
+
+    def add(self, name: str, data, metadata: dict = None) -> None:
+        """Add or overwrite a dataset in the holder."""
+        self._store[name] = {
+            "data": data,
+            "metadata": metadata or {}
+        }
+
+    def get(self, name: str):
+        """Retrieve the raw data object by name."""
+        if name not in self._store:
+            raise KeyError(f"Dataset '{name}' not found.")
+        return self._store[name]["data"]
+
+    def get_info(self, name: str) -> dict:
+        """Retrieve metadata for a specific dataset."""
+        if name not in self._store:
+            raise KeyError(f"Dataset '{name}' not found.")
+        return self._store[name]["metadata"]
+
+    def list_datasets(self) -> list:
+        """List all stored dataset names."""
+        return list(self._store.keys())
+
+    def remove(self, name: str) -> None:
+        """Remove a dataset from the holder."""
+        if name in self._store:
+            del self._store[name]
+        else:
+            raise KeyError(f"Dataset '{name}' not found.")
+
+    def clear(self) -> None:
+        """Clear all datasets."""
+        self._store.clear()
+
+    def save_to_disk(self, filepath: str) -> None:
+        """Save the entire dataset holder to a file."""
+        with open(filepath, 'wb') as f:
+            pickle.dump(self._store, f)
+        print(f"Successfully saved {len(self._store)} dataset(s) to {filepath}")
+
+    def load_from_disk(self, filepath: str) -> None:
+        """Load datasets from a file, merging or overwriting current datasets."""
+        path = Path(filepath)
+        if not path.exists():
+            raise FileNotFoundError(f"No file found at {filepath}")
+            
+        with open(filepath, 'rb') as f:
+            loaded_store = pickle.load(f)
+            
+        self._store.update(loaded_store)
+        print(f"Successfully loaded {len(loaded_store)} dataset(s) from {filepath}")
+
+    def _calculate_size(self, data) -> int:
+        """Helper method to accurately calculate memory size in bytes."""
+        # Use deep evaluation if it is a Pandas object to catch object/string column sizes
+        if hasattr(data, 'memory_usage'):
+            return int(data.memory_usage(deep=True).sum())
+        return sys.getsizeof(data)
+
+    def get_memory_report(self) -> dict:
+        """Returns a breakdown of memory usage for all stored datasets."""
+        report = {}
+        total_bytes = 0
+        for name, content in self._store.items():
+            size_bytes = self._calculate_size(content["data"])
+            total_bytes += size_bytes
+            report[name] = self._format_size(size_bytes)
+        report["TOTAL_LIBRARY_SIZE"] = self._format_size(total_bytes)
+        return report
+
+    def _format_size(self, size_bytes: int) -> str:
+        """Helper method to format bytes into KB, MB, or GB."""
+        for unit in ['Bytes', 'KB', 'MB', 'GB']:
+            if size_bytes < 1024:
+                return f"{size_bytes:.2f} {unit}"
+            size_bytes /= 1024
+        return f"{size_bytes:.2f} TB"
+
+    def scan_directory(self, folder_path: str) -> dict:
+        """Scans a directory and identifies which files are valid datasets."""
+        path = Path(folder_path)
+        if not path.exists() or not path.is_dir():
+            raise NotADirectoryError(f"'{folder_path}' is not a valid directory.")
+
+        results = {"datasets_found": {}, "total_datasets_count": 0}
+
+        for file in path.iterdir():
+            if file.is_file():
+                ext = file.suffix.lower()
+                if ext in self._dataset_extensions:
+                    results["datasets_found"][file.name] = {
+                        "type": self._dataset_extensions[ext],
+                        "file_size": self._format_size(file.stat().st_size),
+                        "full_path": str(file.resolve())
+                    }
+                    results["total_datasets_count"] += 1
+        return results
+
+    def load_file(self, file_path: str, dataset_name: str = None) -> None:
+        """Automatically reads a single dataset file into memory based on its extension."""
+        path = Path(file_path)
+        if not path.exists():
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        ext = path.suffix.lower()
+        name = dataset_name or path.stem
+        metadata = {"source_file": str(path.resolve()), "file_type": ext}
+
+        try:
+            if ext == '.csv':
+                data = pd.read_csv(path)
+            elif ext in ['.xlsx', '.xls']:
+                data = pd.read_excel(path)
+            elif ext == '.parquet':
+                data = pd.read_parquet(path)
+            elif ext in ['.pkl', '.pickle']:
+                with open(path, 'rb') as f:
+                    data = pickle.load(f)
+            elif ext == '.json':
+                with open(path, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+            else:
+                raise ValueError(f"Unsupported file format: {ext}")
+
+            self.add(name, data, metadata=metadata)
+            print(f"Successfully ingested dataset '{name}' into library.")
+
+        except json.JSONDecodeError:
+            print(f"Error: '{name}' is not a valid JSON file. It might be empty or formatted incorrectly.")
+        except Exception as e:
+            print(f"Failed to read dataset '{name}': {e}")

dataset_holder-0.1.0/dataset_holder.egg-info/PKG-INFO

@@ -0,0 +1,30 @@
+Metadata-Version: 2.4
+Name: dataset_holder
+Version: 0.1.0
+Summary: A central repository to hold, manage, monitor, scan, and ingest Python datasets.
+Author-email: Your Name <your.email@example.com>
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: pandas
+Requires-Dist: openpyxl
+Requires-Dist: pyarrow
+
+# Dataset Holder
+
+A lightweight Python library to manage, monitor, scan, and ingest data sets easily.
+
+## Installation
+```bash
+pip install dataset_holder
+```
+
+## Quick Start
+```python
+from dataset_holder import DatasetHolder
+holder = DatasetHolder()
+holder.load_file("my_data.csv")
+print(holder.get_memory_report())
+```

dataset_holder-0.1.0/dataset_holder.egg-info/SOURCES.txt

@@ -0,0 +1,9 @@
+README.md
+pyproject.toml
+dataset_holder/__init__.py
+dataset_holder/dataset_holder.py
+dataset_holder.egg-info/PKG-INFO
+dataset_holder.egg-info/SOURCES.txt
+dataset_holder.egg-info/dependency_links.txt
+dataset_holder.egg-info/requires.txt
+dataset_holder.egg-info/top_level.txt

dataset_holder-0.1.0/dataset_holder.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

dataset_holder-0.1.0/dataset_holder.egg-info/requires.txt

@@ -0,0 +1,3 @@
+pandas
+openpyxl
+pyarrow

dataset_holder-0.1.0/dataset_holder.egg-info/top_level.txt

@@ -0,0 +1 @@
+dataset_holder

dataset_holder-0.1.0/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "dataset_holder"
+version = "0.1.0"
+authors = [
+  { name="Your Name", email="your.email@example.com" },
+]
+description = "A central repository to hold, manage, monitor, scan, and ingest Python datasets."
+readme = "README.md"
+requires-python = ">=3.8"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "pandas",
+    "openpyxl",
+    "pyarrow",
+]

dataset_holder-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+