silver-data 0.1.0__py3-none-any.whl

silver_data/__init__.py

@@ -0,0 +1,19 @@
+from .dataset import Dataset
+from .models import (
+    DataRecord,
+    DatasetColumn,
+    DatasetReport,
+    DatasetSplit,
+    DatasetValidation,
+    Primitive,
+)
+
+__all__ = [
+    "Dataset",
+    "DatasetColumn",
+    "DatasetReport",
+    "DatasetValidation",
+    "DatasetSplit",
+    "Primitive",
+    "DataRecord",
+]

silver_data/dataset.py

@@ -0,0 +1,160 @@
+import csv
+import hashlib
+import json
+from typing import Any, List, Union
+
+from .models import (
+    DataRecord,
+    DatasetColumn,
+    DatasetReport,
+    DatasetSplit,
+    DatasetValidation,
+)
+
+
+class Dataset:
+    def __init__(self, name: str, rows: List[DataRecord]):
+        if not name.strip():
+            raise ValueError("Dataset name is required")
+        self.name = name
+        self._rows = [dict(row) for row in rows]
+
+    @classmethod
+    def from_records(cls, name: str, rows: List[DataRecord]) -> "Dataset":
+        if len(rows) == 0:
+            raise ValueError("Dataset must contain at least one row")
+        return cls(name, [dict(row) for row in rows])
+
+    @classmethod
+    def from_json(
+        cls, name: str, value: Union[List[DataRecord], DataRecord]
+    ) -> "Dataset":
+        if isinstance(value, dict):
+            return cls.from_records(name, [value])
+        return cls.from_records(name, value)
+
+    @classmethod
+    def from_jsonl(cls, name: str, value: str) -> "Dataset":
+        lines = [line.strip() for line in value.strip().splitlines() if line.strip()]
+        records = []
+        for line in lines:
+            record = json.loads(line)
+            if not isinstance(record, dict) or isinstance(record, list):
+                raise ValueError("JSONL rows must be objects")
+            records.append(record)
+        return cls.from_records(name, records)
+
+    @classmethod
+    def from_csv(cls, name: str, path: str) -> "Dataset":
+        try:
+            import pandas as pd
+        except ImportError:
+            pd = None
+        if pd is not None:
+            return cls.from_records(name, pd.read_csv(path).to_dict(orient="records"))
+        with open(path, newline="", encoding="utf-8") as stream:
+            return cls.from_records(name, list(csv.DictReader(stream)))
+
+    @classmethod
+    def from_pandas(cls, name: str, df: Any) -> "Dataset":
+        if not hasattr(df, "to_dict"):
+            raise TypeError("from_pandas requires an object with to_dict")
+        return cls.from_records(name, df.to_dict(orient="records"))
+
+    def records(self) -> List[DataRecord]:
+        return [dict(row) for row in self._rows]
+
+    def columns(self) -> List[str]:
+        all_columns = set()
+        for row in self._rows:
+            all_columns.update(row.keys())
+        return sorted(all_columns)
+
+    def inspect(self) -> DatasetReport:
+        columns_report = []
+        for name in self.columns():
+            values = [row.get(name) for row in self._rows]
+            present = [value for value in values if value is not None]
+            if present:
+                types = {type(value).__name__ for value in present}
+                if len(types) == 1:
+                    type_name = type(present[0]).__name__
+                    value_type = {
+                        "str": "string",
+                        "int": "number",
+                        "float": "number",
+                        "bool": "boolean",
+                    }.get(type_name, "mixed")
+                else:
+                    value_type = "mixed"
+            else:
+                value_type = "null"
+            columns_report.append(
+                DatasetColumn(
+                    name=name,
+                    value_type=value_type,
+                    missing=len(values) - len(present),
+                    unique=len({json.dumps(value, sort_keys=True) for value in values}),
+                )
+            )
+        return DatasetReport(
+            name=self.name,
+            rows=len(self._rows),
+            columns=columns_report,
+            fingerprint=self.fingerprint(),
+        )
+
+    def validate(self) -> DatasetValidation:
+        errors = []
+        warnings = []
+        column_count = len(self.columns())
+        inconsistent_rows = [
+            index
+            for index, row in enumerate(self._rows)
+            if len(row.keys()) != column_count
+        ]
+        if inconsistent_rows:
+            warnings.append(
+                f"Rows do not all contain the same columns: {inconsistent_rows[:5]}"
+            )
+        for column in self.inspect().columns:
+            if column.missing > 0:
+                warnings.append(f"{column.name} has {column.missing} missing values")
+        return DatasetValidation(
+            valid=len(errors) == 0, errors=errors, warnings=warnings
+        )
+
+    def fingerprint(self) -> str:
+        data = json.dumps({"name": self.name, "rows": self._rows}, sort_keys=True)
+        return hashlib.md5(data.encode()).hexdigest()[:8]
+
+    def split(
+        self, train: float = 0.8, validation: float = 0.1, test: float = 0.1
+    ) -> DatasetSplit:
+        if train < 0 or validation < 0 or test < 0:
+            raise ValueError("Dataset split ratios must be non-negative")
+        if abs(train + validation + test - 1.0) > 1e-9:
+            raise ValueError("Dataset split ratios must sum to 1")
+        train_end = int(len(self._rows) * train)
+        validation_end = train_end + int(len(self._rows) * validation)
+        return DatasetSplit(
+            train=Dataset._from_rows(f"{self.name}/train", self._rows[:train_end]),
+            validation=Dataset._from_rows(
+                f"{self.name}/validation", self._rows[train_end:validation_end]
+            ),
+            test=Dataset._from_rows(f"{self.name}/test", self._rows[validation_end:]),
+        )
+
+    @classmethod
+    def _from_rows(cls, name: str, rows: List[DataRecord]) -> "Dataset":
+        return cls(name, [dict(row) for row in rows])
+
+    def to_pandas(self) -> Any:
+        try:
+            import pandas as pd
+        except ImportError as error:
+            raise ImportError(
+                "Dataset.to_pandas() requires the optional 'pandas' dependency; "
+                "install silver-data[pandas]"
+            ) from error
+        return pd.DataFrame(self._rows, columns=self.columns())

silver_data/models.py

@@ -0,0 +1,35 @@
+from dataclasses import dataclass
+from typing import Dict, List, Literal, Union
+
+Primitive = Union[str, int, float, bool, None]
+DataRecord = Dict[str, Primitive]
+
+
+@dataclass(frozen=True)
+class DatasetColumn:
+    name: str
+    value_type: Literal["string", "number", "boolean", "null", "mixed"]
+    missing: int
+    unique: int
+
+
+@dataclass(frozen=True)
+class DatasetReport:
+    name: str
+    rows: int
+    columns: List[DatasetColumn]
+    fingerprint: str
+
+
+@dataclass(frozen=True)
+class DatasetValidation:
+    valid: bool
+    errors: List[str]
+    warnings: List[str]
+
+
+@dataclass(frozen=True)
+class DatasetSplit:
+    train: "Dataset"
+    validation: "Dataset"
+    test: "Dataset"

silver_data-0.1.0.dist-info/METADATA

@@ -0,0 +1,225 @@
+Metadata-Version: 2.4
+Name: silver-data
+Version: 0.1.0
+Summary: Inspectable, deterministic dataset contracts and loaders for Silver.
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/adfgdartec/silver-data
+Project-URL: Repository, https://github.com/adfgdartec/silver-data
+Project-URL: Issues, https://github.com/adfgdartec/silver-data/issues
+Keywords: machine-learning,datasets,csv,data-validation,python
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: pandas
+Requires-Dist: pandas>=1.0.0; extra == "pandas"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pandas>=1.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: flake8>=6.0.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: build>=0.10.0; extra == "dev"
+Requires-Dist: twine>=4.0.0; extra == "dev"
+Dynamic: license-file
+
+# silver-data
+
+[![Python Version](https://img.shields.io/badge/python-3.8%2B-blue.svg)](https://www.python.org/downloads/)
+[![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](LICENSE)
+[![Tests](https://img.shields.io/badge/tests-passing-brightgreen.svg)](tests/)
+[![Code Style](https://img.shields.io/badge/code%20style-flake8-blue.svg)](https://flake8.pycqa.org/)
+
+Inspectable, deterministic dataset contracts and loaders for Silver. A Python package designed for ML researchers who need reliable dataset handling with built-in validation and reproducibility features.
+
+The base install uses only the Python standard library for records, JSON, JSONL,
+and CSV. Add pandas only when you need DataFrame conversion:
+
+```bash
+pip install 'silver-data[pandas]'
+```
+
+## Installation
+
+```bash
+pip install silver-data
+```
+
+## Quick Start
+
+```python
+from silver_data import Dataset
+
+# Load from CSV file
+dataset = Dataset.from_csv("my_data", "path/to/data.csv")
+
+# Optional: load from pandas
+import pandas as pd
+df = pd.read_csv("path/to/data.csv")
+dataset = Dataset.from_pandas("my_data", df)
+
+# Inspect dataset
+report = dataset.inspect()
+print(f"Rows: {report.rows}, Columns: {len(report.columns)}")
+for col in report.columns:
+    print(f"  {col.name}: {col.value_type} ({col.unique} unique, {col.missing} missing)")
+
+# Validate dataset
+validation = dataset.validate()
+if not validation.valid:
+    print("Errors:", validation.errors)
+if validation.warnings:
+    print("Warnings:", validation.warnings)
+
+# Split dataset for ML workflows
+train, val, test = dataset.split(train=0.8, validation=0.1, test=0.1)
+print(f"Train: {len(train.records())}, Val: {len(val.records())}, Test: {len(test.records())}")
+```
+
+## Features
+
+- **Multiple Data Sources**: Load from CSV, JSON, JSONL, and pandas DataFrames
+- **Dataset Inspection**: Get detailed column statistics and metadata
+- **Data Validation**: Automatic detection of missing values, inconsistent columns, and data quality issues
+- **Deterministic Fingerprinting**: Generate unique identifiers for datasets to ensure reproducibility
+- **Smart Splitting**: Train/validation/test splitting with customizable ratios
+- **Immutable Design**: Safe data handling with copy-on-write semantics
+- **Type Safety**: Full type hints for better IDE support and fewer bugs
+
+## Use Cases
+
+### ML Pipeline Integration
+
+```python
+from silver_data import Dataset
+import pandas as pd
+
+# Load and validate training data
+df = pd.read_csv("train.csv")
+dataset = Dataset.from_pandas("training", df)
+
+# Ensure data quality before training
+validation = dataset.validate()
+if not validation.valid:
+    raise ValueError(f"Dataset validation failed: {validation.errors}")
+
+# Split for cross-validation
+train_split, val_split, test_split = dataset.split(train=0.7, validation=0.15, test=0.15)
+
+# Use fingerprints for caching
+cache_key = dataset.fingerprint()
+print(f"Dataset fingerprint: {cache_key}")
+```
+
+### Data Quality Monitoring
+
+```python
+from silver_data import Dataset
+
+# Monitor data drift over time
+dataset_v1 = Dataset.from_csv("data_v1", "data_2024_01.csv")
+dataset_v2 = Dataset.from_csv("data_v2", "data_2024_02.csv")
+
+if dataset_v1.fingerprint() != dataset_v2.fingerprint():
+    print("Dataset has changed - retrain models")
+
+# Check for new data quality issues
+report_v2 = dataset_v2.inspect()
+for col in report_v2.columns:
+    if col.missing > len(dataset_v2.records()) * 0.1:  # More than 10% missing
+        print(f"Warning: {col.name} has high missing rate: {col.missing}")
+```
+
+### Experiment Reproducibility
+
+```python
+from silver_data import Dataset
+
+# Ensure exact same data across experiments
+dataset = Dataset.from_csv("experiment", "data.csv")
+experiment_id = f"exp_{dataset.fingerprint()}"
+
+# Log for reproducibility
+print(f"Running experiment {experiment_id} with dataset fingerprint {dataset.fingerprint()}")
+```
+
+## Advanced Usage
+
+### Custom Data Loading
+
+```python
+from silver_data import Dataset
+import json
+
+# Load from custom JSON format
+with open("custom_data.json") as f:
+    data = json.load(f)
+dataset = Dataset.from_json("custom", data)
+
+# Load from streaming JSONL
+with open("streaming_data.jsonl") as f:
+    dataset = Dataset.from_jsonl("streaming", f.read())
+```
+
+### Data Type Analysis
+
+```python
+from silver_data import Dataset
+
+dataset = Dataset.from_csv("analysis", "mixed_data.csv")
+report = dataset.inspect()
+
+# Analyze column types
+string_cols = [c.name for c in report.columns if c.value_type == "string"]
+numeric_cols = [c.name for c in report.columns if c.value_type == "number"]
+mixed_cols = [c.name for c in report.columns if c.value_type == "mixed"]
+
+print(f"String columns: {string_cols}")
+print(f"Numeric columns: {numeric_cols}")
+print(f"Mixed type columns: {mixed_cols}")
+```
+
+## Requirements
+
+- Python 3.8+
+- pandas 1.0+
+
+## Development
+
+```bash
+# Install development dependencies
+pip install -e ".[dev]"
+
+# Run tests
+pytest
+
+# Run tests with coverage
+pytest --cov=silver_data --cov-report=html
+
+# Run linting
+flake8 src/ tests/
+mypy src/
+```
+
+## Contributing
+
+Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
+
+## License
+
+Apache-2.0 - see [LICENSE](LICENSE) file for details.
+
+## Related Packages
+
+- [silver-run](https://github.com/adfgdartec/silver-run) - Training lifecycle management
+- [silver-diagnostics](https://github.com/adfgdartec/silver-diagnostics) - ML diagnostics
+- [silver-adapters](https://github.com/adfgdartec/silver-adapters) - Framework adapters

silver_data-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+silver_data/__init__.py,sha256=xafu4vIPLaZ26ypIrmMyrzElSbkBb6ieG_KGEKGV6xA,315
+silver_data/dataset.py,sha256=SIikl97ZpervJsT4PkWsf7H-P8gjphMOZN6emOvl-X4,5876
+silver_data/models.py,sha256=EaxPmK8L-s3FMkwIir2xvtlA6xP-7zgRVM-KEB08hP4,683
+silver_data-0.1.0.dist-info/licenses/LICENSE,sha256=9uEgZddcCAZ2jYIKny4rlbkQMZFhKRvSGbxctAg4wjI,517
+silver_data-0.1.0.dist-info/METADATA,sha256=azOwbu62C6CwY9ptzl762I1jnOlKbKDtfKjXWFFUXYQ,6952
+silver_data-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+silver_data-0.1.0.dist-info/top_level.txt,sha256=PlSkMomi6jHDrFFWnkgEeH_LAYiD7OWITwI4ckTVrkY,12
+silver_data-0.1.0.dist-info/RECORD,,

silver_data-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

silver_data-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,12 @@
+Apache License
+Version 2.0, January 2004
+
+Copyright 2026 Silver Contributors
+
+Licensed under the Apache License, Version 2.0. You may obtain a copy of the
+License at https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed
+under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.

silver_data-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+silver_data