aptdata 0.0.2__py3-none-any.whl

aptdata/plugins/governance/classification.py

@@ -0,0 +1,44 @@
+"""Data classification convenience re-exports and policy definitions."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+# Re-export for convenience so callers can do:
+#   from aptdata.plugins.governance.classification import ColumnClassification
+from aptdata.plugins.quality.contract import ColumnClassification
+
+
+@dataclass
+class DataClassificationPolicy:
+    """Policy defining how data with a specific classification should be handled.
+
+    Parameters
+    ----------
+    name:
+        Policy name.
+    description:
+        Human-readable description of the policy.
+    pii_columns:
+        Column names that contain personally identifiable information.
+    retention_days:
+        Required data retention period in days.
+    encryption_required:
+        Whether data at rest must be encrypted.
+    access_roles:
+        Roles permitted to access data governed by this policy.
+    metadata:
+        Arbitrary extra metadata.
+    """
+
+    name: str
+    description: str = ""
+    pii_columns: list[str] = field(default_factory=list)
+    retention_days: int = 0
+    encryption_required: bool = False
+    access_roles: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+__all__ = ["ColumnClassification", "DataClassificationPolicy"]

aptdata/plugins/governance/lineage_store.py

@@ -0,0 +1,49 @@
+"""In-memory lineage graph store for persisting and querying workflow runs."""
+
+from __future__ import annotations
+
+from aptdata.core.lineage import LineageGraph
+
+
+class LineageStore:
+    """In-memory store for :class:`~aptdata.core.lineage.LineageGraph` objects.
+
+    Each workflow run produces exactly one :class:`LineageGraph` which is
+    saved under its :attr:`~LineageGraph.run_id`.
+
+    Examples
+    --------
+    ::
+
+        store = LineageStore()
+        store.save(graph)
+        loaded = store.load(run_id)
+        runs = store.list_runs()
+        graphs = store.query_by_dataset("s3://bucket/data.parquet")
+    """
+
+    def __init__(self) -> None:
+        self._store: dict[str, LineageGraph] = {}
+
+    def save(self, graph: LineageGraph) -> None:
+        """Persist *graph* under its :attr:`~LineageGraph.run_id`."""
+        self._store[graph.run_id] = graph
+
+    def load(self, run_id: str) -> LineageGraph | None:
+        """Return the graph for *run_id*, or ``None`` if not found."""
+        return self._store.get(run_id)
+
+    def list_runs(self) -> list[str]:
+        """Return a sorted list of all stored run IDs."""
+        return sorted(self._store)
+
+    def query_by_dataset(self, uri: str) -> list[LineageGraph]:
+        """Return all graphs that contain at least one node referencing *uri*."""
+        return [
+            graph
+            for graph in self._store.values()
+            if any(node.dataset_uri == uri for node in graph.nodes)
+        ]
+
+
+__all__ = ["LineageStore"]

aptdata/plugins/governance/rules.py

@@ -0,0 +1,180 @@
+"""Business rules registry with audit logging."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any
+
+
+class RuleStatus(str, Enum):
+    """Execution status of a business rule application."""
+
+    APPLIED = "APPLIED"
+    SKIPPED = "SKIPPED"
+    FAILED = "FAILED"
+
+
+@dataclass
+class BusinessRule:
+    """Declaration of a single business rule.
+
+    Parameters
+    ----------
+    rule_id:
+        Unique identifier (e.g. ``"BR-001"``).
+    name:
+        Human-readable rule name.
+    version:
+        Semantic version string (e.g. ``"1.0.0"``).
+    owner:
+        Team or person responsible for this rule.
+    description:
+        Detailed description of what the rule enforces.
+    expression:
+        Human-readable expression or pseudo-code representing the rule logic.
+    tags:
+        Free-form classification tags.
+    effective_from:
+        ISO-8601 date/time from which the rule is effective.
+    effective_until:
+        ISO-8601 date/time after which the rule expires (empty = no expiry).
+    metadata:
+        Arbitrary extra metadata.
+    """
+
+    rule_id: str
+    name: str
+    version: str = "1.0.0"
+    owner: str = ""
+    description: str = ""
+    expression: str = ""
+    tags: list[str] = field(default_factory=list)
+    effective_from: str = field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat()
+    )
+    effective_until: str = ""
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class RuleAuditEntry:
+    """Audit record for a single rule application.
+
+    Parameters
+    ----------
+    rule_id:
+        ID of the rule that was applied.
+    rule_version:
+        Version of the rule that was applied.
+    status:
+        Outcome of the rule execution.
+    workflow_name:
+        Workflow in which the rule was executed.
+    step_name:
+        Step within the workflow.
+    trace_id:
+        OpenTelemetry trace identifier.
+    timestamp:
+        UTC ISO-8601 timestamp of the audit entry.
+    rows_affected:
+        Number of rows affected by the rule.
+    details:
+        Human-readable summary of what the rule did.
+    metadata:
+        Arbitrary extra metadata.
+    """
+
+    rule_id: str
+    rule_version: str = "1.0.0"
+    status: RuleStatus = RuleStatus.APPLIED
+    workflow_name: str = ""
+    step_name: str = ""
+    trace_id: str = ""
+    timestamp: str = field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat()
+    )
+    rows_affected: int = 0
+    details: str = ""
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class RuleRegistry:
+    """In-memory registry for :class:`BusinessRule` definitions and audit logs.
+
+    Examples
+    --------
+    ::
+
+        registry = RuleRegistry()
+        registry.register(BusinessRule(rule_id="BR-001", name="Age must be positive"))
+        rule = registry.get("BR-001")
+        rules = registry.list_rules(tag="finance")
+    """
+
+    def __init__(self) -> None:
+        self._rules: dict[str, BusinessRule] = {}
+        self._audit_log: list[RuleAuditEntry] = []
+
+    def register(self, rule: BusinessRule) -> None:
+        """Register *rule* under its :attr:`~BusinessRule.rule_id`."""
+        self._rules[rule.rule_id] = rule
+
+    def get(self, rule_id: str) -> BusinessRule | None:
+        """Return the rule with *rule_id*, or ``None`` if not found."""
+        return self._rules.get(rule_id)
+
+    def list_rules(
+        self,
+        owner: str | None = None,
+        tag: str | None = None,
+    ) -> list[BusinessRule]:
+        """Return all registered rules, optionally filtered by *owner* or *tag*.
+
+        Parameters
+        ----------
+        owner:
+            If provided, only rules owned by this owner are returned.
+        tag:
+            If provided, only rules with this tag are returned.
+        """
+        results = list(self._rules.values())
+        if owner is not None:
+            results = [r for r in results if r.owner == owner]
+        if tag is not None:
+            results = [r for r in results if tag in r.tags]
+        return results
+
+    def record_audit(self, entry: RuleAuditEntry) -> None:
+        """Append *entry* to the audit log."""
+        self._audit_log.append(entry)
+
+    def get_audit_log(
+        self,
+        rule_id: str | None = None,
+        trace_id: str | None = None,
+    ) -> list[RuleAuditEntry]:
+        """Return audit entries, optionally filtered by *rule_id* or *trace_id*.
+
+        Parameters
+        ----------
+        rule_id:
+            If provided, only entries for this rule are returned.
+        trace_id:
+            If provided, only entries with this trace ID are returned.
+        """
+        entries = list(self._audit_log)
+        if rule_id is not None:
+            entries = [e for e in entries if e.rule_id == rule_id]
+        if trace_id is not None:
+            entries = [e for e in entries if e.trace_id == trace_id]
+        return entries
+
+
+__all__ = [
+    "RuleStatus",
+    "BusinessRule",
+    "RuleAuditEntry",
+    "RuleRegistry",
+]

aptdata/plugins/local_fs.py

@@ -0,0 +1,241 @@
+"""Local filesystem plugin — CSV, JSON, and Parquet readers / writers.
+
+All classes produce or consume :class:`~aptdata.plugins.dataset.InMemoryDataset`
+instances backed by lists of dictionaries (records).
+
+CSV and JSON support is built-in (stdlib ``csv`` / ``json``).
+Parquet support requires the optional ``pyarrow`` package; a friendly
+:class:`~aptdata.plugins.manager.PluginDependencyError` is raised if
+it is not installed.
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+from pathlib import Path
+from typing import Any
+
+from aptdata.core.dataset import BaseDataset
+from aptdata.plugins.base import BaseReader, BaseWriter
+from aptdata.plugins.dataset import InMemoryDataset
+from aptdata.plugins.manager import PluginDependencyError
+
+# ---------------------------------------------------------------------------
+# CSV
+# ---------------------------------------------------------------------------
+
+
+class CSVReader(BaseReader):
+    """Read a CSV file into an :class:`InMemoryDataset`.
+
+    Parameters
+    ----------
+    filepath:
+        Path to the CSV file on the local filesystem.
+    encoding:
+        File encoding (default ``"utf-8"``).
+    delimiter:
+        Column delimiter (default ``","``).
+    """
+
+    def __init__(
+        self,
+        filepath: str,
+        *,
+        encoding: str = "utf-8",
+        delimiter: str = ",",
+    ) -> None:
+        self.filepath = Path(filepath)
+        self.encoding = encoding
+        self.delimiter = delimiter
+
+    def read(self, **kwargs: Any) -> InMemoryDataset:
+        with open(self.filepath, newline="", encoding=self.encoding) as fh:
+            reader = csv.DictReader(fh, delimiter=self.delimiter)
+            records = list(reader)
+        ds = InMemoryDataset(uri=str(self.filepath))
+        ds.write(records)
+        return ds
+
+
+class CSVWriter(BaseWriter):
+    """Write an :class:`InMemoryDataset` to a CSV file.
+
+    Parameters
+    ----------
+    filepath:
+        Destination path on the local filesystem.
+    encoding:
+        File encoding (default ``"utf-8"``).
+    delimiter:
+        Column delimiter (default ``","``).
+    """
+
+    def __init__(
+        self,
+        filepath: str,
+        *,
+        encoding: str = "utf-8",
+        delimiter: str = ",",
+    ) -> None:
+        self.filepath = Path(filepath)
+        self.encoding = encoding
+        self.delimiter = delimiter
+
+    def write(self, dataset: BaseDataset, **kwargs: Any) -> None:
+        records: list[dict[str, Any]] = dataset.read()
+        if not records:
+            self.filepath.parent.mkdir(parents=True, exist_ok=True)
+            self.filepath.write_text("", encoding=self.encoding)
+            return
+
+        fieldnames = list(records[0].keys())
+        self.filepath.parent.mkdir(parents=True, exist_ok=True)
+        with open(self.filepath, "w", newline="", encoding=self.encoding) as fh:
+            writer = csv.DictWriter(fh, fieldnames=fieldnames, delimiter=self.delimiter)
+            writer.writeheader()
+            writer.writerows(records)
+
+
+# ---------------------------------------------------------------------------
+# JSON
+# ---------------------------------------------------------------------------
+
+
+class JSONReader(BaseReader):
+    """Read a JSON file (array of objects) into an :class:`InMemoryDataset`.
+
+    Parameters
+    ----------
+    filepath:
+        Path to the JSON file.
+    encoding:
+        File encoding (default ``"utf-8"``).
+    """
+
+    def __init__(self, filepath: str, *, encoding: str = "utf-8") -> None:
+        self.filepath = Path(filepath)
+        self.encoding = encoding
+
+    def read(self, **kwargs: Any) -> InMemoryDataset:
+        with open(self.filepath, encoding=self.encoding) as fh:
+            data = json.load(fh)
+        if not isinstance(data, list):
+            raise ValueError("JSON file must contain an array of objects.")
+        ds = InMemoryDataset(uri=str(self.filepath))
+        ds.write(data)
+        return ds
+
+
+class JSONWriter(BaseWriter):
+    """Write an :class:`InMemoryDataset` to a JSON file (array of objects).
+
+    Parameters
+    ----------
+    filepath:
+        Destination path.
+    encoding:
+        File encoding (default ``"utf-8"``).
+    indent:
+        JSON indentation level (default ``2``).
+    """
+
+    def __init__(
+        self,
+        filepath: str,
+        *,
+        encoding: str = "utf-8",
+        indent: int = 2,
+    ) -> None:
+        self.filepath = Path(filepath)
+        self.encoding = encoding
+        self.indent = indent
+
+    def write(self, dataset: BaseDataset, **kwargs: Any) -> None:
+        records: list[dict[str, Any]] = dataset.read()
+        self.filepath.parent.mkdir(parents=True, exist_ok=True)
+        with open(self.filepath, "w", encoding=self.encoding) as fh:
+            json.dump(records, fh, indent=self.indent, ensure_ascii=False)
+
+
+# ---------------------------------------------------------------------------
+# Parquet (optional dependency: pyarrow)
+# ---------------------------------------------------------------------------
+
+
+class ParquetReader(BaseReader):
+    """Read a Parquet file into an :class:`InMemoryDataset`.
+
+    Requires the ``pyarrow`` package.
+
+    Parameters
+    ----------
+    filepath:
+        Path to the ``.parquet`` file.
+    """
+
+    def __init__(self, filepath: str) -> None:
+        self.filepath = Path(filepath)
+
+    def read(self, **kwargs: Any) -> InMemoryDataset:
+        try:
+            import pyarrow.parquet as pq  # noqa: WPS433
+        except ImportError:
+            raise PluginDependencyError("ParquetReader", "pyarrow") from None
+
+        table = pq.read_table(str(self.filepath))
+        records: list[dict[str, Any]] = table.to_pydict()
+        # Convert columnar dict to list-of-dicts
+        keys = list(records.keys())
+        row_count = len(records[keys[0]]) if keys else 0
+        rows = [{k: records[k][i] for k in keys} for i in range(row_count)]
+
+        ds = InMemoryDataset(uri=str(self.filepath))
+        ds.write(rows)
+        return ds
+
+
+class ParquetWriter(BaseWriter):
+    """Write an :class:`InMemoryDataset` to a Parquet file.
+
+    Requires the ``pyarrow`` package.
+
+    Parameters
+    ----------
+    filepath:
+        Destination ``.parquet`` path.
+    """
+
+    def __init__(self, filepath: str) -> None:
+        self.filepath = Path(filepath)
+
+    def write(self, dataset: BaseDataset, **kwargs: Any) -> None:
+        try:
+            import pyarrow as pa  # noqa: WPS433
+            import pyarrow.parquet as pq  # noqa: WPS433
+        except ImportError:
+            raise PluginDependencyError("ParquetWriter", "pyarrow") from None
+
+        records: list[dict[str, Any]] = dataset.read()
+        if not records:
+            # Write an empty parquet with no columns
+            table = pa.table({})
+        else:
+            # Convert list-of-dicts to columnar dict
+            keys = list(records[0].keys())
+            columnar = {k: [r[k] for r in records] for k in keys}
+            table = pa.table(columnar)
+
+        self.filepath.parent.mkdir(parents=True, exist_ok=True)
+        pq.write_table(table, str(self.filepath))
+
+
+__all__ = [
+    "CSVReader",
+    "CSVWriter",
+    "JSONReader",
+    "JSONWriter",
+    "ParquetReader",
+    "ParquetWriter",
+]

aptdata/plugins/manager.py

@@ -0,0 +1,142 @@
+"""Dynamic plugin manager for aptdata.
+
+Provides :class:`PluginManager` which can discover, register, and
+instantiate reader / writer plugins.  If a plugin requires an
+optional third-party library that is not installed, a friendly
+:class:`PluginDependencyError` is raised.
+"""
+
+from __future__ import annotations
+
+import importlib
+import inspect
+from typing import Any
+
+from aptdata.plugins.base import BaseReader, BaseWriter
+
+# ---------------------------------------------------------------------------
+# Custom errors
+# ---------------------------------------------------------------------------
+
+
+class PluginDependencyError(ImportError):
+    """Raised when a plugin requires a library that is not installed."""
+
+    def __init__(self, plugin_name: str, package: str) -> None:
+        self.plugin_name = plugin_name
+        self.package = package
+        super().__init__(
+            f"Plugin '{plugin_name}' requires the '{package}' package. "
+            f"Install it with:  pip install {package}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Plugin Manager
+# ---------------------------------------------------------------------------
+
+
+class PluginManager:
+    """Registry for reader / writer plugin classes.
+
+    Plugins are registered under a unique *name* and can be retrieved or
+    listed later.  The manager also supports loading a plugin module by
+    its dotted Python path so that dynamic/entry-point-style discovery is
+    straightforward.
+    """
+
+    def __init__(self) -> None:
+        self._readers: dict[str, type[BaseReader]] = {}
+        self._writers: dict[str, type[BaseWriter]] = {}
+
+    # -- registration -------------------------------------------------------
+
+    def register_reader(self, name: str, reader_cls: type[BaseReader]) -> None:
+        """Register *reader_cls* under *name*."""
+        self._readers[name] = reader_cls
+
+    def register_writer(self, name: str, writer_cls: type[BaseWriter]) -> None:
+        """Register *writer_cls* under *name*."""
+        self._writers[name] = writer_cls
+
+    # -- lookup -------------------------------------------------------------
+
+    def get_reader(self, name: str) -> type[BaseReader] | None:
+        """Return the reader class registered under *name*, or ``None``."""
+        return self._readers.get(name)
+
+    def get_writer(self, name: str) -> type[BaseWriter] | None:
+        """Return the writer class registered under *name*, or ``None``."""
+        return self._writers.get(name)
+
+    # -- listing ------------------------------------------------------------
+
+    def list_readers(self) -> list[str]:
+        """Return a sorted list of registered reader names."""
+        return sorted(self._readers)
+
+    def list_writers(self) -> list[str]:
+        """Return a sorted list of registered writer names."""
+        return sorted(self._writers)
+
+    def list_plugins(self) -> dict[str, list[str]]:
+        """Return all registered plugins grouped by kind."""
+        return {
+            "readers": self.list_readers(),
+            "writers": self.list_writers(),
+        }
+
+    def get_plugin_schema(self, name: str) -> dict[str, Any]:
+        """Return constructor argument schema for a reader/writer plugin."""
+        plugin_cls: type[Any] | None = self.get_reader(name) or self.get_writer(name)
+        if plugin_cls is None:
+            raise KeyError(f"Plugin '{name}' is not registered.")
+
+        signature = inspect.signature(plugin_cls.__init__)
+        args: list[dict[str, Any]] = []
+        for param_name, param in signature.parameters.items():
+            if param_name == "self":
+                continue
+            args.append(
+                {
+                    "name": param_name,
+                    "required": param.default is inspect.Parameter.empty,
+                    "default": None
+                    if param.default is inspect.Parameter.empty
+                    else param.default,
+                }
+            )
+
+        plugin_type = "reader" if self.get_reader(name) is not None else "writer"
+        return {"name": name, "type": plugin_type, "arguments": args}
+
+    def preview_dataset(self, plugin_name: str, **kwargs: Any) -> list[dict[str, Any]]:
+        """Run a reader plugin and return the first five records."""
+        reader_cls = self.get_reader(plugin_name)
+        if reader_cls is None:
+            raise KeyError(f"Reader plugin '{plugin_name}' is not registered.")
+        reader = reader_cls(**kwargs)
+        dataset = reader.read()
+        records = dataset.read()
+        return records[:5]
+
+    # -- dynamic loading ----------------------------------------------------
+
+    def load_module(self, module_path: str) -> Any:
+        """Import *module_path* and return the module object.
+
+        This is useful for loading plugin modules dynamically, e.g. from
+        an entry-point or a configuration file.
+
+        Raises
+        ------
+        ModuleNotFoundError
+            If the module cannot be imported.
+        """
+        return importlib.import_module(module_path)
+
+
+#: Global singleton – import this in plugin modules and application code.
+plugin_manager = PluginManager()
+
+__all__ = ["PluginManager", "PluginDependencyError", "plugin_manager"]