sunstone-py 0.4.2__py3-none-any.whl → 0.5.1__py3-none-any.whl

sunstone/_release.py

@@ -164,29 +164,12 @@ def get_last_tag() -> str | None:
 def generate_changelog_from_git() -> str:
     """Generate changelog entries from git commits since last tag using Claude."""
     last_tag = get_last_tag()
-    if last_tag:
-        commit_range = f"{last_tag}..HEAD"
-    else:
-        commit_range = "HEAD"
-
-    # Get commits since last tag
-    result = run_git("log", commit_range, "--pretty=format:%s")
-    if result.returncode != 0 or not result.stdout.strip():
-        return ""
-
-    commits = result.stdout.strip()
-
-    prompt = f"""Convert these git commit messages into Keep a Changelog format entries.
-Categorize under: Added, Changed, Fixed, Removed, Security (only include categories that apply).
-Be concise. Skip merge commits, version bump commits, and release commits.
-Output ONLY the markdown entries with ### headers for categories, nothing else.
-
-Commits:
-{commits}"""
+    if last_tag is None:
+        last_tag = "HEAD~1"
 
     print("Generating changelog entries with Claude...")
     claude_result = subprocess.run(
-        ["claude", "-p", "--model=haiku", prompt],
+        ["claude", "-p", f"/generate-changelog {last_tag}"],
         capture_output=True,
         text=True,
         cwd=get_root_dir(),

sunstone/dataframe.py

@@ -323,7 +323,7 @@ class DataFrame:
             path_or_buf: File path for the output CSV.
             slug: Dataset slug (required in relaxed mode if not registered).
             name: Dataset name (required in relaxed mode if not registered).
-            publish: Whether to publish the dataset.
+            publish: bool = False,
             **kwargs: Additional arguments passed to pandas.to_csv.
 
         Raises:
@@ -366,6 +366,9 @@ class DataFrame:
         # Record the operation
         self.lineage.add_operation(f"to_csv({dataset.slug})")
 
+        # Persist lineage metadata to datasets.yaml
+        manager.update_output_lineage(slug=dataset.slug, lineage=self.lineage, strict=self.strict_mode)
+
     def _infer_field_schema(self) -> List[FieldSchema]:
         """
         Infer field schema from the DataFrame.
@@ -510,13 +513,13 @@ class DataFrame:
             project_path=self.lineage.project_path,
         )
 
-    def _wrap_result(self, result: Any, operation: str = "pandas_operation") -> Any:
+    def _wrap_result(self, result: Any, operation: Optional[str] = None) -> Any:
         """
         Wrap a pandas result in a Sunstone DataFrame if applicable.
 
         Args:
             result: The result from a pandas operation.
-            operation: Name of the operation performed.
+            operation: Name of the operation performed. If None, no operation is recorded.
 
         Returns:
             Wrapped DataFrame if result is a DataFrame, otherwise the result.
@@ -527,7 +530,8 @@ class DataFrame:
                 operations=self.lineage.operations.copy(),
                 project_path=self.lineage.project_path,
             )
-            new_lineage.add_operation(operation)
+            if operation is not None:
+                new_lineage.add_operation(operation)
 
             return DataFrame(
                 data=result,
@@ -537,6 +541,28 @@ class DataFrame:
             )
         return result
 
+    # Methods that don't represent meaningful data transformations
+    # These return DataFrames but shouldn't be tracked in lineage
+    _NON_TRACKING_METHODS = frozenset(
+        {
+            # Copy operations - same data, no transformation
+            "copy",
+            # Index operations - same data, different index
+            "reset_index",
+            "set_index",
+            "reindex",
+            # Type conversions without data change
+            "astype",
+            "infer_objects",
+            # Column/index renaming - same data, different labels
+            "rename",
+            "rename_axis",
+            # Reshaping without data loss
+            "T",
+            "transpose",
+        }
+    )
+
     def __getattr__(self, name: str) -> Any:
         """
         Delegate attribute access to the underlying pandas DataFrame.
@@ -557,11 +583,14 @@ class DataFrame:
 
             def wrapper(*args: Any, **kwargs: Any) -> Any:
                 result = attr(*args, **kwargs)
+                # Don't track non-transforming methods
+                if name in DataFrame._NON_TRACKING_METHODS:
+                    return self._wrap_result(result, operation=None)
                 return self._wrap_result(result, operation=f"{name}")
 
             return wrapper
 
-        return self._wrap_result(attr, operation=f"access_attribute_{name}")
+        return self._wrap_result(attr, operation=None)  # Don't track attribute access
 
     def __getitem__(self, key: Any) -> Any:
         """
@@ -574,7 +603,9 @@ class DataFrame:
             The item from the underlying DataFrame, wrapped if it's a DataFrame.
         """
         result = self.data[key]
-        return self._wrap_result(result, operation="__getitem__")
+        # Don't track __getitem__ as an operation - it's just column/row access
+        # not a meaningful transformation
+        return self._wrap_result(result, operation=None)
 
     def __setitem__(self, key: Any, value: Any) -> None:
         """
@@ -585,7 +616,8 @@ class DataFrame:
             value: Value to assign.
         """
         self.data[key] = value
-        self.lineage.add_operation("__setitem__")
+        # Track column assignment in lineage
+        self.lineage.add_operation(f"__setitem__({key!r})")
 
     def __repr__(self) -> str:
         """String representation of the DataFrame."""

sunstone/datasets.py

@@ -4,19 +4,27 @@ Parser and manager for datasets.yaml files.
 
 import ipaddress
 import logging
+import os
 import socket
+import tempfile
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 from urllib.parse import urljoin, urlparse
 
 import requests
-import yaml
+from ruamel.yaml import YAML
 
 from .exceptions import DatasetNotFoundError, DatasetValidationError
-from .lineage import DatasetMetadata, FieldSchema, Source, SourceLocation
+from .lineage import DatasetMetadata, FieldSchema, LineageMetadata, Source, SourceLocation
 
 logger = logging.getLogger(__name__)
 
+# Configure ruamel.yaml for round-trip parsing (preserves comments) with proper indentation
+_yaml = YAML()
+_yaml.preserve_quotes = True
+_yaml.default_flow_style = False
+_yaml.indent(mapping=2, sequence=4, offset=2)
+
 
 def _is_public_url(url: str) -> bool:
     """
@@ -109,7 +117,7 @@ class DatasetsManager:
     def _load(self) -> None:
         """Load and parse the datasets.yaml file."""
         with open(self.datasets_file, "r") as f:
-            self._data = yaml.safe_load(f) or {}
+            self._data = _yaml.load(f) or {}
 
         if "inputs" not in self._data:
             self._data["inputs"] = []
@@ -119,7 +127,7 @@ class DatasetsManager:
     def _save(self) -> None:
         """Save the current data back to datasets.yaml."""
         with open(self.datasets_file, "w") as f:
-            yaml.dump(self._data, f, default_flow_style=False, sort_keys=False)
+            _yaml.dump(self._data, f)
 
     def _parse_source_location(self, loc_data: Dict[str, Any]) -> SourceLocation:
         """Parse source location data from YAML."""
@@ -372,6 +380,92 @@ class DatasetsManager:
 
         raise DatasetNotFoundError(f"Output dataset with slug '{slug}' not found")
 
+    def update_output_lineage(self, slug: str, lineage: LineageMetadata, strict: bool = False) -> None:
+        """
+        Update lineage metadata for an output dataset.
+
+        In strict mode, validates that the lineage matches what would be written
+        without modifying the file. In relaxed mode, updates the file with lineage.
+
+        Args:
+            slug: The slug of the output dataset to update.
+            lineage: The lineage metadata to persist.
+            strict: If True, validate without modifying. If False, update the file.
+
+        Raises:
+            DatasetNotFoundError: If the dataset doesn't exist.
+            DatasetValidationError: In strict mode, if lineage differs from what's in the file.
+        """
+        # Find the output dataset
+        dataset_idx = None
+        for i, dataset_data in enumerate(self._data["outputs"]):
+            if dataset_data["slug"] == slug:
+                dataset_idx = i
+                break
+
+        if dataset_idx is None:
+            raise DatasetNotFoundError(f"Output dataset with slug '{slug}' not found")
+
+        # Build lineage metadata to add
+        lineage_data: dict[str, Any] = {}
+
+        if lineage.sources:
+            lineage_data["sources"] = [
+                {
+                    "slug": src.slug,
+                    "name": src.name,
+                }
+                for src in lineage.sources
+            ]
+
+        if lineage.operations:
+            lineage_data["operations"] = lineage.operations.copy()
+
+        if lineage.created_at:
+            lineage_data["created_at"] = lineage.created_at.isoformat()
+
+        # Create a copy of the data with updated lineage
+        updated_data = self._data.copy()
+        updated_data["outputs"] = [dict(d) for d in self._data["outputs"]]
+        updated_data["outputs"][dataset_idx] = dict(self._data["outputs"][dataset_idx])
+
+        # Add or update lineage in the copy
+        if lineage_data:
+            updated_data["outputs"][dataset_idx]["lineage"] = lineage_data
+
+        # Write to temp file
+        temp_fd, temp_path = tempfile.mkstemp(suffix=".yaml", prefix="datasets_", dir=self.project_path)
+
+        try:
+            with os.fdopen(temp_fd, "w") as f:
+                _yaml.dump(updated_data, f)
+
+            if strict:
+                # In strict mode, check if files differ
+                import filecmp
+
+                if not filecmp.cmp(self.datasets_file, temp_path, shallow=False):
+                    # Files differ - this is an error in strict mode
+                    os.unlink(temp_path)
+                    raise DatasetValidationError(
+                        f"In strict mode, lineage metadata for '{slug}' would be updated in datasets.yaml. "
+                        f"Expected lineage is already present in the file, but found differences."
+                    )
+                else:
+                    # Files are the same - clean up temp file
+                    os.unlink(temp_path)
+            else:
+                # In relaxed mode, replace the file
+                os.replace(temp_path, self.datasets_file)
+                # Reload the data
+                self._load()
+
+        except Exception:
+            # Clean up temp file on error
+            if os.path.exists(temp_path):
+                os.unlink(temp_path)
+            raise
+
     def get_absolute_path(self, location: str) -> Path:
         """
         Get the absolute path for a dataset location.

{sunstone_py-0.4.2.dist-info → sunstone_py-0.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sunstone-py
-Version: 0.4.2
+Version: 0.5.1
 Summary: Python library for managing datasets with lineage tracking in Sunstone projects
 Author-email: Sunstone Institute <stig@sunstone.institute>
 License: MIT
@@ -24,6 +24,7 @@ Requires-Dist: google-auth>=2.43.0
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: requests>=2.31.0
+Requires-Dist: ruamel-yaml>=0.18
 Dynamic: license-file
 
 # sunstone-py
@@ -324,14 +325,14 @@ uv run pytest
 ### Type Checking
 
 ```bash
-uv run mypy src/sunstone
+uv run mypy
 ```
 
 ### Linting and Formatting
 
 ```bash
-uv run ruff check src/sunstone
-uv run ruff format src/sunstone
+uv run ruff check
+uv run ruff format
 ```
 
 ## About Sunstone Institute

sunstone_py-0.5.1.dist-info/RECORD

@@ -0,0 +1,15 @@
+sunstone/__init__.py,sha256=LC0ZtmxP26eXPLKejbg7UStcHOnE_lwttNTL4m3F4yM,2032
+sunstone/_release.py,sha256=_yjAl_vZQ_5IYr0ugPlqtmUvsGnyGDx7LyiI_2HToVM,14649
+sunstone/dataframe.py,sha256=UJgQx7auiNb6hSIvhB8EQs2afu-7S22xdWL5DZUr29g,23602
+sunstone/datasets.py,sha256=LdHk3Vkfc7QH2VxhSskRCm9wUFSkldCmgS_1c2KDAPA,21142
+sunstone/exceptions.py,sha256=fiixXazur3LtQGy21bGEaSr356DObFcYxQJ3FvOxNec,623
+sunstone/lineage.py,sha256=B9GKMu5-v8Izos5G40K_EvsCPJL3Z2Tg1T_Fc7ezSMI,5240
+sunstone/pandas.py,sha256=CLEqIIgTbMmpH73TPy_vDUPxQa37Hpmqn4r6No8PJwo,8188
+sunstone/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sunstone/validation.py,sha256=1356vcUc72a1zGBUe9Xjrcb5h41Xo53PaK2nnQ_FuSM,8286
+sunstone_py-0.5.1.dist-info/licenses/LICENSE,sha256=pB6VuR4QRjwjMjy8RSNGho-N1SUdu07ntIhT5lrhkzU,1078
+sunstone_py-0.5.1.dist-info/METADATA,sha256=DMLR03NMB5_t14rsBo4GtqY0oQFHnKQtbdUGEfxFcq8,9563
+sunstone_py-0.5.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sunstone_py-0.5.1.dist-info/entry_points.txt,sha256=0h6E88rH9a_503BAzXvFPR-UfmkrRFjcOf29DXgJNjk,51
+sunstone_py-0.5.1.dist-info/top_level.txt,sha256=A2fW-7JO10rlx_L28Bc4FVvWt2R8kgvS8_TGPBhQp3c,9
+sunstone_py-0.5.1.dist-info/RECORD,,

sunstone_py-0.4.2.dist-info/RECORD

@@ -1,15 +0,0 @@
-sunstone/__init__.py,sha256=LC0ZtmxP26eXPLKejbg7UStcHOnE_lwttNTL4m3F4yM,2032
-sunstone/_release.py,sha256=FXqmg9MtMRW9-1DUXrO0PgViUTgUXnMFOx_HUC2n854,15264
-sunstone/dataframe.py,sha256=3wP91L0J3Ptgg41tCRPm84UxTJsj4fy2aBCmCu15qoE,22312
-sunstone/datasets.py,sha256=rrakdvgX7EOCWWWrm8wDqOqXIRqaq-KNUqjrwsm66OI,17590
-sunstone/exceptions.py,sha256=fiixXazur3LtQGy21bGEaSr356DObFcYxQJ3FvOxNec,623
-sunstone/lineage.py,sha256=B9GKMu5-v8Izos5G40K_EvsCPJL3Z2Tg1T_Fc7ezSMI,5240
-sunstone/pandas.py,sha256=CLEqIIgTbMmpH73TPy_vDUPxQa37Hpmqn4r6No8PJwo,8188
-sunstone/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sunstone/validation.py,sha256=1356vcUc72a1zGBUe9Xjrcb5h41Xo53PaK2nnQ_FuSM,8286
-sunstone_py-0.4.2.dist-info/licenses/LICENSE,sha256=pB6VuR4QRjwjMjy8RSNGho-N1SUdu07ntIhT5lrhkzU,1078
-sunstone_py-0.4.2.dist-info/METADATA,sha256=ApYL9p7R7ibk6e3A4T43x9t5Cg2HzwiRBAwfmaIdFa0,9569
-sunstone_py-0.4.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sunstone_py-0.4.2.dist-info/entry_points.txt,sha256=0h6E88rH9a_503BAzXvFPR-UfmkrRFjcOf29DXgJNjk,51
-sunstone_py-0.4.2.dist-info/top_level.txt,sha256=A2fW-7JO10rlx_L28Bc4FVvWt2R8kgvS8_TGPBhQp3c,9
-sunstone_py-0.4.2.dist-info/RECORD,,