dataframely 1.4.0__tar.gz → 1.5.0__tar.gz

{dataframely-1.4.0 → dataframely-1.5.0}/.copier-answers.yml

@@ -1,5 +1,5 @@
 # This file is managed by Copier; DO NOT EDIT OR REMOVE.
-_commit: v0.3.0
+_commit: v0.4.1
 _src_path: https://github.com/quantco/copier-template-python-open-source
 add_autobump_workflow: false
 author_email: oliver.borchert@quantco.com

dataframely-1.5.0/.gitattributes

@@ -0,0 +1 @@
+pixi.lock merge=binary linguist-language=YAML linguist-generated=true

{dataframely-1.4.0 → dataframely-1.5.0}/.github/workflows/build.yml

@@ -10,6 +10,8 @@ jobs:
   build-sdist:
     name: Build Sdist
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:

{dataframely-1.4.0 → dataframely-1.5.0}/.github/workflows/chore.yml

@@ -21,14 +21,14 @@ jobs:
     steps:
       - name: Check valid conventional commit message
         id: lint
-        uses: amannn/action-semantic-pull-request@v5
+        uses: amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017 # v5.5.3
         with:
           subjectPattern: ^[A-Z].+[^. ]$ # subject must start with uppercase letter and may not end with a dot/space
         env:
           GITHUB_TOKEN: ${{ github.token }}
       - name: Post comment about invalid PR title
         if: failure()
-        uses: marocchino/sticky-pull-request-comment@v2
+        uses: marocchino/sticky-pull-request-comment@52423e01640425a022ef5fd42c6fb5f633a02728 # v2.9.1
         with:
           header: conventional-commit-pr-title
           message: |
@@ -45,7 +45,7 @@ jobs:
             </details>
       - name: Delete comment about invalid PR title
         if: success()
-        uses: marocchino/sticky-pull-request-comment@v2
+        uses: marocchino/sticky-pull-request-comment@52423e01640425a022ef5fd42c6fb5f633a02728 # v2.9.1
         with:
           header: conventional-commit-pr-title
           delete: true
@@ -58,7 +58,7 @@ jobs:
       pull-requests: write
     steps:
       - name: ${{ github.event_name == 'pull_request_target' && 'Assign labels' || 'Update release draft' }}
-        uses: release-drafter/release-drafter@v6
+        uses: release-drafter/release-drafter@b1476f6e6eb133afa41ed8589daba6dc69b4d3f5 # v6.1.0
         with:
           disable-releaser: ${{ github.event_name == 'pull_request_target' }}
           disable-autolabeler: ${{ github.event_name == 'push' }}

{dataframely-1.4.0 → dataframely-1.5.0}/.github/workflows/ci.yml

@@ -9,6 +9,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   pre-commit-checks:
     name: Pre-commit Checks
@@ -40,7 +43,7 @@ jobs:
         environment: [py311, py312, py313]
     steps:
       - name: Checkout branch
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Set up pixi
         uses: prefix-dev/setup-pixi@19eac09b398e3d0c747adc7921926a6d802df4da # v0.8.8
         with:
@@ -50,7 +53,7 @@ jobs:
       - name: Run pytest
         run: pixi run -e ${{ matrix.environment }} test-coverage --color=yes
       - name: Upload codecov
-        uses: codecov/codecov-action@v5
+        uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
         with:
           files: ./coverage.xml
           token: ${{ secrets.CODECOV_TOKEN }}

dataframely-1.5.0/.github/workflows/scorecard.yml

@@ -0,0 +1,79 @@
+# This workflow uses actions that are not certified by GitHub. They are provided
+# by a third-party and are governed by separate terms of service, privacy
+# policy, and support documentation.
+
+name: Scorecard supply-chain security
+on:
+  # For Branch-Protection check. Only the default branch is supported. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+  branch_protection_rule:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+    - cron: "34 5 * * 0"
+  workflow_dispatch:
+  push:
+    branches: ["main"]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    # `publish_results: true` only works when run from the default branch. conditional can be removed if disabled.
+    if: (github.event.repository.default_branch == github.ref_name || github.event_name == 'pull_request') && github.repository == 'quantco/dataframely'
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+      # Uncomment the permissions below if installing in a private repository.
+      # contents: read
+      # actions: read
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
+          # - you want to enable the Branch-Protection check on a *public* repository, or
+          # - you are installing Scorecard on a *private* repository
+          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional.
+          # repo_token: ${{ secrets.SCORECARD_TOKEN }}
+
+          # Public repositories:
+          #   - Publish results to OpenSSF REST API for easy access by consumers
+          #   - Allows the repository to include the Scorecard badge.
+          #   - See https://github.com/ossf/scorecard-action#publishing-results.
+          # For private repositories:
+          #   - `publish_results` will always be set to `false`, regardless
+          #     of the value entered here.
+          publish_results: true
+
+          # (Optional) Uncomment file_mode if you have a .gitattributes with files marked export-ignore
+          # file_mode: git
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard (optional).
+      # Commenting out will disable upload of results to your repo's Code Scanning dashboard
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
+        with:
+          sarif_file: results.sarif

{dataframely-1.4.0 → dataframely-1.5.0}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: dataframely
-Version: 1.4.0
+Version: 1.5.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: numpy
-Requires-Dist: polars>=1.12
+Requires-Dist: polars>=1.30
 License-File: LICENSE
 Summary: A declarative, polars-native data frame validation library
 Author-email: Andreas Albert <andreas.albert@quantco.com>, Daniel Elsner <daniel.elsner@quantco.com>, Oliver Borchert <oliver.borchert@quantco.com>

dataframely-1.5.0/SECURITY.md

@@ -0,0 +1,7 @@
+# Reporting Security Issues
+
+We take security bugs in our projects seriously. We appreciate your efforts to responsibly disclose your findings, and will make every effort to acknowledge your contributions.
+
+To report a security issue, please use the GitHub Security Advisory ["Report a Vulnerability"](https://github.com/quantco/dataframely/security/advisories/new) tab.
+
+We will send a response indicating the next steps in handling your report. After the initial reply to your report, the security team will keep you informed of the progress towards a fix and full announcement, and may ask for additional information or guidance.

{dataframely-1.4.0 → dataframely-1.5.0}/dataframely/collection.py

@@ -1,12 +1,11 @@
 # Copyright (c) QuantCo 2025-2025
 # SPDX-License-Identifier: BSD-3-Clause
 
-import sys
 import warnings
 from abc import ABC
 from collections.abc import Mapping, Sequence
 from pathlib import Path
-from typing import Any, Generic, Self, TypeVar, cast
+from typing import Any, Self, cast
 
 import polars as pl
 import polars.exceptions as plexc
@@ -18,15 +17,8 @@ from ._polars import FrameType, join_all_inner, join_all_outer
 from .failure import FailureInfo
 from .random import Generator
 
-if sys.version_info >= (3, 13):
-    SamplingType = TypeVar(
-        "SamplingType", bound=Mapping[str, Any], default=Mapping[str, Any]
-    )
-else:  # pragma: no cover
-    SamplingType = TypeVar("SamplingType", bound=Mapping[str, Any])
 
-
-class Collection(BaseCollection, ABC, Generic[SamplingType]):
+class Collection(BaseCollection, ABC):
     """Base class for all collections of data frames with a predefined schema.
 
     A collection is comprised of a set of *members* which are collectively "consistent",
@@ -86,7 +78,7 @@ class Collection(BaseCollection, ABC, Generic[SamplingType]):
         cls,
         num_rows: int | None = None,
         *,
-        overrides: Sequence[SamplingType] | None = None,
+        overrides: Sequence[Mapping[str, Any]] | None = None,
         generator: Generator | None = None,
     ) -> Self:
         """Create a random sample from the members of this collection.
@@ -162,10 +154,11 @@ class Collection(BaseCollection, ABC, Generic[SamplingType]):
         samples = (
             overrides
             if overrides is not None
-            else [cast(SamplingType, {}) for _ in range(cast(int, num_rows))]
+            else [{} for _ in range(cast(int, num_rows))]
         )
         processed_samples = [
-            cls._preprocess_sample(sample, i, g) for i, sample in enumerate(samples)
+            cls._preprocess_sample(dict(sample.items()), i, g)
+            for i, sample in enumerate(samples)
         ]
 
         # 2) Ensure that all samples have primary keys assigned to ensure that we
@@ -234,8 +227,8 @@ class Collection(BaseCollection, ABC, Generic[SamplingType]):
 
     @classmethod
     def _preprocess_sample(
-        cls, sample: SamplingType, index: int, generator: Generator
-    ) -> SamplingType:
+        cls, sample: dict[str, Any], index: int, generator: Generator
+    ) -> dict[str, Any]:
         """Overridable method to preprocess a sample passed to :meth:`sample`.
 
         The purpose of this method is to (1) set the primary key columns to enable

{dataframely-1.4.0 → dataframely-1.5.0}/dataframely/columns/datetime.py

@@ -8,6 +8,7 @@ from collections.abc import Callable
 from typing import Any, cast
 
 import polars as pl
+from polars._typing import TimeUnit
 
 from dataframely._compat import pa, sa, sa_mssql, sa_TypeEngine
 from dataframely._polars import (
@@ -300,6 +301,7 @@ class Datetime(OrdinalMixin[dt.datetime], Column):
         max_exclusive: dt.datetime | None = None,
         resolution: str | None = None,
         time_zone: str | dt.tzinfo | None = None,
+        time_unit: TimeUnit = "us",
         check: (
             Callable[[pl.Expr], pl.Expr]
             | list[Callable[[pl.Expr], pl.Expr]]
@@ -330,6 +332,7 @@ class Datetime(OrdinalMixin[dt.datetime], Column):
             time_zone: The time zone that datetimes in the column must have. The time
                 zone must use a valid IANA time zone name identifier e.x. ``Etc/UTC`` or
                 ``America/New_York``.
+            time_unit: Unit of time. Defaults to ``us`` (microseconds).
             check: A custom rule or multiple rules to run for this column. This can be:
                 - A single callable that returns a non-aggregated boolean expression.
                 The name of the rule is derived from the callable name, or defaults to
@@ -373,10 +376,11 @@ class Datetime(OrdinalMixin[dt.datetime], Column):
         )
         self.resolution = resolution
         self.time_zone = time_zone
+        self.time_unit = time_unit
 
     @property
     def dtype(self) -> pl.DataType:
-        return pl.Datetime(time_zone=self.time_zone)
+        return pl.Datetime(time_zone=self.time_zone, time_unit=self.time_unit)
 
     def validation_rules(self, expr: pl.Expr) -> dict[str, pl.Expr]:
         result = super().validation_rules(expr)
@@ -400,7 +404,7 @@ class Datetime(OrdinalMixin[dt.datetime], Column):
             if isinstance(self.time_zone, dt.tzinfo)
             else self.time_zone
         )
-        return pa.timestamp("us", time_zone)
+        return pa.timestamp(self.time_unit, time_zone)
 
     def _sample_unchecked(self, generator: Generator, n: int) -> pl.Series:
         return generator.sample_datetime(
@@ -416,6 +420,7 @@ class Datetime(OrdinalMixin[dt.datetime], Column):
                 allow_null_response=True,
             ),
             resolution=self.resolution,
+            time_unit=self.time_unit,
             time_zone=self.time_zone,
             null_probability=self._null_probability,
         )

{dataframely-1.4.0 → dataframely-1.5.0}/dataframely/failure.py

@@ -79,17 +79,13 @@ class FailureInfo(Generic[S]):
         Args:
             file: The file path or writable file-like object to write to.
         """
-        # NOTE: We add a dummy column with metadata in the column name to allow writing
-        #  the rule columns and schema to the same file.
         metadata_json = json.dumps(
             {
                 "rule_columns": self._rule_columns,
                 "schema": f"{self.schema.__module__}.{self.schema.__name__}",
             }
         )
-        self._df.with_columns(
-            pl.lit(None).alias(metadata_json),
-        ).write_parquet(file)
+        self._df.write_parquet(file, metadata={"dataframely": metadata_json})
 
     @classmethod
     def scan_parquet(cls, source: str | Path | IO[bytes]) -> Self:
@@ -102,15 +98,27 @@ class FailureInfo(Generic[S]):
             The failure info object.
         """
         lf = pl.scan_parquet(source)
-        # NOTE: In `write_parquet`, the rule columns are added as the name of the last
-        #  column.
-        last_column = lf.collect_schema().names()[-1]
-        metadata = json.loads(last_column)
-        rule_columns = metadata["rule_columns"]
-        *schema_module_parts, schema_name = metadata["schema"].split(".")
+
+        # We can read the rule columns either from the metadata of the Parquet file
+        # or, to remain backwards-compatible, from the last column of the lazy frame if
+        # the parquet file is missing metadata.
+        rule_columns: list[str]
+        schema_name: str
+        if (meta := pl.read_parquet_metadata(source).get("dataframely")) is not None:
+            metadata = json.loads(meta)
+            rule_columns = metadata["rule_columns"]
+            schema_name = metadata["schema"]
+        else:
+            last_column = lf.collect_schema().names()[-1]
+            metadata = json.loads(last_column)
+            rule_columns = metadata["rule_columns"]
+            schema_name = metadata["schema"]
+            lf = lf.drop(last_column)
+
+        *schema_module_parts, schema_name = schema_name.split(".")
         module = importlib.import_module(".".join(schema_module_parts))
         schema = cast(type[S], getattr(module, schema_name))
-        return cls(lf.drop(last_column), rule_columns, schema=schema)
+        return cls(lf, rule_columns, schema=schema)
 
 
 # ------------------------------------ COMPUTATION ----------------------------------- #

{dataframely-1.4.0 → dataframely-1.5.0}/dataframely/random.py

@@ -7,6 +7,7 @@ from typing import TypeVar
 
 import numpy as np
 import polars as pl
+from polars._typing import TimeUnit
 
 from ._extre import sample as extre_sample
 from ._polars import (
@@ -294,6 +295,7 @@ class Generator:
         max: dt.datetime | None,
         resolution: str | None = None,
         time_zone: str | dt.tzinfo | None = None,
+        time_unit: TimeUnit = "us",
         null_probability: float = 0.0,
     ) -> pl.Series:
         """Sample a list of datetimes in the provided range.
@@ -303,7 +305,9 @@ class Generator:
             min: The minimum datetime to sample (inclusive).
             max: The maximum datetime to sample (exclusive). '10000-01-01' when ``None``.
             resolution: The resolution that datetimes in the column must have. This uses
-                the formatting language used by :mod:`polars` datetime ``round`` method.
+                the formatting language used by :mod:`polars` datetime ``round``
+                method.
+            time_unit: The time unit of the datetime column. Defaults to ``us`` (microseconds).
             time_zone: The time zone that datetimes in the column must have. The time
                 zone must use a valid IANA time zone name identifier e.x. ``Etc/UTC`` or
                 ``America/New_York``.
@@ -333,7 +337,7 @@ class Generator:
             )
             # NOTE: polars tracks datetimes relative to epoch
             - _datetime_to_microseconds(EPOCH_DATETIME)
-        ).cast(pl.Datetime(time_zone=time_zone))
+        ).cast(pl.Datetime(time_unit=time_unit, time_zone=time_zone))
 
         if resolution is not None:
             return result.dt.truncate(resolution)