smallcat-chonk 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
smallcat/__init__.py ADDED
@@ -0,0 +1,112 @@
1
+ """Smallcat — a lightweight, data catalog for data loading and saving.
2
+
3
+ Smallcat provides a simple abstraction over data I/O by describing datasets,
4
+ connections, and their options declaratively in YAML, while exposing a clean,
5
+ typed Python API for loading and saving data in code.
6
+
7
+ Concepts
8
+ --------
9
+ * **Datasets**
10
+ Each entry in the catalog defines a file format (e.g., `parquet`, `csv`,
11
+ `excel`, `delta`). The format maps to a Dataset class that knows how to
12
+ read and write that file type using DuckDB and PyArrow under the hood.
13
+
14
+ * **Connections**
15
+ Each dataset references a connection that provides its base URI and
16
+ credentials. Connections follow the same structure as Airflow connections,
17
+ but Smallcat includes its own lightweight adapter so it can also run
18
+ standalone—without requiring Airflow to be installed.
19
+
20
+ * **Load and Save Options**
21
+ Datasets can define optional `load` and `save` sections in YAML, which
22
+ map to Pydantic models that validate the arguments passed to the underlying
23
+ reader or writer.
24
+
25
+ Airflow Compatibility
26
+ ---------------------
27
+ Smallcat aims to integrate smoothly with Airflow but can operate completely
28
+ independently.
29
+ If Airflow is present, connections may be resolved by ID through
30
+ `BaseHook.get_connection`.
31
+ If not, connection dictionaries are handled directly through Smallcat's
32
+ `ConnectionLike` adapter.
33
+
34
+ Key Features
35
+ ------------
36
+ * Declarative configuration of datasets and connections in YAML.
37
+ * Unified, typed API to load or save data as PyArrow or Polars objects.
38
+ * Automatic URI resolution (e.g., `file://`, `gs://`) from connection info.
39
+ * Extensible dataset types—new formats can be added by subclassing
40
+ :class:`smallcat.datasets.base_dataset.BaseDataset`.
41
+
42
+ Built-in Datasets
43
+ -----------------
44
+ Smallcat currently includes datasets for:
45
+ * Parquet
46
+ * CSV
47
+ * Excel (.xlsx)
48
+ * Delta Lake (delta-rs)
49
+
50
+ Each dataset class supports `load_arrow_table` and `save_arrow_table` for
51
+ consistent usage across all formats.
52
+
53
+ In short, Smallcat unifies the way your code accesses data across file formats
54
+ and environments—whether local, cloud, or orchestrated—while remaining simple,
55
+ typed, and dependency-light.
56
+ """
57
+
58
+ import importlib
59
+ import typing
60
+
61
+ if typing.TYPE_CHECKING:
62
+ from smallcat.connections_config import Connections
63
+ from smallcat.models import (
64
+ Model,
65
+ ModelFileFormat,
66
+ ModelRegistry,
67
+ ModelValidationMode,
68
+ ModelWriteMode,
69
+ use_registry,
70
+ )
71
+ from smallcat.models.registry import default_registry as registry
72
+
73
+ # Public names are resolved on first access, so `import smallcat` stays light:
74
+ # it does not import Polars, DuckDB or Airflow. Each maps to (module, attribute).
75
+ _LAZY_EXPORTS = {
76
+ "Connections": ("smallcat.connections_config", "Connections"),
77
+ "Model": ("smallcat.models", "Model"),
78
+ "ModelFileFormat": ("smallcat.models", "ModelFileFormat"),
79
+ "ModelRegistry": ("smallcat.models", "ModelRegistry"),
80
+ "ModelValidationMode": ("smallcat.models", "ModelValidationMode"),
81
+ "ModelWriteMode": ("smallcat.models", "ModelWriteMode"),
82
+ "registry": ("smallcat.models.registry", "default_registry"),
83
+ "use_registry": ("smallcat.models", "use_registry"),
84
+ }
85
+
86
+ __all__ = [
87
+ "Connections",
88
+ "Model",
89
+ "ModelFileFormat",
90
+ "ModelRegistry",
91
+ "ModelValidationMode",
92
+ "ModelWriteMode",
93
+ "registry",
94
+ "use_registry",
95
+ ]
96
+
97
+
98
+ def __getattr__(name: str) -> object:
99
+ """Resolve the public names of the package on first access."""
100
+ target = _LAZY_EXPORTS.get(name)
101
+ if target is None:
102
+ msg = f"module 'smallcat' has no attribute {name!r}"
103
+ raise AttributeError(msg)
104
+ module, attribute = target
105
+ value = getattr(importlib.import_module(module), attribute)
106
+ globals()[name] = value
107
+ return value
108
+
109
+
110
+ def __dir__() -> list[str]:
111
+ """List the public names, including the lazy ones."""
112
+ return sorted({*globals(), *_LAZY_EXPORTS})
smallcat/_airflow.py ADDED
@@ -0,0 +1,110 @@
1
+ """Lazy access to Airflow, which is an optional dependency of Smallcat.
2
+
3
+ Importing Airflow is slow (hundreds of modules) and can have side effects, so
4
+ it must never happen when ``smallcat`` is imported. Callers resolve it on
5
+ demand through :func:`get_base_hook`.
6
+ """
7
+
8
+ import json
9
+ from typing import Any
10
+
11
+ DISTRIBUTION_NAME = "smallcat-chonk"
12
+ """The name this package is published under (``pip install smallcat-chonk``).
13
+
14
+ It is not the name to import: the package stays ``smallcat``. A test checks that
15
+ it matches ``name`` in ``pyproject.toml``, so a rename cannot leave it behind.
16
+ """
17
+
18
+ AIRFLOW_INSTALL_HINT = (
19
+ f"Install it with `pip install {DISTRIBUTION_NAME}[airflow]`, or use the "
20
+ "Apache Airflow already available in your environment."
21
+ )
22
+ """Appended to the errors raised when an operation needs Airflow."""
23
+
24
+
25
+ def get_base_hook() -> Any: # noqa: ANN401
26
+ """Return Airflow's ``BaseHook`` class, or ``None`` if Airflow is missing.
27
+
28
+ Tries the Airflow 3 SDK first and falls back to the Airflow 2 location, so
29
+ the same import path is used everywhere in Smallcat. Airflow is imported
30
+ on the first call, not when Smallcat is imported.
31
+
32
+ Returns:
33
+ The ``BaseHook`` class, or ``None`` when Airflow is not installed.
34
+ """
35
+ try:
36
+ from airflow.sdk import BaseHook # type: ignore[attr-defined]
37
+ except ImportError:
38
+ try:
39
+ from airflow.hooks.base import BaseHook # type: ignore[attr-defined,no-redef]
40
+ except ImportError:
41
+ return None
42
+ return BaseHook
43
+
44
+
45
+ _MISSING_VARIABLE_MARKER = "SUPERVISOR_COMMS"
46
+ """Early Airflow 3 releases raise an ``ImportError`` naming this symbol when a
47
+ Variable does not exist outside a task runner: they try to reach the task
48
+ supervisor over a channel that is not there. Not verified against those releases;
49
+ the symbol comes from how they were reported."""
50
+
51
+
52
+ def _is_variable_not_found(error: Exception) -> bool:
53
+ """Tell whether ``error`` is Airflow 3's "this Variable does not exist".
54
+
55
+ Recent Airflow 3 releases (checked against 3.3.2) raise
56
+ ``AirflowRuntimeError`` with ``error.error == ErrorType.VARIABLE_NOT_FOUND``.
57
+ The error is recognised by its shape instead of by importing
58
+ ``airflow.sdk.exceptions``, which does not exist in Airflow 2.
59
+ """
60
+ if type(error).__name__ != "AirflowRuntimeError":
61
+ return False
62
+ kind = getattr(getattr(error, "error", None), "error", None)
63
+ return getattr(kind, "name", None) == "VARIABLE_NOT_FOUND"
64
+
65
+
66
+ def get_variable_json(variable_id: str) -> Any: # noqa: ANN401
67
+ """Return the content of an Airflow Variable holding JSON.
68
+
69
+ The Variable is read without asking Airflow to deserialize it, so a secrets
70
+ backend that already returns an object (for example
71
+ ``LocalFilesystemBackend``) works too: a string is parsed as JSON and
72
+ anything else is returned as it is.
73
+
74
+ Args:
75
+ variable_id: The Airflow Variable ID to read.
76
+
77
+ Returns:
78
+ The decoded content of the Variable.
79
+
80
+ Raises:
81
+ KeyError: If the Variable does not exist.
82
+ ValueError: If the Variable is not valid JSON. The message names the
83
+ Variable but never quotes its content, which may hold secrets.
84
+ ImportError: If Airflow, or something it needs, cannot be imported.
85
+ Only the errors Airflow 3 raises for a missing Variable are turned
86
+ into a ``KeyError``; anything else is left as it is.
87
+ """
88
+ try:
89
+ from airflow.sdk import Variable
90
+ except ImportError:
91
+ from airflow.models import Variable # type: ignore[attr-defined,no-redef]
92
+
93
+ try:
94
+ value = Variable.get(variable_id)
95
+ except Exception as e:
96
+ missing = _is_variable_not_found(e) or (
97
+ isinstance(e, ImportError) and _MISSING_VARIABLE_MARKER in str(e)
98
+ )
99
+ if not missing:
100
+ raise
101
+ msg = f"Variable {variable_id} not found in Airflow"
102
+ raise KeyError(msg) from e
103
+
104
+ if isinstance(value, str):
105
+ try:
106
+ return json.loads(value)
107
+ except json.JSONDecodeError as e:
108
+ msg = f"Variable {variable_id} in Airflow is not valid JSON ({e.msg})"
109
+ raise ValueError(msg) from None
110
+ return value