smallcat-chonk 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- smallcat/__init__.py +112 -0
- smallcat/_airflow.py +110 -0
- smallcat/catalog.py +642 -0
- smallcat/connections.py +524 -0
- smallcat/connections_config.py +351 -0
- smallcat/datasets/__init__.py +58 -0
- smallcat/datasets/base_dataset.py +891 -0
- smallcat/datasets/csv_dataset.py +161 -0
- smallcat/datasets/delta_table_dataset.py +447 -0
- smallcat/datasets/excel_dataset.py +130 -0
- smallcat/datasets/parquet_dataset.py +151 -0
- smallcat/models/__init__.py +65 -0
- smallcat/models/enums.py +72 -0
- smallcat/models/errors.py +60 -0
- smallcat/models/formats.py +106 -0
- smallcat/models/lookups.py +314 -0
- smallcat/models/meta.py +280 -0
- smallcat/models/model.py +101 -0
- smallcat/models/query.py +439 -0
- smallcat/models/registry.py +286 -0
- smallcat/models/schema.py +318 -0
- smallcat/path_utils.py +175 -0
- smallcat/py.typed +0 -0
- smallcat_chonk-0.6.0.dist-info/METADATA +214 -0
- smallcat_chonk-0.6.0.dist-info/RECORD +27 -0
- smallcat_chonk-0.6.0.dist-info/WHEEL +4 -0
- smallcat_chonk-0.6.0.dist-info/licenses/LICENSE +7 -0
smallcat/__init__.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Smallcat — a lightweight, data catalog for data loading and saving.
|
|
2
|
+
|
|
3
|
+
Smallcat provides a simple abstraction over data I/O by describing datasets,
|
|
4
|
+
connections, and their options declaratively in YAML, while exposing a clean,
|
|
5
|
+
typed Python API for loading and saving data in code.
|
|
6
|
+
|
|
7
|
+
Concepts
|
|
8
|
+
--------
|
|
9
|
+
* **Datasets**
|
|
10
|
+
Each entry in the catalog defines a file format (e.g., `parquet`, `csv`,
|
|
11
|
+
`excel`, `delta`). The format maps to a Dataset class that knows how to
|
|
12
|
+
read and write that file type using DuckDB and PyArrow under the hood.
|
|
13
|
+
|
|
14
|
+
* **Connections**
|
|
15
|
+
Each dataset references a connection that provides its base URI and
|
|
16
|
+
credentials. Connections follow the same structure as Airflow connections,
|
|
17
|
+
but Smallcat includes its own lightweight adapter so it can also run
|
|
18
|
+
standalone—without requiring Airflow to be installed.
|
|
19
|
+
|
|
20
|
+
* **Load and Save Options**
|
|
21
|
+
Datasets can define optional `load` and `save` sections in YAML, which
|
|
22
|
+
map to Pydantic models that validate the arguments passed to the underlying
|
|
23
|
+
reader or writer.
|
|
24
|
+
|
|
25
|
+
Airflow Compatibility
|
|
26
|
+
---------------------
|
|
27
|
+
Smallcat aims to integrate smoothly with Airflow but can operate completely
|
|
28
|
+
independently.
|
|
29
|
+
If Airflow is present, connections may be resolved by ID through
|
|
30
|
+
`BaseHook.get_connection`.
|
|
31
|
+
If not, connection dictionaries are handled directly through Smallcat's
|
|
32
|
+
`ConnectionLike` adapter.
|
|
33
|
+
|
|
34
|
+
Key Features
|
|
35
|
+
------------
|
|
36
|
+
* Declarative configuration of datasets and connections in YAML.
|
|
37
|
+
* Unified, typed API to load or save data as PyArrow or Polars objects.
|
|
38
|
+
* Automatic URI resolution (e.g., `file://`, `gs://`) from connection info.
|
|
39
|
+
* Extensible dataset types—new formats can be added by subclassing
|
|
40
|
+
:class:`smallcat.datasets.base_dataset.BaseDataset`.
|
|
41
|
+
|
|
42
|
+
Built-in Datasets
|
|
43
|
+
-----------------
|
|
44
|
+
Smallcat currently includes datasets for:
|
|
45
|
+
* Parquet
|
|
46
|
+
* CSV
|
|
47
|
+
* Excel (.xlsx)
|
|
48
|
+
* Delta Lake (delta-rs)
|
|
49
|
+
|
|
50
|
+
Each dataset class supports `load_arrow_table` and `save_arrow_table` for
|
|
51
|
+
consistent usage across all formats.
|
|
52
|
+
|
|
53
|
+
In short, Smallcat unifies the way your code accesses data across file formats
|
|
54
|
+
and environments—whether local, cloud, or orchestrated—while remaining simple,
|
|
55
|
+
typed, and dependency-light.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
import importlib
|
|
59
|
+
import typing
|
|
60
|
+
|
|
61
|
+
if typing.TYPE_CHECKING:
|
|
62
|
+
from smallcat.connections_config import Connections
|
|
63
|
+
from smallcat.models import (
|
|
64
|
+
Model,
|
|
65
|
+
ModelFileFormat,
|
|
66
|
+
ModelRegistry,
|
|
67
|
+
ModelValidationMode,
|
|
68
|
+
ModelWriteMode,
|
|
69
|
+
use_registry,
|
|
70
|
+
)
|
|
71
|
+
from smallcat.models.registry import default_registry as registry
|
|
72
|
+
|
|
73
|
+
# Public names are resolved on first access, so `import smallcat` stays light:
|
|
74
|
+
# it does not import Polars, DuckDB or Airflow. Each maps to (module, attribute).
|
|
75
|
+
_LAZY_EXPORTS = {
|
|
76
|
+
"Connections": ("smallcat.connections_config", "Connections"),
|
|
77
|
+
"Model": ("smallcat.models", "Model"),
|
|
78
|
+
"ModelFileFormat": ("smallcat.models", "ModelFileFormat"),
|
|
79
|
+
"ModelRegistry": ("smallcat.models", "ModelRegistry"),
|
|
80
|
+
"ModelValidationMode": ("smallcat.models", "ModelValidationMode"),
|
|
81
|
+
"ModelWriteMode": ("smallcat.models", "ModelWriteMode"),
|
|
82
|
+
"registry": ("smallcat.models.registry", "default_registry"),
|
|
83
|
+
"use_registry": ("smallcat.models", "use_registry"),
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
__all__ = [
|
|
87
|
+
"Connections",
|
|
88
|
+
"Model",
|
|
89
|
+
"ModelFileFormat",
|
|
90
|
+
"ModelRegistry",
|
|
91
|
+
"ModelValidationMode",
|
|
92
|
+
"ModelWriteMode",
|
|
93
|
+
"registry",
|
|
94
|
+
"use_registry",
|
|
95
|
+
]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def __getattr__(name: str) -> object:
|
|
99
|
+
"""Resolve the public names of the package on first access."""
|
|
100
|
+
target = _LAZY_EXPORTS.get(name)
|
|
101
|
+
if target is None:
|
|
102
|
+
msg = f"module 'smallcat' has no attribute {name!r}"
|
|
103
|
+
raise AttributeError(msg)
|
|
104
|
+
module, attribute = target
|
|
105
|
+
value = getattr(importlib.import_module(module), attribute)
|
|
106
|
+
globals()[name] = value
|
|
107
|
+
return value
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def __dir__() -> list[str]:
|
|
111
|
+
"""List the public names, including the lazy ones."""
|
|
112
|
+
return sorted({*globals(), *_LAZY_EXPORTS})
|
smallcat/_airflow.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Lazy access to Airflow, which is an optional dependency of Smallcat.
|
|
2
|
+
|
|
3
|
+
Importing Airflow is slow (hundreds of modules) and can have side effects, so
|
|
4
|
+
it must never happen when ``smallcat`` is imported. Callers resolve it on
|
|
5
|
+
demand through :func:`get_base_hook`.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
DISTRIBUTION_NAME = "smallcat-chonk"
|
|
12
|
+
"""The name this package is published under (``pip install smallcat-chonk``).
|
|
13
|
+
|
|
14
|
+
It is not the name to import: the package stays ``smallcat``. A test checks that
|
|
15
|
+
it matches ``name`` in ``pyproject.toml``, so a rename cannot leave it behind.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
AIRFLOW_INSTALL_HINT = (
|
|
19
|
+
f"Install it with `pip install {DISTRIBUTION_NAME}[airflow]`, or use the "
|
|
20
|
+
"Apache Airflow already available in your environment."
|
|
21
|
+
)
|
|
22
|
+
"""Appended to the errors raised when an operation needs Airflow."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def get_base_hook() -> Any: # noqa: ANN401
|
|
26
|
+
"""Return Airflow's ``BaseHook`` class, or ``None`` if Airflow is missing.
|
|
27
|
+
|
|
28
|
+
Tries the Airflow 3 SDK first and falls back to the Airflow 2 location, so
|
|
29
|
+
the same import path is used everywhere in Smallcat. Airflow is imported
|
|
30
|
+
on the first call, not when Smallcat is imported.
|
|
31
|
+
|
|
32
|
+
Returns:
|
|
33
|
+
The ``BaseHook`` class, or ``None`` when Airflow is not installed.
|
|
34
|
+
"""
|
|
35
|
+
try:
|
|
36
|
+
from airflow.sdk import BaseHook # type: ignore[attr-defined]
|
|
37
|
+
except ImportError:
|
|
38
|
+
try:
|
|
39
|
+
from airflow.hooks.base import BaseHook # type: ignore[attr-defined,no-redef]
|
|
40
|
+
except ImportError:
|
|
41
|
+
return None
|
|
42
|
+
return BaseHook
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
_MISSING_VARIABLE_MARKER = "SUPERVISOR_COMMS"
|
|
46
|
+
"""Early Airflow 3 releases raise an ``ImportError`` naming this symbol when a
|
|
47
|
+
Variable does not exist outside a task runner: they try to reach the task
|
|
48
|
+
supervisor over a channel that is not there. Not verified against those releases;
|
|
49
|
+
the symbol comes from how they were reported."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _is_variable_not_found(error: Exception) -> bool:
|
|
53
|
+
"""Tell whether ``error`` is Airflow 3's "this Variable does not exist".
|
|
54
|
+
|
|
55
|
+
Recent Airflow 3 releases (checked against 3.3.2) raise
|
|
56
|
+
``AirflowRuntimeError`` with ``error.error == ErrorType.VARIABLE_NOT_FOUND``.
|
|
57
|
+
The error is recognised by its shape instead of by importing
|
|
58
|
+
``airflow.sdk.exceptions``, which does not exist in Airflow 2.
|
|
59
|
+
"""
|
|
60
|
+
if type(error).__name__ != "AirflowRuntimeError":
|
|
61
|
+
return False
|
|
62
|
+
kind = getattr(getattr(error, "error", None), "error", None)
|
|
63
|
+
return getattr(kind, "name", None) == "VARIABLE_NOT_FOUND"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def get_variable_json(variable_id: str) -> Any: # noqa: ANN401
|
|
67
|
+
"""Return the content of an Airflow Variable holding JSON.
|
|
68
|
+
|
|
69
|
+
The Variable is read without asking Airflow to deserialize it, so a secrets
|
|
70
|
+
backend that already returns an object (for example
|
|
71
|
+
``LocalFilesystemBackend``) works too: a string is parsed as JSON and
|
|
72
|
+
anything else is returned as it is.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
variable_id: The Airflow Variable ID to read.
|
|
76
|
+
|
|
77
|
+
Returns:
|
|
78
|
+
The decoded content of the Variable.
|
|
79
|
+
|
|
80
|
+
Raises:
|
|
81
|
+
KeyError: If the Variable does not exist.
|
|
82
|
+
ValueError: If the Variable is not valid JSON. The message names the
|
|
83
|
+
Variable but never quotes its content, which may hold secrets.
|
|
84
|
+
ImportError: If Airflow, or something it needs, cannot be imported.
|
|
85
|
+
Only the errors Airflow 3 raises for a missing Variable are turned
|
|
86
|
+
into a ``KeyError``; anything else is left as it is.
|
|
87
|
+
"""
|
|
88
|
+
try:
|
|
89
|
+
from airflow.sdk import Variable
|
|
90
|
+
except ImportError:
|
|
91
|
+
from airflow.models import Variable # type: ignore[attr-defined,no-redef]
|
|
92
|
+
|
|
93
|
+
try:
|
|
94
|
+
value = Variable.get(variable_id)
|
|
95
|
+
except Exception as e:
|
|
96
|
+
missing = _is_variable_not_found(e) or (
|
|
97
|
+
isinstance(e, ImportError) and _MISSING_VARIABLE_MARKER in str(e)
|
|
98
|
+
)
|
|
99
|
+
if not missing:
|
|
100
|
+
raise
|
|
101
|
+
msg = f"Variable {variable_id} not found in Airflow"
|
|
102
|
+
raise KeyError(msg) from e
|
|
103
|
+
|
|
104
|
+
if isinstance(value, str):
|
|
105
|
+
try:
|
|
106
|
+
return json.loads(value)
|
|
107
|
+
except json.JSONDecodeError as e:
|
|
108
|
+
msg = f"Variable {variable_id} in Airflow is not valid JSON ({e.msg})"
|
|
109
|
+
raise ValueError(msg) from None
|
|
110
|
+
return value
|