DataExcept 1.5.0__tar.gz → 1.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataexcept-1.5.0 → dataexcept-1.7.0}/CHANGELOG.md +106 -8
- {dataexcept-1.5.0 → dataexcept-1.7.0}/CITATION.cff +2 -2
- {dataexcept-1.5.0 → dataexcept-1.7.0}/PKG-INFO +23 -5
- {dataexcept-1.5.0 → dataexcept-1.7.0}/README.md +22 -4
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/__init__.py +32 -12
- dataexcept-1.7.0/dataexcept/broker_exceptions.py +247 -0
- dataexcept-1.7.0/dataexcept/http_context.py +136 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/logging_helpers.py +42 -10
- dataexcept-1.7.0/dataexcept/observability.py +107 -0
- dataexcept-1.7.0/dataexcept/opentelemetry.py +173 -0
- dataexcept-1.7.0/dataexcept/orchestrator_context.py +110 -0
- dataexcept-1.7.0/dataexcept/pino.py +158 -0
- dataexcept-1.7.0/dataexcept/schema.py +82 -0
- dataexcept-1.7.0/dataexcept/schemas/pino-1.0.0.json +96 -0
- dataexcept-1.7.0/dataexcept/sentry.py +111 -0
- dataexcept-1.7.0/dataexcept/trace_context.py +192 -0
- dataexcept-1.7.0/dataexcept/worker_context.py +167 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/pyproject.toml +3 -3
- dataexcept-1.5.0/dataexcept/schema.py +0 -61
- {dataexcept-1.5.0 → dataexcept-1.7.0}/LICENSE +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/__main__.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/_causes.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/_previews.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/_validation.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/base.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/database_exceptions.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/dataengineering_exceptions.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/datascience_exceptions/__init__.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/datascience_exceptions/base.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/datascience_exceptions/ingestion.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/datascience_exceptions/operations.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/datascience_exceptions/training.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/exceptions/__init__.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/exceptions/authentication.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/exceptions/base.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/exceptions/configuration.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/exceptions/external.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/exceptions/lifecycle.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/exceptions/notification.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/exceptions/parsing.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/exceptions/scheduling.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/exceptions/validation.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/failure_metadata.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/io_exceptions.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/network_exceptions.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/pandas_exceptions.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/pipeline_exceptions.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/py.typed +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/redaction.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/schemas/envelope-1.0.0.json +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/security_exceptions.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/serialization.py +0 -0
- {dataexcept-1.5.0 → dataexcept-1.7.0}/dataexcept/wrapping.py +0 -0
|
@@ -7,18 +7,114 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [1.7.0] - 2026-09-17
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- **A product-neutral observability context.** `OperationContext` carries
|
|
15
|
+
stable system, component and operation labels separately from request, job,
|
|
16
|
+
correlation, trace and span identifiers. `exception_to_observability_event()`
|
|
17
|
+
combines that context with the existing redacted exception envelope without
|
|
18
|
+
coupling DataExcept to a framework or telemetry SDK.
|
|
19
|
+
- **Dependency-free OpenTelemetry support.** Exception conversion exposes the
|
|
20
|
+
standard `exception.*` attributes plus `dataexcept.failure.*` recovery
|
|
21
|
+
metadata and can record an exception through the minimal span interface
|
|
22
|
+
without importing OpenTelemetry.
|
|
23
|
+
- **Dependency-free Sentry enrichment.** Sentry events can carry the redacted
|
|
24
|
+
DataExcept envelope and operation context while only low-cardinality fields
|
|
25
|
+
become tags.
|
|
26
|
+
- **W3C Trace Context continuity.** `traceparent` can be parsed without a
|
|
27
|
+
tracing runtime, with `tracestate` and `baggage` preserved for forwarding.
|
|
28
|
+
Invalid or all-zero identifiers are rejected rather than replaced with
|
|
29
|
+
invented provenance, and an incoming parent span ID is never mislabelled as
|
|
30
|
+
a local span ID.
|
|
31
|
+
- **Framework-neutral execution-boundary adapters** for HTTP requests,
|
|
32
|
+
background workers and workflow/orchestrator steps. They map stable operation
|
|
33
|
+
names and correlation identifiers onto the common context while keeping raw
|
|
34
|
+
request paths, payloads and task arguments out of indexed telemetry. The
|
|
35
|
+
adapters depend only on plain mappings and therefore fit ASGI/WSGI services,
|
|
36
|
+
serverless gateways, Celery/RQ/Arq/Dramatiq-style workers, Airflow/Dagster/
|
|
37
|
+
Prefect/Argo-style orchestrators and custom runtimes without importing them.
|
|
38
|
+
|
|
39
|
+
### Changed
|
|
40
|
+
|
|
41
|
+
- Logging helpers can attach `OperationContext` separately from free-form
|
|
42
|
+
application context, keeping filterable operation identity distinct from
|
|
43
|
+
arbitrary payload-shaped values.
|
|
44
|
+
- OpenTelemetry and Sentry now project the same operation context so logs,
|
|
45
|
+
traces and error trackers can correlate one failure consistently.
|
|
46
|
+
- The documentation home page and the README now link every guide the site
|
|
47
|
+
publishes. Each release had added its guide to the navigation and stopped
|
|
48
|
+
there, leaving cause-aware exceptions, failure metadata, parsing context,
|
|
49
|
+
message brokers and the Pino profile reachable only from the sidebar. A test
|
|
50
|
+
now fails when a page in the navigation is linked from neither landing page,
|
|
51
|
+
so the two cannot drift apart again.
|
|
52
|
+
|
|
53
|
+
## [1.6.0] - 2026-09-06
|
|
54
|
+
|
|
55
|
+
### Added
|
|
56
|
+
|
|
57
|
+
- **A first-class message-broker hierarchy.** `MessageBrokerError` is a domain
|
|
58
|
+
root of its own, with `BrokerConnectionError`, `BrokerTimeoutError`,
|
|
59
|
+
`MessagePublishError`, `MessageConsumeError` and
|
|
60
|
+
`MessageAcknowledgementError` beneath it. Broker work was reaching for
|
|
61
|
+
`ServiceConnectionError` and `OperationTimeoutError`, which catch the failure
|
|
62
|
+
but lose which operation failed and the topic, partition, offset and consumer
|
|
63
|
+
group it failed at.
|
|
64
|
+
- Structured broker context — `broker`, `topic`, `partition`, `offset`,
|
|
65
|
+
`consumer_group`, `operation` and `timeout_seconds` — with `broker` redacted
|
|
66
|
+
when it is a URL, since a bootstrap address routinely carries credentials.
|
|
67
|
+
The five concrete classes take `cause=` on the canonical contract and set
|
|
68
|
+
`__cause__`; `MessageBrokerError` itself is a plain base, for catching.
|
|
69
|
+
- **A published Pino profile for the exception envelope.** `exception_to_pino()`
|
|
70
|
+
and `envelope_to_pino()` project an envelope onto the value Pino logs under
|
|
71
|
+
its error key, and `pino_profile_schema()`, `PINO_PROFILE_VERSION` and
|
|
72
|
+
`PINO_PROFILE_ID` expose its versioned JSON Schema. The projection renames
|
|
73
|
+
`exceptions` to `errors`, matching JavaScript's `AggregateError`, and changes
|
|
74
|
+
nothing else: identity, attributes, failure metadata, the whole cause,
|
|
75
|
+
context and member tree, and the redaction already applied to them all come
|
|
76
|
+
through unchanged.
|
|
77
|
+
- An optional `stack` field, emitted only when a real stack representation
|
|
78
|
+
exists. `include_stack=True` renders the exception's own traceback, redacted
|
|
79
|
+
like any other exported string, and omits the field rather than inventing one
|
|
80
|
+
when the exception was never raised.
|
|
81
|
+
- The Pino projection of every published envelope fixture, beside it under
|
|
82
|
+
`docs/schema/fixtures/pino/`, so an implementation in another language can
|
|
83
|
+
check itself against a pair of files rather than against prose.
|
|
84
|
+
- Documented JavaScript and TypeScript usage, including the projection restated
|
|
85
|
+
in JavaScript and a contract test over the fixture pairs.
|
|
86
|
+
|
|
87
|
+
Neither addition brings a dependency. The broker hierarchy describes the
|
|
88
|
+
operation rather than the product, so it fits Kafka, RabbitMQ, Pulsar and NATS
|
|
89
|
+
without any broker client; retryability is left unclassified, because a
|
|
90
|
+
rejected publish may be a leader election or a missing topic and the exception
|
|
91
|
+
cannot tell which. The Pino profile adds a JSON document and a function that
|
|
92
|
+
renames one field — no Node.js or Pino, in either direction.
|
|
93
|
+
|
|
94
|
+
### Changed
|
|
95
|
+
|
|
96
|
+
- Pull requests now run the release gate's wheel verification: the built wheel
|
|
97
|
+
is installed, the source tree is deleted, and the suite runs against the
|
|
98
|
+
installed distribution. A test that quietly depends on the source tree used
|
|
99
|
+
to surface only at the last job before publication, which is where it
|
|
100
|
+
surfaced for 1.5.0.
|
|
101
|
+
- The pre-commit hooks are pinned to the versions `poetry.lock` pins, and a
|
|
102
|
+
test derives one from the other. They had drifted to isort 8 and ruff 0.16.4
|
|
103
|
+
against a lock on isort 9 and ruff 0.16.5, which is enough to format a commit
|
|
104
|
+
differently from what CI then checks.
|
|
105
|
+
|
|
10
106
|
## [1.5.0] - 2026-09-06
|
|
11
107
|
|
|
12
108
|
### Added
|
|
13
109
|
|
|
14
110
|
- **Redaction-safe source context for parsing failures.** `ParsingError` and
|
|
15
|
-
`DeserializationError`
|
|
16
|
-
`
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
public attribute.
|
|
111
|
+
`DeserializationError` take keyword-only `source`, `preview` and `cause`, and
|
|
112
|
+
`ParsingError` gains a keyword-only `format` that `DeserializationError`
|
|
113
|
+
already took positionally — so a failure on untrusted content can be
|
|
114
|
+
described without keeping the content. `source` is redacted when it is a URL
|
|
115
|
+
and left alone when it is a file path; `preview` is bounded to 200 characters
|
|
116
|
+
including its truncation marker, decodes bytes as UTF-8 with any undecodable
|
|
117
|
+
byte shown as `\xNN`, and is redacted like any other public attribute.
|
|
22
118
|
- `cause=` on both classes, following the canonical 1.4.0 contract: recorded as
|
|
23
119
|
`.cause` and set as `__cause__`.
|
|
24
120
|
- **A published JSON Schema for the structured envelope.** The payload
|
|
@@ -666,7 +762,9 @@ First public release.
|
|
|
666
762
|
- Published to PyPI via OIDC trusted publishing; no long-lived API token is
|
|
667
763
|
involved in a release.
|
|
668
764
|
|
|
669
|
-
[Unreleased]: https://github.com/DiogoRibeiro7/DataExcept/compare/v1.
|
|
765
|
+
[Unreleased]: https://github.com/DiogoRibeiro7/DataExcept/compare/v1.7.0...HEAD
|
|
766
|
+
[1.7.0]: https://github.com/DiogoRibeiro7/DataExcept/compare/v1.6.0...v1.7.0
|
|
767
|
+
[1.6.0]: https://github.com/DiogoRibeiro7/DataExcept/compare/v1.5.0...v1.6.0
|
|
670
768
|
[1.5.0]: https://github.com/DiogoRibeiro7/DataExcept/compare/v1.4.0...v1.5.0
|
|
671
769
|
[1.4.0]: https://github.com/DiogoRibeiro7/DataExcept/compare/v1.3.0...v1.4.0
|
|
672
770
|
[1.3.0]: https://github.com/DiogoRibeiro7/DataExcept/compare/v1.2.0...v1.3.0
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
cff-version: 1.2.0
|
|
2
2
|
message: "If you use this software, please cite it using the following metadata."
|
|
3
3
|
title: "DataExcept"
|
|
4
|
-
version: "1.
|
|
4
|
+
version: "1.7.0"
|
|
5
5
|
authors:
|
|
6
6
|
- family-names: "Ribeiro"
|
|
7
7
|
given-names: "Diogo"
|
|
@@ -11,4 +11,4 @@ authors:
|
|
|
11
11
|
type: software
|
|
12
12
|
url: "https://github.com/DiogoRibeiro7/DataExcept"
|
|
13
13
|
repository-code: "https://github.com/DiogoRibeiro7/DataExcept"
|
|
14
|
-
date-released: "2026-09-
|
|
14
|
+
date-released: "2026-09-17"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: DataExcept
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.7.0
|
|
4
4
|
Summary: A Python package providing structured, easily-extendable custom exception types.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -44,8 +44,8 @@ Description-Content-Type: text/markdown
|
|
|
44
44
|
|
|
45
45
|
- **🏗️ Hierarchical Structure**: Catch one specific error, a whole domain, or every operational error via `DataExceptError`
|
|
46
46
|
- **📦 One Import**: Every exception is available from `dataexcept` directly, or from its domain module — same objects either way
|
|
47
|
-
- **📊 Data Science Focused**:
|
|
48
|
-
- **🧭 Language-neutral envelopes**: exceptions export to strict JSON against a published, versioned schema — so a non-Python consumer can validate what it receives
|
|
47
|
+
- **📊 Data Science Focused**: 106 exception classes covering ML pipelines, feature engineering, model training
|
|
48
|
+
- **🧭 Language-neutral envelopes**: exceptions export to strict JSON against a published, versioned schema — so a non-Python consumer can validate what it receives, or read it through the [Pino projection](https://diogoribeiro7.github.io/DataExcept/pino/)
|
|
49
49
|
- **🔧 Production Ready**: Logging helpers, error context, and exceptions that pickle — so they cross a process boundary with their message, attributes and cause intact
|
|
50
50
|
- **📚 Academic Quality**: Proper documentation, type hints, and citation support
|
|
51
51
|
- **🐍 Python 3.10 – 3.14**: Every supported version tested in CI, with full type safety
|
|
@@ -186,6 +186,17 @@ DatabaseConnectionError("postgresql://prod-db:5432/analytics")
|
|
|
186
186
|
QueryExecutionError("SELECT * FROM large_table", original=TimeoutError())
|
|
187
187
|
```
|
|
188
188
|
|
|
189
|
+
### 📨 Message Brokers
|
|
190
|
+
|
|
191
|
+
```python
|
|
192
|
+
from dataexcept.broker_exceptions import *
|
|
193
|
+
|
|
194
|
+
BrokerConnectionError("kafka://broker.internal:9092", cause=OSError("refused"))
|
|
195
|
+
MessagePublishError("orders", partition=3, cause=OSError("leader not available"))
|
|
196
|
+
MessageConsumeError("orders", partition=3, offset=1042, consumer_group="billing")
|
|
197
|
+
MessageAcknowledgementError("orders", partition=3, offset=1042)
|
|
198
|
+
```
|
|
199
|
+
|
|
189
200
|
## 🔍 Advanced Features
|
|
190
201
|
|
|
191
202
|
### Smart Logging Integration
|
|
@@ -233,7 +244,7 @@ document Python emits.
|
|
|
233
244
|
### Command Line Interface
|
|
234
245
|
|
|
235
246
|
```bash
|
|
236
|
-
# List every exception class the package exports (
|
|
247
|
+
# List every exception class the package exports (106 of them, alphabetically)
|
|
237
248
|
$ dataexcept list
|
|
238
249
|
ApiError
|
|
239
250
|
AuthenticationError
|
|
@@ -383,14 +394,21 @@ through [SECURITY.md](SECURITY.md), not the public issue tracker.
|
|
|
383
394
|
- **API Stability**: [What is public and what may change](https://diogoribeiro7.github.io/DataExcept/stability/)
|
|
384
395
|
- **Upgrading from 0.x**: [Migration guide](https://diogoribeiro7.github.io/DataExcept/migration/)
|
|
385
396
|
- **Advanced Usage**: [Advanced Guide](https://diogoribeiro7.github.io/DataExcept/advanced_usage/)
|
|
397
|
+
- **Cause-aware Exceptions**: [Wrapping a third-party failure](https://diogoribeiro7.github.io/DataExcept/causes/)
|
|
398
|
+
- **Failure Metadata**: [Transient, permanent, or unclassified](https://diogoribeiro7.github.io/DataExcept/failure_metadata/)
|
|
399
|
+
- **Parsing Context**: [Reporting a failure without keeping the payload](https://diogoribeiro7.github.io/DataExcept/parsing_context/)
|
|
400
|
+
- **Message Brokers**: [Publish, consume and acknowledgement failures](https://diogoribeiro7.github.io/DataExcept/brokers/)
|
|
386
401
|
- **Envelope Schema**: [The published envelope contract](https://diogoribeiro7.github.io/DataExcept/envelope_schema/)
|
|
402
|
+
- **Pino Interoperability**: [Consuming envelopes from Node.js](https://diogoribeiro7.github.io/DataExcept/pino/)
|
|
387
403
|
- **CLI Reference**: [CLI Guide](https://diogoribeiro7.github.io/DataExcept/cli/)
|
|
388
404
|
- **Changelog**: [CHANGELOG.md](CHANGELOG.md)
|
|
389
405
|
|
|
390
406
|
## 🎓 Citation
|
|
391
407
|
|
|
392
408
|
If you use DataExcept in your research, please cite the exact release you used.
|
|
393
|
-
The canonical release metadata is maintained in [CITATION.cff](CITATION.cff)
|
|
409
|
+
The canonical release metadata is maintained in [CITATION.cff](CITATION.cff), and
|
|
410
|
+
the [citation guide](https://diogoribeiro7.github.io/DataExcept/citation/)
|
|
411
|
+
explains how to get the APA and BibTeX forms from it.
|
|
394
412
|
|
|
395
413
|
## 📄 License
|
|
396
414
|
|
|
@@ -17,8 +17,8 @@
|
|
|
17
17
|
|
|
18
18
|
- **🏗️ Hierarchical Structure**: Catch one specific error, a whole domain, or every operational error via `DataExceptError`
|
|
19
19
|
- **📦 One Import**: Every exception is available from `dataexcept` directly, or from its domain module — same objects either way
|
|
20
|
-
- **📊 Data Science Focused**:
|
|
21
|
-
- **🧭 Language-neutral envelopes**: exceptions export to strict JSON against a published, versioned schema — so a non-Python consumer can validate what it receives
|
|
20
|
+
- **📊 Data Science Focused**: 106 exception classes covering ML pipelines, feature engineering, model training
|
|
21
|
+
- **🧭 Language-neutral envelopes**: exceptions export to strict JSON against a published, versioned schema — so a non-Python consumer can validate what it receives, or read it through the [Pino projection](https://diogoribeiro7.github.io/DataExcept/pino/)
|
|
22
22
|
- **🔧 Production Ready**: Logging helpers, error context, and exceptions that pickle — so they cross a process boundary with their message, attributes and cause intact
|
|
23
23
|
- **📚 Academic Quality**: Proper documentation, type hints, and citation support
|
|
24
24
|
- **🐍 Python 3.10 – 3.14**: Every supported version tested in CI, with full type safety
|
|
@@ -159,6 +159,17 @@ DatabaseConnectionError("postgresql://prod-db:5432/analytics")
|
|
|
159
159
|
QueryExecutionError("SELECT * FROM large_table", original=TimeoutError())
|
|
160
160
|
```
|
|
161
161
|
|
|
162
|
+
### 📨 Message Brokers
|
|
163
|
+
|
|
164
|
+
```python
|
|
165
|
+
from dataexcept.broker_exceptions import *
|
|
166
|
+
|
|
167
|
+
BrokerConnectionError("kafka://broker.internal:9092", cause=OSError("refused"))
|
|
168
|
+
MessagePublishError("orders", partition=3, cause=OSError("leader not available"))
|
|
169
|
+
MessageConsumeError("orders", partition=3, offset=1042, consumer_group="billing")
|
|
170
|
+
MessageAcknowledgementError("orders", partition=3, offset=1042)
|
|
171
|
+
```
|
|
172
|
+
|
|
162
173
|
## 🔍 Advanced Features
|
|
163
174
|
|
|
164
175
|
### Smart Logging Integration
|
|
@@ -206,7 +217,7 @@ document Python emits.
|
|
|
206
217
|
### Command Line Interface
|
|
207
218
|
|
|
208
219
|
```bash
|
|
209
|
-
# List every exception class the package exports (
|
|
220
|
+
# List every exception class the package exports (106 of them, alphabetically)
|
|
210
221
|
$ dataexcept list
|
|
211
222
|
ApiError
|
|
212
223
|
AuthenticationError
|
|
@@ -356,14 +367,21 @@ through [SECURITY.md](SECURITY.md), not the public issue tracker.
|
|
|
356
367
|
- **API Stability**: [What is public and what may change](https://diogoribeiro7.github.io/DataExcept/stability/)
|
|
357
368
|
- **Upgrading from 0.x**: [Migration guide](https://diogoribeiro7.github.io/DataExcept/migration/)
|
|
358
369
|
- **Advanced Usage**: [Advanced Guide](https://diogoribeiro7.github.io/DataExcept/advanced_usage/)
|
|
370
|
+
- **Cause-aware Exceptions**: [Wrapping a third-party failure](https://diogoribeiro7.github.io/DataExcept/causes/)
|
|
371
|
+
- **Failure Metadata**: [Transient, permanent, or unclassified](https://diogoribeiro7.github.io/DataExcept/failure_metadata/)
|
|
372
|
+
- **Parsing Context**: [Reporting a failure without keeping the payload](https://diogoribeiro7.github.io/DataExcept/parsing_context/)
|
|
373
|
+
- **Message Brokers**: [Publish, consume and acknowledgement failures](https://diogoribeiro7.github.io/DataExcept/brokers/)
|
|
359
374
|
- **Envelope Schema**: [The published envelope contract](https://diogoribeiro7.github.io/DataExcept/envelope_schema/)
|
|
375
|
+
- **Pino Interoperability**: [Consuming envelopes from Node.js](https://diogoribeiro7.github.io/DataExcept/pino/)
|
|
360
376
|
- **CLI Reference**: [CLI Guide](https://diogoribeiro7.github.io/DataExcept/cli/)
|
|
361
377
|
- **Changelog**: [CHANGELOG.md](CHANGELOG.md)
|
|
362
378
|
|
|
363
379
|
## 🎓 Citation
|
|
364
380
|
|
|
365
381
|
If you use DataExcept in your research, please cite the exact release you used.
|
|
366
|
-
The canonical release metadata is maintained in [CITATION.cff](CITATION.cff)
|
|
382
|
+
The canonical release metadata is maintained in [CITATION.cff](CITATION.cff), and
|
|
383
|
+
the [citation guide](https://diogoribeiro7.github.io/DataExcept/citation/)
|
|
384
|
+
explains how to get the APA and BibTeX forms from it.
|
|
367
385
|
|
|
368
386
|
## 📄 License
|
|
369
387
|
|
|
@@ -25,6 +25,7 @@ except ModuleNotFoundError: # pragma: no cover - fallback for Python <3.11
|
|
|
25
25
|
from importlib import metadata
|
|
26
26
|
|
|
27
27
|
from . import ( # noqa: F401
|
|
28
|
+
broker_exceptions,
|
|
28
29
|
database_exceptions,
|
|
29
30
|
dataengineering_exceptions,
|
|
30
31
|
datascience_exceptions,
|
|
@@ -37,6 +38,14 @@ from . import ( # noqa: F401
|
|
|
37
38
|
security_exceptions,
|
|
38
39
|
)
|
|
39
40
|
from .base import DataExceptError, UnpicklableCause, UnpicklableValue
|
|
41
|
+
from .broker_exceptions import (
|
|
42
|
+
BrokerConnectionError,
|
|
43
|
+
BrokerTimeoutError,
|
|
44
|
+
MessageAcknowledgementError,
|
|
45
|
+
MessageBrokerError,
|
|
46
|
+
MessageConsumeError,
|
|
47
|
+
MessagePublishError,
|
|
48
|
+
)
|
|
40
49
|
from .database_exceptions import (
|
|
41
50
|
DatabaseConnectionError,
|
|
42
51
|
DatabaseError,
|
|
@@ -112,24 +121,15 @@ from .exceptions import (
|
|
|
112
121
|
WebhookError,
|
|
113
122
|
)
|
|
114
123
|
from .failure_metadata import FailureKind, FailureMetadata
|
|
115
|
-
from .io_exceptions import
|
|
116
|
-
|
|
117
|
-
FileLockError,
|
|
118
|
-
FileReadError,
|
|
119
|
-
FileWriteError,
|
|
120
|
-
)
|
|
121
|
-
from .logging_helpers import (
|
|
122
|
-
Context,
|
|
123
|
-
log_and_raise,
|
|
124
|
-
log_exception,
|
|
125
|
-
log_then_raise,
|
|
126
|
-
)
|
|
124
|
+
from .io_exceptions import CustomIOError, FileLockError, FileReadError, FileWriteError
|
|
125
|
+
from .logging_helpers import Context, log_and_raise, log_exception, log_then_raise
|
|
127
126
|
from .network_exceptions import (
|
|
128
127
|
ConnectionTimeoutError,
|
|
129
128
|
HostUnreachableError,
|
|
130
129
|
NetworkError,
|
|
131
130
|
ProtocolError,
|
|
132
131
|
)
|
|
132
|
+
from .observability import OperationContext, exception_to_observability_event
|
|
133
133
|
from .pandas_exceptions import (
|
|
134
134
|
DtypeMismatchError,
|
|
135
135
|
IndexAlignmentError,
|
|
@@ -138,6 +138,7 @@ from .pandas_exceptions import (
|
|
|
138
138
|
PandasError,
|
|
139
139
|
PandasIOError,
|
|
140
140
|
)
|
|
141
|
+
from .pino import envelope_to_pino, exception_to_pino
|
|
141
142
|
from .pipeline_exceptions import (
|
|
142
143
|
ApiError,
|
|
143
144
|
DataFetchError,
|
|
@@ -157,7 +158,10 @@ from .pipeline_exceptions import (
|
|
|
157
158
|
from .schema import (
|
|
158
159
|
ENVELOPE_SCHEMA_ID,
|
|
159
160
|
ENVELOPE_SCHEMA_VERSION,
|
|
161
|
+
PINO_PROFILE_ID,
|
|
162
|
+
PINO_PROFILE_VERSION,
|
|
160
163
|
envelope_schema,
|
|
164
|
+
pino_profile_schema,
|
|
161
165
|
)
|
|
162
166
|
from .security_exceptions import (
|
|
163
167
|
DecryptionError,
|
|
@@ -165,6 +169,7 @@ from .security_exceptions import (
|
|
|
165
169
|
InvalidTokenError,
|
|
166
170
|
SecurityError,
|
|
167
171
|
)
|
|
172
|
+
from .sentry import enrich_sentry_event
|
|
168
173
|
from .serialization import exception_to_dict, exception_to_json
|
|
169
174
|
from .wrapping import wrap, wrapping
|
|
170
175
|
|
|
@@ -179,6 +184,8 @@ __all__ = [
|
|
|
179
184
|
"AuthorizationError",
|
|
180
185
|
"BatchProcessingError",
|
|
181
186
|
"BiasDetectionError",
|
|
187
|
+
"BrokerConnectionError",
|
|
188
|
+
"BrokerTimeoutError",
|
|
182
189
|
"ConfigurationError",
|
|
183
190
|
"ConnectionTimeoutError",
|
|
184
191
|
"ConvergenceError",
|
|
@@ -230,6 +237,10 @@ __all__ = [
|
|
|
230
237
|
"JobCancellationError",
|
|
231
238
|
"JobError",
|
|
232
239
|
"MergeKeyError",
|
|
240
|
+
"MessageAcknowledgementError",
|
|
241
|
+
"MessageBrokerError",
|
|
242
|
+
"MessageConsumeError",
|
|
243
|
+
"MessagePublishError",
|
|
233
244
|
"MissingColumnError",
|
|
234
245
|
"MissingDataError",
|
|
235
246
|
"MissingPartitionError",
|
|
@@ -276,13 +287,22 @@ __all__ = [
|
|
|
276
287
|
"wrapping",
|
|
277
288
|
"exception_to_dict",
|
|
278
289
|
"exception_to_json",
|
|
290
|
+
"exception_to_observability_event",
|
|
291
|
+
"OperationContext",
|
|
279
292
|
"ENVELOPE_SCHEMA_ID",
|
|
280
293
|
"ENVELOPE_SCHEMA_VERSION",
|
|
281
294
|
"envelope_schema",
|
|
295
|
+
"PINO_PROFILE_ID",
|
|
296
|
+
"PINO_PROFILE_VERSION",
|
|
297
|
+
"pino_profile_schema",
|
|
298
|
+
"envelope_to_pino",
|
|
299
|
+
"exception_to_pino",
|
|
300
|
+
"enrich_sentry_event",
|
|
282
301
|
"Context",
|
|
283
302
|
"log_and_raise",
|
|
284
303
|
"log_exception",
|
|
285
304
|
"log_then_raise",
|
|
305
|
+
"broker_exceptions",
|
|
286
306
|
"database_exceptions",
|
|
287
307
|
"dataengineering_exceptions",
|
|
288
308
|
"datascience_exceptions",
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""Custom exceptions for message-broker operations.
|
|
2
|
+
|
|
3
|
+
A broker is a data-engineering boundary like a database or an object store, and
|
|
4
|
+
it fails in ways that are specific to it: a publish is rejected, a consumer
|
|
5
|
+
cannot reach its group, an offset is never committed. Mapping those onto
|
|
6
|
+
``ServiceConnectionError`` and ``OperationTimeoutError`` loses which of them
|
|
7
|
+
happened, and with it the topic, partition, offset and consumer group that say
|
|
8
|
+
where.
|
|
9
|
+
|
|
10
|
+
The hierarchy is deliberately about the *operation* rather than the product.
|
|
11
|
+
Kafka, RabbitMQ, Pulsar and NATS disagree about almost everything else, but all
|
|
12
|
+
four connect, publish, consume and acknowledge, so a pipeline can catch these
|
|
13
|
+
without knowing which broker or client library is underneath -- and DataExcept
|
|
14
|
+
depends on none of them.
|
|
15
|
+
|
|
16
|
+
Failure metadata stays at the conservative default. A broker refusing a publish
|
|
17
|
+
may be a leader election that resolves in a second or a topic that does not
|
|
18
|
+
exist, and the exception cannot tell which. An integration that *does* know --
|
|
19
|
+
because it read the broker's own error code -- attaches that with
|
|
20
|
+
``with_failure_metadata`` or ``wrap(..., failure_metadata=...)``.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from typing import Optional
|
|
26
|
+
|
|
27
|
+
from ._causes import resolve_cause
|
|
28
|
+
from .base import DataExceptError
|
|
29
|
+
from .redaction import redact_if_url
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _describe_position(
|
|
33
|
+
topic: str,
|
|
34
|
+
partition: Optional[int] = None,
|
|
35
|
+
offset: Optional[int] = None,
|
|
36
|
+
consumer_group: Optional[str] = None,
|
|
37
|
+
) -> str:
|
|
38
|
+
"""Render as much of a message's coordinates as the caller supplied."""
|
|
39
|
+
parts = [f"topic '{topic}'"]
|
|
40
|
+
if partition is not None:
|
|
41
|
+
parts.append(f"partition {partition}")
|
|
42
|
+
if offset is not None:
|
|
43
|
+
parts.append(f"offset {offset}")
|
|
44
|
+
if consumer_group is not None:
|
|
45
|
+
parts.append(f"group '{consumer_group}'")
|
|
46
|
+
return ", ".join(parts)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _on_broker(broker: Optional[str]) -> str:
|
|
50
|
+
return f" on broker '{broker}'" if broker else ""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class MessageBrokerError(DataExceptError):
|
|
54
|
+
"""Base exception for message-broker failures.
|
|
55
|
+
|
|
56
|
+
Catching this catches every broker failure the library raises, without
|
|
57
|
+
catching a database or HTTP one.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
pass
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class BrokerConnectionError(MessageBrokerError):
|
|
64
|
+
"""Raised when a connection to the broker cannot be established."""
|
|
65
|
+
|
|
66
|
+
def __init__(
|
|
67
|
+
self,
|
|
68
|
+
broker: str,
|
|
69
|
+
message: Optional[str] = None,
|
|
70
|
+
*,
|
|
71
|
+
cause: Optional[Exception] = None,
|
|
72
|
+
) -> None:
|
|
73
|
+
"""Initialize BrokerConnectionError.
|
|
74
|
+
|
|
75
|
+
Args:
|
|
76
|
+
broker: The broker being connected to. A bootstrap address, a
|
|
77
|
+
cluster name, or a URL -- redacted when it is a URL, since one
|
|
78
|
+
routinely carries credentials.
|
|
79
|
+
message: Optional custom error message.
|
|
80
|
+
cause: Optional underlying exception, also set as ``__cause__``.
|
|
81
|
+
"""
|
|
82
|
+
self.broker = redact_if_url(broker)
|
|
83
|
+
self.cause = resolve_cause(cause=cause)
|
|
84
|
+
default = f"Failed to connect to message broker '{self.broker}'"
|
|
85
|
+
if self.cause:
|
|
86
|
+
default += f": {self.cause}"
|
|
87
|
+
super().__init__(message or default)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class BrokerTimeoutError(MessageBrokerError):
|
|
91
|
+
"""Raised when a broker operation exceeds its time limit."""
|
|
92
|
+
|
|
93
|
+
def __init__(
|
|
94
|
+
self,
|
|
95
|
+
broker: str,
|
|
96
|
+
operation: Optional[str] = None,
|
|
97
|
+
timeout_seconds: Optional[float] = None,
|
|
98
|
+
message: Optional[str] = None,
|
|
99
|
+
*,
|
|
100
|
+
cause: Optional[Exception] = None,
|
|
101
|
+
) -> None:
|
|
102
|
+
"""Initialize BrokerTimeoutError.
|
|
103
|
+
|
|
104
|
+
Args:
|
|
105
|
+
broker: The broker the operation was waiting on.
|
|
106
|
+
operation: What was being attempted, such as ``"publish"``.
|
|
107
|
+
timeout_seconds: The limit that was exceeded.
|
|
108
|
+
message: Optional custom error message.
|
|
109
|
+
cause: Optional underlying exception, also set as ``__cause__``.
|
|
110
|
+
"""
|
|
111
|
+
self.broker = redact_if_url(broker)
|
|
112
|
+
self.operation = operation
|
|
113
|
+
self.timeout_seconds = timeout_seconds
|
|
114
|
+
self.cause = resolve_cause(cause=cause)
|
|
115
|
+
attempted = (
|
|
116
|
+
f"Broker operation '{operation}'" if operation else "Broker operation"
|
|
117
|
+
)
|
|
118
|
+
default = f"{attempted} timed out"
|
|
119
|
+
if timeout_seconds is not None:
|
|
120
|
+
default += f" after {timeout_seconds}s"
|
|
121
|
+
default += _on_broker(self.broker)
|
|
122
|
+
if self.cause:
|
|
123
|
+
default += f": {self.cause}"
|
|
124
|
+
super().__init__(message or default)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class MessagePublishError(MessageBrokerError):
|
|
128
|
+
"""Raised when publishing a message fails."""
|
|
129
|
+
|
|
130
|
+
def __init__(
|
|
131
|
+
self,
|
|
132
|
+
topic: str,
|
|
133
|
+
broker: Optional[str] = None,
|
|
134
|
+
partition: Optional[int] = None,
|
|
135
|
+
message: Optional[str] = None,
|
|
136
|
+
*,
|
|
137
|
+
cause: Optional[Exception] = None,
|
|
138
|
+
) -> None:
|
|
139
|
+
"""Initialize MessagePublishError.
|
|
140
|
+
|
|
141
|
+
Args:
|
|
142
|
+
topic: The topic, queue or subject published to.
|
|
143
|
+
broker: Optional broker the publish was addressed to.
|
|
144
|
+
partition: Optional partition the message was keyed to.
|
|
145
|
+
message: Optional custom error message.
|
|
146
|
+
cause: Optional underlying exception, also set as ``__cause__``.
|
|
147
|
+
"""
|
|
148
|
+
self.topic = topic
|
|
149
|
+
self.broker = redact_if_url(broker)
|
|
150
|
+
self.partition = partition
|
|
151
|
+
self.cause = resolve_cause(cause=cause)
|
|
152
|
+
position = _describe_position(topic, partition)
|
|
153
|
+
default = f"Failed to publish to {position}{_on_broker(self.broker)}"
|
|
154
|
+
if self.cause:
|
|
155
|
+
default += f": {self.cause}"
|
|
156
|
+
super().__init__(message or default)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class MessageConsumeError(MessageBrokerError):
|
|
160
|
+
"""Raised when consuming a message fails."""
|
|
161
|
+
|
|
162
|
+
def __init__(
|
|
163
|
+
self,
|
|
164
|
+
topic: str,
|
|
165
|
+
broker: Optional[str] = None,
|
|
166
|
+
partition: Optional[int] = None,
|
|
167
|
+
offset: Optional[int] = None,
|
|
168
|
+
consumer_group: Optional[str] = None,
|
|
169
|
+
message: Optional[str] = None,
|
|
170
|
+
*,
|
|
171
|
+
cause: Optional[Exception] = None,
|
|
172
|
+
) -> None:
|
|
173
|
+
"""Initialize MessageConsumeError.
|
|
174
|
+
|
|
175
|
+
Args:
|
|
176
|
+
topic: The topic, queue or subject consumed from.
|
|
177
|
+
broker: Optional broker the consumer was reading from.
|
|
178
|
+
partition: Optional partition being read.
|
|
179
|
+
offset: Optional offset the failure happened at.
|
|
180
|
+
consumer_group: Optional consumer group the reader belongs to.
|
|
181
|
+
message: Optional custom error message.
|
|
182
|
+
cause: Optional underlying exception, also set as ``__cause__``.
|
|
183
|
+
"""
|
|
184
|
+
self.topic = topic
|
|
185
|
+
self.broker = redact_if_url(broker)
|
|
186
|
+
self.partition = partition
|
|
187
|
+
self.offset = offset
|
|
188
|
+
self.consumer_group = consumer_group
|
|
189
|
+
self.cause = resolve_cause(cause=cause)
|
|
190
|
+
position = _describe_position(topic, partition, offset, consumer_group)
|
|
191
|
+
default = f"Failed to consume from {position}{_on_broker(self.broker)}"
|
|
192
|
+
if self.cause:
|
|
193
|
+
default += f": {self.cause}"
|
|
194
|
+
super().__init__(message or default)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
class MessageAcknowledgementError(MessageBrokerError):
|
|
198
|
+
"""Raised when acknowledging or committing a message fails.
|
|
199
|
+
|
|
200
|
+
Distinct from a consume failure on purpose: the message was read and
|
|
201
|
+
processed, and it is the record of that which did not stick -- so it will
|
|
202
|
+
be delivered again.
|
|
203
|
+
"""
|
|
204
|
+
|
|
205
|
+
def __init__(
|
|
206
|
+
self,
|
|
207
|
+
topic: str,
|
|
208
|
+
broker: Optional[str] = None,
|
|
209
|
+
partition: Optional[int] = None,
|
|
210
|
+
offset: Optional[int] = None,
|
|
211
|
+
consumer_group: Optional[str] = None,
|
|
212
|
+
message: Optional[str] = None,
|
|
213
|
+
*,
|
|
214
|
+
cause: Optional[Exception] = None,
|
|
215
|
+
) -> None:
|
|
216
|
+
"""Initialize MessageAcknowledgementError.
|
|
217
|
+
|
|
218
|
+
Args:
|
|
219
|
+
topic: The topic, queue or subject the message came from.
|
|
220
|
+
broker: Optional broker the acknowledgement was sent to.
|
|
221
|
+
partition: Optional partition the message was read from.
|
|
222
|
+
offset: Optional offset that failed to commit.
|
|
223
|
+
consumer_group: Optional consumer group committing the offset.
|
|
224
|
+
message: Optional custom error message.
|
|
225
|
+
cause: Optional underlying exception, also set as ``__cause__``.
|
|
226
|
+
"""
|
|
227
|
+
self.topic = topic
|
|
228
|
+
self.broker = redact_if_url(broker)
|
|
229
|
+
self.partition = partition
|
|
230
|
+
self.offset = offset
|
|
231
|
+
self.consumer_group = consumer_group
|
|
232
|
+
self.cause = resolve_cause(cause=cause)
|
|
233
|
+
position = _describe_position(topic, partition, offset, consumer_group)
|
|
234
|
+
default = f"Failed to acknowledge {position}{_on_broker(self.broker)}"
|
|
235
|
+
if self.cause:
|
|
236
|
+
default += f": {self.cause}"
|
|
237
|
+
super().__init__(message or default)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
__all__ = [
|
|
241
|
+
"MessageBrokerError",
|
|
242
|
+
"BrokerConnectionError",
|
|
243
|
+
"BrokerTimeoutError",
|
|
244
|
+
"MessagePublishError",
|
|
245
|
+
"MessageConsumeError",
|
|
246
|
+
"MessageAcknowledgementError",
|
|
247
|
+
]
|