ergon-framework-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ergon/__init__.py +13 -0
- ergon/bootstrap/src/__project__/__init__.py +0 -0
- ergon/bootstrap/src/__project__/_observability/docker-compose.telemetry.yml +124 -0
- ergon/bootstrap/src/__project__/_observability/grafana.yaml +17 -0
- ergon/bootstrap/src/__project__/_observability/loki.yaml +48 -0
- ergon/bootstrap/src/__project__/_observability/otel-collector-config.yaml +53 -0
- ergon/bootstrap/src/__project__/_observability/prometheus.yaml +11 -0
- ergon/bootstrap/src/__project__/_observability/tempo.yaml +24 -0
- ergon/bootstrap/src/__project__/connectors/__init__.py +0 -0
- ergon/bootstrap/src/__project__/main.py +9 -0
- ergon/bootstrap/src/__project__/tasks/__init__.py +0 -0
- ergon/bootstrap/src/__project__/tasks/constants.py +13 -0
- ergon/bootstrap/src/__project__/tasks/example_task/__init__.py +0 -0
- ergon/bootstrap/src/__project__/tasks/example_task/config.py +4 -0
- ergon/bootstrap/src/__project__/tasks/example_task/exceptions.py +4 -0
- ergon/bootstrap/src/__project__/tasks/example_task/helpers.py +4 -0
- ergon/bootstrap/src/__project__/tasks/example_task/schemas.py +5 -0
- ergon/bootstrap/src/__project__/tasks/example_task/task.py +1 -0
- ergon/bootstrap/src/__project__/tasks/exceptions.py +0 -0
- ergon/bootstrap/src/__project__/tasks/helpers.py +0 -0
- ergon/bootstrap/src/__project__/tasks/schemas.py +0 -0
- ergon/bootstrap/src/__project__/tasks/settings.py +5 -0
- ergon/cli.py +174 -0
- ergon/connector/__init__.py +64 -0
- ergon/connector/connector.py +97 -0
- ergon/connector/excel/__init__.py +18 -0
- ergon/connector/excel/connector.py +175 -0
- ergon/connector/excel/models.py +24 -0
- ergon/connector/excel/service.py +98 -0
- ergon/connector/pipefy/__init__.py +21 -0
- ergon/connector/pipefy/async_connector.py +48 -0
- ergon/connector/pipefy/async_service.py +907 -0
- ergon/connector/pipefy/connector.py +36 -0
- ergon/connector/pipefy/models.py +48 -0
- ergon/connector/pipefy/service.py +1016 -0
- ergon/connector/pipefy/version.py +1 -0
- ergon/connector/postgres/__init__.py +11 -0
- ergon/connector/postgres/async_connector.py +119 -0
- ergon/connector/postgres/async_service.py +116 -0
- ergon/connector/postgres/models.py +34 -0
- ergon/connector/rabbitmq/__init__.py +25 -0
- ergon/connector/rabbitmq/async_connector.py +120 -0
- ergon/connector/rabbitmq/async_service.py +417 -0
- ergon/connector/rabbitmq/connector.py +54 -0
- ergon/connector/rabbitmq/helper.py +14 -0
- ergon/connector/rabbitmq/models.py +92 -0
- ergon/connector/rabbitmq/service.py +199 -0
- ergon/connector/sqs/__init__.py +15 -0
- ergon/connector/sqs/async_connector.py +120 -0
- ergon/connector/sqs/async_service.py +246 -0
- ergon/connector/sqs/connector.py +120 -0
- ergon/connector/sqs/models.py +36 -0
- ergon/connector/sqs/service.py +219 -0
- ergon/connector/transaction.py +14 -0
- ergon/py.typed +0 -0
- ergon/service/__init__.py +5 -0
- ergon/service/service.py +17 -0
- ergon/task/__init__.py +13 -0
- ergon/task/base.py +222 -0
- ergon/task/exceptions.py +217 -0
- ergon/task/helpers.py +691 -0
- ergon/task/manager.py +85 -0
- ergon/task/mixins/__init__.py +13 -0
- ergon/task/mixins/consumer.py +858 -0
- ergon/task/mixins/metrics.py +457 -0
- ergon/task/mixins/producer.py +486 -0
- ergon/task/policies.py +229 -0
- ergon/task/runner.py +386 -0
- ergon/task/utils.py +64 -0
- ergon/telemetry/__init__.py +7 -0
- ergon/telemetry/_resource.py +13 -0
- ergon/telemetry/logging.py +370 -0
- ergon/telemetry/metrics.py +101 -0
- ergon/telemetry/tracing.py +152 -0
- ergon/utils/__init__.py +5 -0
- ergon/utils/env.py +26 -0
- ergon_framework_python-0.1.0.dist-info/METADATA +449 -0
- ergon_framework_python-0.1.0.dist-info/RECORD +82 -0
- ergon_framework_python-0.1.0.dist-info/WHEEL +5 -0
- ergon_framework_python-0.1.0.dist-info/entry_points.txt +2 -0
- ergon_framework_python-0.1.0.dist-info/licenses/LICENSE +21 -0
- ergon_framework_python-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,449 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ergon-framework-python
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Ergon internal task-oriented project bootstrapper
|
|
5
|
+
Author-email: Ergondata Technologies <anza.vossos@protonmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/danzanellovossos/ergon-framework
|
|
8
|
+
Project-URL: Repository, https://github.com/danzanellovossos/ergon-framework
|
|
9
|
+
Project-URL: Issues, https://github.com/danzanellovossos/ergon-framework/issues
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: aio-pika>=9.0.0
|
|
20
|
+
Requires-Dist: aiobotocore>=2.13.0
|
|
21
|
+
Requires-Dist: asyncpg>=0.29.0
|
|
22
|
+
Requires-Dist: boto3>=1.35.0
|
|
23
|
+
Requires-Dist: httpx>=0.28.0
|
|
24
|
+
Requires-Dist: more-itertools>=10.8.0
|
|
25
|
+
Requires-Dist: openpyxl>=3.1.5
|
|
26
|
+
Requires-Dist: opentelemetry-api>=1.38.0
|
|
27
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.38.0
|
|
28
|
+
Requires-Dist: opentelemetry-sdk>=1.38.0
|
|
29
|
+
Requires-Dist: pika>=1.3.0
|
|
30
|
+
Requires-Dist: pydantic>=2.12.4
|
|
31
|
+
Requires-Dist: python-dotenv>=1.2.1
|
|
32
|
+
Requires-Dist: python-json-logger>=4.0.0
|
|
33
|
+
Requires-Dist: requests>=2.32.5
|
|
34
|
+
Dynamic: license-file
|
|
35
|
+
|
|
36
|
+
# Ergon Framework — Python SDK
|
|
37
|
+
|
|
38
|
+
[](https://www.python.org/downloads/)
|
|
39
|
+
[](https://opensource.org/licenses/MIT)
|
|
40
|
+
[](https://github.com/astral-sh/ruff)
|
|
41
|
+
|
|
42
|
+
**The official Python implementation of the Ergon Framework—a transaction-first, observability-native execution engine for building high-throughput automation pipelines.**
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## Installation
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install ergon-framework-python
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
For local development, install in editable mode:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
pip install -e /path/to/ergon-framework/sdks/python
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
📖 **[Full Getting Started Guide](docs/getting-started.md)** — Complete setup, project configuration, and task registration
|
|
59
|
+
|
|
60
|
+
---
|
|
61
|
+
|
|
62
|
+
## Quick Start
|
|
63
|
+
|
|
64
|
+
### 1. Define a Consumer Task
|
|
65
|
+
|
|
66
|
+
Create a task by inheriting from `ConsumerTask` and implementing `process_transaction`:
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from typing import Any
|
|
70
|
+
from ergon.task.mixins import ConsumerTask
|
|
71
|
+
from ergon.connector import Transaction
|
|
72
|
+
|
|
73
|
+
class OrderProcessor(ConsumerTask):
|
|
74
|
+
"""Consumes order transactions and processes them."""
|
|
75
|
+
|
|
76
|
+
def process_transaction(self, transaction: Transaction) -> Any:
|
|
77
|
+
# Pure business logic—no I/O, no retries, no protocol details
|
|
78
|
+
order = transaction.payload
|
|
79
|
+
total = order["quantity"] * order["unit_price"]
|
|
80
|
+
return {"order_id": order["id"], "total": total, "status": "processed"}
|
|
81
|
+
|
|
82
|
+
def handle_process_success(self, transaction: Transaction, result: Any):
|
|
83
|
+
# Called after successful processing
|
|
84
|
+
self.output_connector.dispatch_transactions([
|
|
85
|
+
Transaction(id=f"out-{transaction.id}", payload=result, metadata={})
|
|
86
|
+
])
|
|
87
|
+
|
|
88
|
+
def handle_process_exception(self, transaction: Transaction, exc: Exception):
|
|
89
|
+
# Called when processing fails (after all retries exhausted)
|
|
90
|
+
self.dlq_connector.dispatch_transactions([transaction])
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
### 2. Create a Connector
|
|
94
|
+
|
|
95
|
+
Connectors bridge external systems to the transaction interface:
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from typing import List
|
|
99
|
+
from ergon.connector import Connector, Transaction
|
|
100
|
+
import uuid
|
|
101
|
+
|
|
102
|
+
class RabbitMQConnector(Connector):
|
|
103
|
+
def __init__(self, queue_name: str, connection_url: str):
|
|
104
|
+
self.queue_name = queue_name
|
|
105
|
+
# Initialize your RabbitMQ client here
|
|
106
|
+
self.client = create_rabbitmq_client(connection_url)
|
|
107
|
+
|
|
108
|
+
def fetch_transactions(self, batch_size: int, **kwargs) -> List[Transaction]:
|
|
109
|
+
messages = self.client.consume(self.queue_name, max_messages=batch_size)
|
|
110
|
+
return [
|
|
111
|
+
Transaction(
|
|
112
|
+
id=str(uuid.uuid4()),
|
|
113
|
+
payload=msg.body,
|
|
114
|
+
metadata={"delivery_tag": msg.delivery_tag, "routing_key": msg.routing_key}
|
|
115
|
+
)
|
|
116
|
+
for msg in messages
|
|
117
|
+
]
|
|
118
|
+
|
|
119
|
+
def dispatch_transactions(self, transactions: List[Transaction], **kwargs):
|
|
120
|
+
for tx in transactions:
|
|
121
|
+
self.client.publish(self.queue_name, tx.payload)
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
### 3. Configure and Run
|
|
125
|
+
|
|
126
|
+
Use `TaskConfig` to wire everything together:
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
from ergon.task import TaskConfig, runner, policies
|
|
130
|
+
from ergon.connector import ConnectorConfig
|
|
131
|
+
|
|
132
|
+
# Configure the consumer policy
|
|
133
|
+
consumer_policy = policies.ConsumerPolicy()
|
|
134
|
+
consumer_policy.name = "consumer"
|
|
135
|
+
consumer_policy.loop.concurrency.value = 5 # Process 5 transactions in parallel
|
|
136
|
+
consumer_policy.loop.batch.size = 10 # Fetch 10 at a time
|
|
137
|
+
consumer_policy.loop.streaming = True # Keep polling for new messages
|
|
138
|
+
consumer_policy.process.retry.max_attempts = 3 # Retry failed processing 3 times
|
|
139
|
+
consumer_policy.process.retry.backoff = 1.0 # Start with 1s backoff
|
|
140
|
+
consumer_policy.process.retry.backoff_multiplier = 2.0 # Double each retry
|
|
141
|
+
|
|
142
|
+
# Create the task configuration
|
|
143
|
+
config = TaskConfig(
|
|
144
|
+
name="order-processor",
|
|
145
|
+
task=OrderProcessor,
|
|
146
|
+
max_workers=1,
|
|
147
|
+
connectors={
|
|
148
|
+
"input": ConnectorConfig(
|
|
149
|
+
connector=RabbitMQConnector,
|
|
150
|
+
kwargs={"queue_name": "orders", "connection_url": "amqp://localhost"}
|
|
151
|
+
),
|
|
152
|
+
"output": ConnectorConfig(
|
|
153
|
+
connector=RabbitMQConnector,
|
|
154
|
+
kwargs={"queue_name": "processed-orders", "connection_url": "amqp://localhost"}
|
|
155
|
+
),
|
|
156
|
+
},
|
|
157
|
+
policies=[consumer_policy],
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
# Run the task
|
|
161
|
+
if __name__ == "__main__":
|
|
162
|
+
runner.run(config)
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
The framework handles the loop, retry logic, telemetry, and graceful shutdown automatically.
|
|
166
|
+
|
|
167
|
+
---
|
|
168
|
+
|
|
169
|
+
## The Transaction Object
|
|
170
|
+
|
|
171
|
+
In the Python SDK, transactions are implemented as immutable Pydantic models:
|
|
172
|
+
|
|
173
|
+
```python
|
|
174
|
+
from ergon.connector import Transaction
|
|
175
|
+
|
|
176
|
+
# Transactions are created by Connectors when fetching data
|
|
177
|
+
tx = Transaction(
|
|
178
|
+
id="tx-001", # Unique identifier for tracing
|
|
179
|
+
payload={"order_id": 123, "amount": 99.99}, # The actual data
|
|
180
|
+
metadata={"source": "rabbitmq", "queue": "orders"} # Contextual info
|
|
181
|
+
)
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
Transactions are **frozen** (immutable) once created via Pydantic's `frozen=True` configuration.
|
|
185
|
+
|
|
186
|
+
---
|
|
187
|
+
|
|
188
|
+
## Task Classes
|
|
189
|
+
|
|
190
|
+
The Python SDK provides ready-to-use task classes:
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
from ergon.task.mixins import (
|
|
194
|
+
ConsumerTask, # Sync consumer
|
|
195
|
+
ProducerTask, # Sync producer
|
|
196
|
+
HybridTask, # Sync consumer + producer
|
|
197
|
+
AsyncConsumerTask, # Async consumer
|
|
198
|
+
AsyncProducerTask, # Async producer
|
|
199
|
+
AsyncHybridTask, # Async consumer + producer
|
|
200
|
+
)
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
### Synchronous Tasks
|
|
204
|
+
|
|
205
|
+
Best for CPU-bound processing or legacy libraries:
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
from ergon.task.mixins import ConsumerTask
|
|
209
|
+
|
|
210
|
+
class SyncProcessor(ConsumerTask):
|
|
211
|
+
def process_transaction(self, transaction):
|
|
212
|
+
# Runs in ThreadPoolExecutor
|
|
213
|
+
return heavy_computation(transaction.payload)
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
- Concurrency controlled by `policy.loop.concurrency.value`
|
|
217
|
+
- Each transaction processed in its own thread
|
|
218
|
+
- Multi-process scaling via `max_workers > 1`
|
|
219
|
+
|
|
220
|
+
### Asynchronous Tasks
|
|
221
|
+
|
|
222
|
+
Best for I/O-bound workloads with high concurrency:
|
|
223
|
+
|
|
224
|
+
```python
|
|
225
|
+
from ergon.task.mixins import AsyncConsumerTask
|
|
226
|
+
|
|
227
|
+
class AsyncProcessor(AsyncConsumerTask):
|
|
228
|
+
async def process_transaction(self, transaction):
|
|
229
|
+
# Runs in asyncio event loop
|
|
230
|
+
result = await self.http_client.post(transaction.payload)
|
|
231
|
+
return result
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
- Concurrency controlled by `asyncio.Semaphore`
|
|
235
|
+
- Thousands of concurrent operations with minimal overhead
|
|
236
|
+
- Perfect for API calls, database queries, network I/O
|
|
237
|
+
|
|
238
|
+
---
|
|
239
|
+
|
|
240
|
+
## Dependency Injection
|
|
241
|
+
|
|
242
|
+
Tasks receive all dependencies automatically via the `TaskMeta` metaclass:
|
|
243
|
+
|
|
244
|
+
```python
|
|
245
|
+
class EnrichmentTask(ConsumerTask):
|
|
246
|
+
def process_transaction(self, transaction: Transaction) -> Any:
|
|
247
|
+
# Access connectors as self.{name}_connector
|
|
248
|
+
count = self.input_connector.get_transactions_count()
|
|
249
|
+
|
|
250
|
+
# Access services as self.{name}_service
|
|
251
|
+
enriched = self.openai_service.complete(transaction.payload["text"])
|
|
252
|
+
|
|
253
|
+
# Access policies as self.{name}_policy
|
|
254
|
+
timeout = self.consumer_policy.loop.timeout
|
|
255
|
+
|
|
256
|
+
return {"enriched": enriched, "pending": count}
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
Configure services in `TaskConfig`:
|
|
260
|
+
|
|
261
|
+
```python
|
|
262
|
+
from ergon.connector import ServiceConfig
|
|
263
|
+
|
|
264
|
+
config = TaskConfig(
|
|
265
|
+
name="enrichment-task",
|
|
266
|
+
task=EnrichmentTask,
|
|
267
|
+
connectors={"input": input_connector_config},
|
|
268
|
+
services={
|
|
269
|
+
"openai": ServiceConfig(
|
|
270
|
+
service=OpenAIService,
|
|
271
|
+
kwargs={"api_key": "sk-...", "model": "gpt-4"}
|
|
272
|
+
)
|
|
273
|
+
},
|
|
274
|
+
policies=[consumer_policy],
|
|
275
|
+
)
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
---
|
|
279
|
+
|
|
280
|
+
## Policy Configuration
|
|
281
|
+
|
|
282
|
+
Policies are created as instances and configured by setting properties directly:
|
|
283
|
+
|
|
284
|
+
```python
|
|
285
|
+
from ergon.task import policies
|
|
286
|
+
|
|
287
|
+
# Consumer policy with full configuration
|
|
288
|
+
policy = policies.ConsumerPolicy()
|
|
289
|
+
policy.name = "my-consumer"
|
|
290
|
+
|
|
291
|
+
# Loop behavior
|
|
292
|
+
policy.loop.concurrency.value = 10 # Parallel transaction processing
|
|
293
|
+
policy.loop.batch.size = 50 # Transactions per fetch
|
|
294
|
+
policy.loop.streaming = True # Continuous polling mode
|
|
295
|
+
policy.loop.timeout = 3600 # Max loop duration (seconds)
|
|
296
|
+
policy.loop.limit = 1000 # Max transactions to process
|
|
297
|
+
|
|
298
|
+
# Empty queue behavior (for streaming mode)
|
|
299
|
+
policy.loop.empty_queue.backoff = 1.0
|
|
300
|
+
policy.loop.empty_queue.backoff_multiplier = 2.0
|
|
301
|
+
policy.loop.empty_queue.backoff_cap = 30.0
|
|
302
|
+
|
|
303
|
+
# Per-step retry configuration
|
|
304
|
+
policy.fetch.retry.max_attempts = 5
|
|
305
|
+
policy.fetch.retry.timeout = 30
|
|
306
|
+
|
|
307
|
+
policy.process.retry.max_attempts = 3
|
|
308
|
+
policy.process.retry.backoff = 1.0
|
|
309
|
+
policy.process.retry.backoff_multiplier = 2.0
|
|
310
|
+
policy.process.retry.backoff_cap = 60.0
|
|
311
|
+
|
|
312
|
+
policy.success.retry.max_attempts = 2
|
|
313
|
+
policy.exception.retry.max_attempts = 2
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
---
|
|
317
|
+
|
|
318
|
+
## Observability
|
|
319
|
+
|
|
320
|
+
The Python SDK integrates **OpenTelemetry** natively:
|
|
321
|
+
|
|
322
|
+
```python
|
|
323
|
+
from ergon.telemetry import logging, tracing, metrics
|
|
324
|
+
|
|
325
|
+
config = TaskConfig(
|
|
326
|
+
name="observed-task",
|
|
327
|
+
task=MyTask,
|
|
328
|
+
connectors={...},
|
|
329
|
+
logging=logging.LoggingConfig(
|
|
330
|
+
level="INFO",
|
|
331
|
+
handlers=[
|
|
332
|
+
logging.ConsoleLogHandler(),
|
|
333
|
+
logging.OTLPLogHandler(endpoint="http://collector:4317"),
|
|
334
|
+
]
|
|
335
|
+
),
|
|
336
|
+
tracing=tracing.TracingConfig(
|
|
337
|
+
processors=[
|
|
338
|
+
tracing.SpanProcessor(
|
|
339
|
+
processor=tracing.BatchSpanProcessor,
|
|
340
|
+
exporters=[tracing.OTLPSpanExporter(endpoint="http://collector:4317")]
|
|
341
|
+
)
|
|
342
|
+
]
|
|
343
|
+
),
|
|
344
|
+
metrics=metrics.MetricsConfig(
|
|
345
|
+
readers=[
|
|
346
|
+
metrics.MetricReader(
|
|
347
|
+
reader=metrics.PeriodicExportingMetricReader,
|
|
348
|
+
exporters=[metrics.OTLPMetricExporter(endpoint="http://collector:4317")]
|
|
349
|
+
)
|
|
350
|
+
]
|
|
351
|
+
),
|
|
352
|
+
)
|
|
353
|
+
```
|
|
354
|
+
|
|
355
|
+
### Logging
|
|
356
|
+
|
|
357
|
+
- Structured JSON logging via `python-json-logger`
|
|
358
|
+
- Automatic `trace_id` and `span_id` injection
|
|
359
|
+
- Multiple handlers: Console, File, RotatingFile, OTLP
|
|
360
|
+
|
|
361
|
+
### Tracing
|
|
362
|
+
|
|
363
|
+
- Hierarchical spans: Task → Batch → Transaction → Attempt
|
|
364
|
+
- Full context propagation across async boundaries
|
|
365
|
+
- Export to Jaeger, Tempo, or any OTLP-compatible backend
|
|
366
|
+
|
|
367
|
+
### Metrics
|
|
368
|
+
|
|
369
|
+
- Push-based via `PeriodicExportingMetricReader`
|
|
370
|
+
- Automatic resource attributes (task name, host, PID, execution ID)
|
|
371
|
+
- Export to Prometheus, OTLP, or console
|
|
372
|
+
|
|
373
|
+
---
|
|
374
|
+
|
|
375
|
+
## Scaling
|
|
376
|
+
|
|
377
|
+
| Axis | Mechanism | Configuration |
|
|
378
|
+
| ------------------- | ---------------------------- | ------------------------------------------ |
|
|
379
|
+
| **Process** | `ProcessPoolExecutor` | `TaskConfig.max_workers` |
|
|
380
|
+
| **Thread** | `ThreadPoolExecutor` | `policy.loop.concurrency.value` |
|
|
381
|
+
| **Async** | `asyncio.Semaphore` | `policy.loop.concurrency.value` |
|
|
382
|
+
| **Connector** | External system partitioning | Service-specific (consumer groups, shards) |
|
|
383
|
+
|
|
384
|
+
```python
|
|
385
|
+
# Multi-process scaling (sync tasks only)
|
|
386
|
+
config = TaskConfig(
|
|
387
|
+
name="scaled-task",
|
|
388
|
+
task=MyTask,
|
|
389
|
+
max_workers=4, # Spawn 4 isolated worker processes
|
|
390
|
+
connectors={...},
|
|
391
|
+
)
|
|
392
|
+
```
|
|
393
|
+
|
|
394
|
+
---
|
|
395
|
+
|
|
396
|
+
## Project Structure
|
|
397
|
+
|
|
398
|
+
After scaffolding with `ergon init`:
|
|
399
|
+
|
|
400
|
+
```
|
|
401
|
+
my-project/
|
|
402
|
+
├── main.py # Application entry point
|
|
403
|
+
├── _observability/ # Telemetry infrastructure configs
|
|
404
|
+
├── connectors/ # Custom connectors and services
|
|
405
|
+
│ └── {connector_name}/ # One submodule per connector
|
|
406
|
+
│ ├── connector.py # Transaction interface
|
|
407
|
+
│ └── service.py # Protocol mechanics
|
|
408
|
+
└── tasks/ # Task definitions and shared modules
|
|
409
|
+
├── settings.py # Global configs (connectors, services, telemetry)
|
|
410
|
+
├── constants.py # Global constants and enums
|
|
411
|
+
├── schemas.py # Shared Pydantic models
|
|
412
|
+
├── exceptions.py # Shared exception classes
|
|
413
|
+
├── helpers.py # Shared utility functions
|
|
414
|
+
└── {task_name}/ # Per-task submodule
|
|
415
|
+
├── task.py # Task implementation
|
|
416
|
+
├── config.py # TaskConfig definition
|
|
417
|
+
├── schemas.py # Task-specific models
|
|
418
|
+
├── exceptions.py # Task-specific exceptions
|
|
419
|
+
└── helpers.py # Task-specific utilities
|
|
420
|
+
```
|
|
421
|
+
|
|
422
|
+
📖 **[Full Project Structure Guide](docs/project-structure.md)** — Detailed documentation on organizing connectors, tasks, and shared modules.
|
|
423
|
+
|
|
424
|
+
---
|
|
425
|
+
|
|
426
|
+
## Documentation
|
|
427
|
+
|
|
428
|
+
### Python SDK Guides
|
|
429
|
+
|
|
430
|
+
- **[Getting Started](docs/getting-started.md)** — Installation, project setup, and running your first task
|
|
431
|
+
- **[CLI Reference](docs/cli.md)** — Commands, options, and exit codes
|
|
432
|
+
- **[Project Structure Guide](docs/project-structure.md)** — How to organize connectors, tasks, and shared modules
|
|
433
|
+
|
|
434
|
+
### Framework Concepts
|
|
435
|
+
|
|
436
|
+
- **[Framework Architecture](../../docs/architecture.md)** — Full system specification and design philosophy
|
|
437
|
+
- **[Transaction Abstraction](../../docs/modules/1.transaction.md)** — Understanding atomicity rules
|
|
438
|
+
- **[Task Module](../../docs/modules/2.task.md)** — Mixins, lifecycles, and execution modes
|
|
439
|
+
- **[Connector Module](../../docs/modules/3.connector.md)** — Building integration boundaries
|
|
440
|
+
- **[Service Module](../../docs/modules/4.service.md)** — Protocol engineering and reliability
|
|
441
|
+
- **[Telemetry Module](../../docs/modules/5.telemetry.md)** — Configuring OTel logs, metrics, and traces
|
|
442
|
+
|
|
443
|
+
---
|
|
444
|
+
|
|
445
|
+
## License
|
|
446
|
+
|
|
447
|
+
This project is licensed under the [MIT License](../../LICENSE).
|
|
448
|
+
|
|
449
|
+
<br/>
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
ergon/__init__.py,sha256=SE3PIboMgB-F2SrifSZA_gQY4WsTsPK-eoCbj5Qpejw,218
|
|
2
|
+
ergon/cli.py,sha256=CF8PvOjIguR18a657aEBEyhFXUoSVtSRVXv5PfnuWtA,6021
|
|
3
|
+
ergon/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
ergon/bootstrap/src/__project__/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
ergon/bootstrap/src/__project__/main.py,sha256=lIxQJL7084Y_QljrC9FZOC-JB5_loGjaIrB2GblCj8k,94
|
|
6
|
+
ergon/bootstrap/src/__project__/_observability/docker-compose.telemetry.yml,sha256=NUZDal-azFdsLoSGEOu5WmkfUey6rwjBqw4Om31DTE4,3529
|
|
7
|
+
ergon/bootstrap/src/__project__/_observability/grafana.yaml,sha256=2pJqqzHvaD45GHoJCkD-9nPI5XC-LErtPAcHkX3zHpY,272
|
|
8
|
+
ergon/bootstrap/src/__project__/_observability/loki.yaml,sha256=b-WGzuIb9ZvZ5-m4oZYrDTg-gYrJc7JQLVd22pYESVg,948
|
|
9
|
+
ergon/bootstrap/src/__project__/_observability/otel-collector-config.yaml,sha256=wlzGr9FhOi9lMKaqdjm4paana-zZv5M6WEGHhq6S_8U,1062
|
|
10
|
+
ergon/bootstrap/src/__project__/_observability/prometheus.yaml,sha256=1KKGqBBf4MXJuSJrLJ0v26ufrqLnRu9nvmAOUTXNHOw,228
|
|
11
|
+
ergon/bootstrap/src/__project__/_observability/tempo.yaml,sha256=QFohNJh4F3asgjXfrhZOwMbiIMSy3sfhB7-XM1zXikA,336
|
|
12
|
+
ergon/bootstrap/src/__project__/connectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
ergon/bootstrap/src/__project__/tasks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
+
ergon/bootstrap/src/__project__/tasks/constants.py,sha256=svwn3_nOy77EaqW_Sgbh0Pb-kRX4eWL5O9elK9xo_f4,324
|
|
15
|
+
ergon/bootstrap/src/__project__/tasks/exceptions.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
|
+
ergon/bootstrap/src/__project__/tasks/helpers.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
|
+
ergon/bootstrap/src/__project__/tasks/schemas.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
18
|
+
ergon/bootstrap/src/__project__/tasks/settings.py,sha256=eSNkPgepRMTDx01fOLl4DhFWVOwXiK6Em2tlDPFMO94,172
|
|
19
|
+
ergon/bootstrap/src/__project__/tasks/example_task/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
|
+
ergon/bootstrap/src/__project__/tasks/example_task/config.py,sha256=kGzV5bXQfylXf6ZNE-hRp19RarL3Pufo1hyxOPZIqRk,112
|
|
21
|
+
ergon/bootstrap/src/__project__/tasks/example_task/exceptions.py,sha256=LTWwquhM0izJnpsv3uHcRLZ-wkZCb4ZTOwsmX_am104,144
|
|
22
|
+
ergon/bootstrap/src/__project__/tasks/example_task/helpers.py,sha256=uR_XZEtxR6JzbJloi7OhJb77f7Wc6pnOe1W3CHvl8vE,122
|
|
23
|
+
ergon/bootstrap/src/__project__/tasks/example_task/schemas.py,sha256=3_qANKlJIQ90gaRNrDuBAM42UNzfDbLI4-F0kkjjWHc,155
|
|
24
|
+
ergon/bootstrap/src/__project__/tasks/example_task/task.py,sha256=XnmVg-B0N8Ld48FMLJgGfTTJLjmpy5Nog_Gaj1ruVEQ,36
|
|
25
|
+
ergon/connector/__init__.py,sha256=Ij7SZhlfrPSr8pycZ1qFZra7HwAhLvaDXaaKS8kZMSg,1546
|
|
26
|
+
ergon/connector/connector.py,sha256=jaF1iXKALE1IHHfcP54mTD5mvj-wzk5iNaLo96IOcDI,3944
|
|
27
|
+
ergon/connector/transaction.py,sha256=LMGEFfisdjGGCPrWps3nYJ-SRBKa-P2pOFmwbSgDN-E,291
|
|
28
|
+
ergon/connector/excel/__init__.py,sha256=FHaqlDXAzV7dmsg3sy3N-3nxFbUswMsp5DdGC7o07p0,480
|
|
29
|
+
ergon/connector/excel/connector.py,sha256=Tx2T9Ggo8pPAaSmua4j_PMu_QiiiezWm1Nru3Jr0E94,5327
|
|
30
|
+
ergon/connector/excel/models.py,sha256=9RSCaFJAaFTTI0BMZ8S87SWvRqpEsMA50Mw7Gc47_dE,905
|
|
31
|
+
ergon/connector/excel/service.py,sha256=H6T4oJ46espyHXoJRLsZT7gEje5elqYozChsRPI4aXc,3339
|
|
32
|
+
ergon/connector/pipefy/__init__.py,sha256=sTI6QMGOi7aMIEzyo56ZzH3KlvWkjaX7Zun1fOv2FOM,477
|
|
33
|
+
ergon/connector/pipefy/async_connector.py,sha256=Ls4fRcTD21gWbeSj32WpSID6b5ZYTNBjWN7kNw-9Ey0,1739
|
|
34
|
+
ergon/connector/pipefy/async_service.py,sha256=Kpd-LVTet3Ve6z85pUWZ2uUWFpgTaF74znbKFzmXAMA,26298
|
|
35
|
+
ergon/connector/pipefy/connector.py,sha256=kPeTfkpfO8V90PRTmVRGJaHD5z4emHepW7LUoa1x3Uw,1311
|
|
36
|
+
ergon/connector/pipefy/models.py,sha256=LrWAFqtNWdPkBI5m73MEytd0PckcG9kUOzMAOJS8or4,2014
|
|
37
|
+
ergon/connector/pipefy/service.py,sha256=n8kf2bMNLBK4Fabj9JLq-wr3Llah8z7pK3Iz8mTB-yk,29139
|
|
38
|
+
ergon/connector/pipefy/version.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
|
|
39
|
+
ergon/connector/postgres/__init__.py,sha256=8KAlpq2wwnkz1Az-e8YuBcrdMqFc6M-jauuwvvCdeLQ,338
|
|
40
|
+
ergon/connector/postgres/async_connector.py,sha256=LgZlN93_fjqFxZ6UPVJ1HbAVHXpS4itW8eCtzqHk1U4,3973
|
|
41
|
+
ergon/connector/postgres/async_service.py,sha256=PdPnNElMKzLoov1as3Kcb_viuSyBaVE6WzgKRjsJ82E,4126
|
|
42
|
+
ergon/connector/postgres/models.py,sha256=DL9wBdZ6QZPH1iVft71cTabnFUCH1Hg1RmG5f5E8578,1749
|
|
43
|
+
ergon/connector/rabbitmq/__init__.py,sha256=UVd16TEBVU_C9A7Uy-1JfhxZrWsRChh4qGFp6_rsXSw,673
|
|
44
|
+
ergon/connector/rabbitmq/async_connector.py,sha256=yQ-1QJvoOFO4QM5z7BrdzmyoGPP8aZYd31jiYjeWybo,4583
|
|
45
|
+
ergon/connector/rabbitmq/async_service.py,sha256=VEPedxp0n3eER5kgl5j86MEPZy1P5uXVjO2gebGE1lA,17172
|
|
46
|
+
ergon/connector/rabbitmq/connector.py,sha256=vsSzbVuuE8rKkwBfPgD97qTsqADxJGALnUvrEwZIL5Q,1633
|
|
47
|
+
ergon/connector/rabbitmq/helper.py,sha256=xlQGx2SGR2DC1fi-5BMlCOvWydWEFkcN7fPJjlNvGLA,372
|
|
48
|
+
ergon/connector/rabbitmq/models.py,sha256=9Zr6xJCIIDzVtL0Hdf2I1hpvNpUxH1c9nvy1UbwtFFg,5515
|
|
49
|
+
ergon/connector/rabbitmq/service.py,sha256=U-KVWtYec630SnexXe-8qmmiMpwYwKhW7lossxbponM,7109
|
|
50
|
+
ergon/connector/sqs/__init__.py,sha256=yagjZQ_uQDo20p18x9gnUsixKUaos_yskQ8RkwFUlcE,394
|
|
51
|
+
ergon/connector/sqs/async_connector.py,sha256=GX2VYplDl5veVwbl2m1sicnWbUHti45x27PUSBq5n6Q,4544
|
|
52
|
+
ergon/connector/sqs/async_service.py,sha256=78kVRkhXxwjM36kohTGneO3Yi-otsyf1mUrL9QeeJ3Q,8565
|
|
53
|
+
ergon/connector/sqs/connector.py,sha256=2kqiEGogQtygjNZk9NPSOSKcjrs67mOERF0LZKjIvbI,4454
|
|
54
|
+
ergon/connector/sqs/models.py,sha256=8NoNxmyV3MmTxQ3sytjok5fa4KpTr1J8-C62x-v1vXo,2064
|
|
55
|
+
ergon/connector/sqs/service.py,sha256=DgeJg_Mjw6iV2EWnEU1RQty53BiXh5aDqLzExspH3Kg,7504
|
|
56
|
+
ergon/service/__init__.py,sha256=UrjnIUFqFzkhKYZP8dmMO_i_pKMmwqeY0JcexvKvAGY,71
|
|
57
|
+
ergon/service/service.py,sha256=X4XaeHTKokkz7TGqR0VaNM1nUQSytR91Zukxj1k48X4,384
|
|
58
|
+
ergon/task/__init__.py,sha256=ekzzJ4fc-qkxy24trjvPc0T63FiOfrOudGommO04VMo,241
|
|
59
|
+
ergon/task/base.py,sha256=oD0moP4nMbEbuAvfjGEC0VcFx1BznLQ1LDj_ESR2KPU,6640
|
|
60
|
+
ergon/task/exceptions.py,sha256=92Kxtj5BRastdvFex7SskPu3VUEqqJyO76bb2mODof8,6824
|
|
61
|
+
ergon/task/helpers.py,sha256=Nyo3csTLtD5UHMhQJV3gJyWKBcZxH9K8H-RQO9g_pA4,21798
|
|
62
|
+
ergon/task/manager.py,sha256=3Xr8RPRjtCEoupMeyYb19uK0TQhoVehFlBZd8ZFZ-YA,2972
|
|
63
|
+
ergon/task/policies.py,sha256=mGb8iafrkVf438xiAUh0FpXqfgtNAM7RyoaF8jHNf80,7042
|
|
64
|
+
ergon/task/runner.py,sha256=5uTGlNcm7PfWSp95zBPwn91pVKdbxdfMK4N68HGPoSM,12945
|
|
65
|
+
ergon/task/utils.py,sha256=OmlP2AhLM32V8O9OMky6DdSvxwmMNn9x2LTguf0y-M0,2501
|
|
66
|
+
ergon/task/mixins/__init__.py,sha256=6A8vNgdjaQYNJvymWNF-kBN2oTaQBYRLctS3Sq1w_-Y,320
|
|
67
|
+
ergon/task/mixins/consumer.py,sha256=B1oX0XfB6hfNnbBf4dpiuxSn5vy-APvnZOZGZw5FZQs,37954
|
|
68
|
+
ergon/task/mixins/metrics.py,sha256=4bavwo8koWiN5ADD8t5qd2BVeVcgxo6ddzq-xOybeTg,13159
|
|
69
|
+
ergon/task/mixins/producer.py,sha256=xENTZdZ7t0dpw5YsreQT20iTycSaj1EA4ny-60u-SWM,20427
|
|
70
|
+
ergon/telemetry/__init__.py,sha256=8waeIAHEx6tMXWfw-zOEKgjrg7V8S1poG070MdAmi_8,100
|
|
71
|
+
ergon/telemetry/_resource.py,sha256=5uf6qG-6tIt2VOO7nZ7vmqUCBoeD0O6U37fGedJZrws,566
|
|
72
|
+
ergon/telemetry/logging.py,sha256=ltHKy6_hUinRJBGMdmpNeJs1C_TcPZMG4EHE-O1Tm8E,11385
|
|
73
|
+
ergon/telemetry/metrics.py,sha256=0T_CuNw9P5AG1U9k1-bNp1g7McyiNLUIt_zKyLOtrnA,2841
|
|
74
|
+
ergon/telemetry/tracing.py,sha256=_ktiZWAgB1DU-uN91Mzr0iYNXxbCN21LnKiRo5SOJxY,4454
|
|
75
|
+
ergon/utils/__init__.py,sha256=M0_VCjMN7jvSPlRZLZHeFKTHAl1zu3BnAhjrzJjDfiI,57
|
|
76
|
+
ergon/utils/env.py,sha256=lji4xENvfHqg2GTGnzG95VKpxxM4f3DHIKKP61xQG08,621
|
|
77
|
+
ergon_framework_python-0.1.0.dist-info/licenses/LICENSE,sha256=bWCIhHzaadz_ecLdCdtwyMafeQ17s2d7_FSSuSuEcDs,1079
|
|
78
|
+
ergon_framework_python-0.1.0.dist-info/METADATA,sha256=UKmLy8Z2Jxs1fozWY3CVDHfU9jduMvhiLXGZTmuSPE0,14739
|
|
79
|
+
ergon_framework_python-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
80
|
+
ergon_framework_python-0.1.0.dist-info/entry_points.txt,sha256=reZ934txb5OrtcNBgTGqCsXa1biUDQWubD2a9fGZynE,42
|
|
81
|
+
ergon_framework_python-0.1.0.dist-info/top_level.txt,sha256=KSPt6N7RzsjU7PRAalWQ9Cul7YTUxbo84yAJ-y2vLm8,6
|
|
82
|
+
ergon_framework_python-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Ergondata-Technologies
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
ergon
|