ergon-framework-python 0.1.0__py3-none-any.whl

ergon/__init__.py

@@ -0,0 +1,13 @@
+from . import connector, service, task, telemetry, utils
+from .task.manager import manager
+
+__all__ = [
+    "service",
+    "connector",
+    "task",
+    "telemetry",
+    "utils",
+    "manager",
+]
+
+__version__ = "0.1.0"

ergon/bootstrap/src/__project__/_observability/docker-compose.telemetry.yml

@@ -0,0 +1,124 @@
+services:
+
+  ############################################################
+  # OpenTelemetry Collector
+  ############################################################
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.114.0
+    container_name: ergon-otel-collector
+    restart: unless-stopped
+    command: ["--config=/etc/otelcol/config.yaml"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otelcol/config.yaml:ro
+    ports:
+      - "4317:4317"     # OTLP gRPC
+      - "4318:4318"     # OTLP HTTP
+      - "9464:9464"     # Prometheus scrape (metrics)
+    depends_on:
+      - tempo
+      - loki
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:13133/"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - ergon-telemetry
+
+  ############################################################
+  # Loki (Logs)
+  ############################################################
+  loki:
+    image: grafana/loki:3.0.0
+    container_name: ergon-loki
+    restart: unless-stopped
+    volumes:
+      - ./loki.yaml:/etc/loki/config.yaml:ro
+      - loki-data:/loki
+    command: -config.file=/etc/loki/config.yaml
+    ports:
+      - "3100:3100"
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:3100/ready"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - ergon-telemetry
+
+  ############################################################
+  # Tempo (Traces)
+  ############################################################
+  tempo:
+    image: grafana/tempo:2.4.1
+    container_name: ergon-tempo
+    restart: unless-stopped
+    volumes:
+      - ./tempo.yaml:/etc/tempo/config.yaml:ro
+      - tempo-data:/var/tempo
+    command: ["-config.file=/etc/tempo/config.yaml"]
+    ports:
+      - "3200:3200"
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:3200/ready"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - ergon-telemetry
+
+  ############################################################
+  # Prometheus (Metrics)
+  ############################################################
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: ergon-prometheus
+    restart: unless-stopped
+    volumes:
+      - ./prometheus.yaml:/etc/prometheus/prometheus.yml:ro
+      - prom-data:/prometheus
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+    ports:
+      - "9090:9090"
+    depends_on:
+      - otel-collector
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:9090/-/ready"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - ergon-telemetry
+
+  ############################################################
+  # Grafana Dashboard UI
+  ############################################################
+  grafana:
+    image: grafana/grafana:10.4.0
+    container_name: ergon-grafana
+    restart: unless-stopped
+    volumes:
+      - ./grafana.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:ro
+      - grafana-data:/var/lib/grafana
+    ports:
+      - "3000:3000"
+    networks:
+      - ergon-telemetry
+
+
+############################################################
+# Volumes
+############################################################
+volumes:
+  loki-data:
+  tempo-data:
+  prom-data:
+  grafana-data:
+
+############################################################
+# Network
+############################################################
+networks:
+  ergon-telemetry:
+    driver: bridge

ergon/bootstrap/src/__project__/_observability/grafana.yaml

@@ -0,0 +1,17 @@
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+
+  - name: Tempo
+    type: tempo
+    access: proxy
+    url: http://tempo:3200
+
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090

ergon/bootstrap/src/__project__/_observability/loki.yaml

@@ -0,0 +1,48 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: "2024-01-01"
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+pattern_ingester:
+  enabled: true
+
+compactor:
+  working_directory: /loki/boltdb-shipper-compactor
+  # shared_store: filesystem  <-- REMOVED (Invalid in Loki 3.0)
+  delete_request_store: filesystem # <-- ADDED (Required for retention)
+  retention_enabled: true 
+
+limits_config:
+  retention_period: 24h
+  allow_structured_metadata: true
+  max_query_length: 720h

ergon/bootstrap/src/__project__/_observability/otel-collector-config.yaml

@@ -0,0 +1,53 @@
+receivers:
+  otlp:
+    protocols:
+      http: 
+        endpoint: 0.0.0.0:4318
+      grpc:
+        endpoint: 0.0.0.0:4317
+
+exporters:
+  loki:
+    endpoint: http://loki:3100/loki/api/v1/push
+    labels:
+      resource:
+        service.name: "service_name"
+        service.version: "service_version"
+        ergon.task.name: "task_name"
+        ergon.task.execution.host.name: "host_name"
+      attributes:
+        level: "level"
+  
+    default_labels_enabled:
+      exporter: false
+      resource: false
+      attributes: false
+      scope: false
+
+  otlphttp/tempo:
+    endpoint: http://tempo:4318
+    tls:
+      insecure: true
+
+  prometheus:
+    endpoint: "0.0.0.0:9464"
+
+processors:
+  batch: {}
+
+service:
+  pipelines:
+    logs:
+      receivers: [otlp]
+      processors: [batch]  # ← Changed: removed "resource"
+      exporters: [loki]
+
+    traces:
+      receivers: [otlp]
+      processors: [batch]  # ← Changed: removed "resource"
+      exporters: [otlphttp/tempo]
+
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [prometheus]

ergon/bootstrap/src/__project__/_observability/prometheus.yaml

@@ -0,0 +1,11 @@
+global:
+  scrape_interval: 5s
+
+scrape_configs:
+  - job_name: "otel-collector"
+    static_configs:
+      - targets: ["otel-collector:9464"]
+
+  - job_name: "ergon_tasks"
+    static_configs:
+      - targets: ["otel-collector:9464"]

ergon/bootstrap/src/__project__/_observability/tempo.yaml

@@ -0,0 +1,24 @@
+server:
+  http_listen_port: 3200
+  grpc_listen_port: 9095
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+        http:
+
+ingester:
+  max_block_bytes: 100_000_000
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    block_retention: 24h
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /var/tempo/traces

ergon/bootstrap/src/__project__/main.py

@@ -0,0 +1,9 @@
+from ergon.cli import ergon
+
+
+def main():
+    ergon()
+
+
+if __name__ == "__main__":
+    main()

ergon/bootstrap/src/__project__/tasks/constants.py

@@ -0,0 +1,13 @@
+from ergon.task.policies import RetryPolicy
+
+
+# ----------------------------------------
+# --- RETRY POLICY CONFIGURATION ---
+# ----------------------------------------
+def default_retry_policy():
+    return RetryPolicy(
+        max_attempts=3,
+        backoff=1,
+        backoff_multiplier=2,
+        backoff_cap=10,
+    )

ergon/bootstrap/src/__project__/tasks/example_task/config.py

@@ -0,0 +1,4 @@
+# Uncomment below to use framework imports
+# ----------------------------------------
+
+# from ergon import task

ergon/bootstrap/src/__project__/tasks/example_task/exceptions.py

@@ -0,0 +1,4 @@
+# Uncomment below to use framework imports
+# ----------------------------------------
+
+# from ergon.task.exceptions import TransactionException

ergon/bootstrap/src/__project__/tasks/example_task/helpers.py

@@ -0,0 +1,4 @@
+# Uncomment below to use framework imports
+# ----------------------------------------
+
+# from .. import helpers as shared

ergon/bootstrap/src/__project__/tasks/example_task/schemas.py

@@ -0,0 +1,5 @@
+# Uncomment below to use framework imports
+# ----------------------------------------
+
+# from pydantic import BaseModel
+# from .. import schemas as shared

ergon/bootstrap/src/__project__/tasks/example_task/task.py

@@ -0,0 +1 @@
+# from ergon import task, connector

ergon/bootstrap/src/__project__/tasks/settings.py

@@ -0,0 +1,5 @@
+# Uncomment below to use framework imports
+# ----------------------------------------
+
+# from ergon import service, connector, telemetry
+# from ergon.utils import load_env

ergon/cli.py

@@ -0,0 +1,174 @@
+import argparse
+import json
+import shutil
+import sys
+import uuid
+from pathlib import Path
+from typing import List, Optional
+
+from .connector import Transaction
+from .task.base import TaskConfig
+from .task.manager import manager
+
+
+# ---------------------------------------------------------------------
+# Utility: bootstrap project
+# ---------------------------------------------------------------------
+def bootstrap_project(project_name: str, target_dir: str):
+    root = Path(__file__).resolve().parent
+    template_src = root / "bootstrap" / "src" / "__project__"
+
+    if not template_src.exists():
+        raise RuntimeError(f"Template missing: {template_src}")
+
+    target = Path(target_dir).resolve()
+    target_src = target / "src" / project_name
+    target_src.mkdir(parents=True, exist_ok=True)
+
+    for item in template_src.rglob("*"):
+        rel = item.relative_to(template_src)
+        dest = target_src / rel
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        if item.is_file():
+            shutil.copy2(item, dest)
+
+    print(f"✔ Bootstrapped successfully at src/{project_name}")
+
+
+# ---------------------------------------------------------------------
+# CLI parser
+# ---------------------------------------------------------------------
+def create_parser():
+    parser = argparse.ArgumentParser(
+        prog="ergon",
+        description="Ergon Task Framework CLI",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # -------------------------------------------------------------
+    # ergon list
+    # -------------------------------------------------------------
+    subparsers.add_parser("list", help="List all registered tasks")
+
+    # -------------------------------------------------------------
+    # ergon run <task>
+    # -------------------------------------------------------------
+    run_parser = subparsers.add_parser("run", help="Run a registered task")
+    run_parser.add_argument("task_name", type=str)
+
+    # -------------------------------------------------------------
+    # ergon process-transaction
+    # -------------------------------------------------------------
+    tx_parser = subparsers.add_parser(
+        "process-transaction",
+        help="Process a single transaction (inline payload)",
+    )
+    tx_parser.add_argument("task", type=str)
+    tx_parser.add_argument("policy", type=str)
+    tx_parser.add_argument(
+        "--payload-json",
+        required=True,
+        help="Transaction payload as JSON string",
+    )
+
+    # -------------------------------------------------------------
+    # ergon process-transaction-by-id
+    # -------------------------------------------------------------
+    tx_id_parser = subparsers.add_parser(
+        "process-transaction-by-id",
+        help="Process a transaction by ID via connector fetch",
+    )
+    tx_id_parser.add_argument("task", type=str)
+    tx_id_parser.add_argument("policy", type=str)
+    tx_id_parser.add_argument("transaction_id", type=str)
+
+    # -------------------------------------------------------------
+    # ergon bootstrap
+    # -------------------------------------------------------------
+    bootstrap_parser = subparsers.add_parser(
+        "bootstrap",
+        help="Create a new project using the Ergon template",
+    )
+    bootstrap_parser.add_argument("project_name", type=str)
+    bootstrap_parser.add_argument(
+        "target_dir",
+        nargs="?",
+        default=".",
+        help="Target directory (default: current directory)",
+    )
+
+    return parser
+
+
+# ---------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------
+def ergon(tasks: Optional[List[TaskConfig]] = None):
+    if tasks:
+        for cfg in tasks:
+            if not manager.get(cfg.name):
+                manager.register(cfg)
+
+    parser = create_parser()
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.print_help()
+        sys.exit(1)
+
+    # -------------------------------------------------------------
+    # LIST TASKS
+    # -------------------------------------------------------------
+    if args.command == "list":
+        task_names = manager.list_tasks()
+        if not task_names:
+            print("No tasks registered. Did you import your config files?")
+        else:
+            print("Registered tasks:")
+            for t in task_names:
+                print(f"  - {t}")
+        return
+
+    # -------------------------------------------------------------
+    # RUN TASK
+    # -------------------------------------------------------------
+    if args.command == "run":
+        manager.run(args.task_name)
+        return
+
+    # -------------------------------------------------------------
+    # PROCESS TRANSACTION (INLINE)
+    # -------------------------------------------------------------
+    if args.command == "process-transaction":
+        try:
+            payload = json.loads(args.payload_json)
+        except json.JSONDecodeError as exc:
+            print(f"[ERROR] Invalid JSON payload: {exc}")
+            sys.exit(2)
+
+        transaction = Transaction(id=str(uuid.uuid4()), payload=payload)
+        manager.process_transaction(
+            task=args.task,
+            policy=args.policy,
+            transaction=transaction,
+        )
+        return
+
+    # -------------------------------------------------------------
+    # PROCESS TRANSACTION BY ID
+    # -------------------------------------------------------------
+    if args.command == "process-transaction-by-id":
+        manager.process_transaction_by_id(
+            task=args.task,
+            policy=args.policy,
+            transaction_id=args.transaction_id,
+        )
+        return
+
+    # -------------------------------------------------------------
+    # BOOTSTRAP
+    # -------------------------------------------------------------
+    if args.command == "bootstrap":
+        bootstrap_project(args.project_name, args.target_dir)
+        return

ergon/connector/__init__.py

@@ -0,0 +1,64 @@
+from .connector import AsyncConnector, Connector, ConnectorConfig
+from .excel import ExcelConnector, ExcelFetchConfig, ExcelRow, ExcelService
+from .postgres import (
+    AsyncPostgresConnector,
+    AsyncPostgresService,
+    PostgresClient,
+    PostgresConsumerConfig,
+    PostgresProducerConfig,
+)
+from .rabbitmq import (
+    AsyncRabbitmqClient,
+    AsyncRabbitMQConnector,
+    AsyncRabbitmqConsumerConfig,
+    AsyncRabbitmqProducerConfig,
+    AsyncRabbitMQService,
+    RabbitmqClient,
+    RabbitMQConnector,
+    RabbitmqConsumerMessage,
+    RabbitmqProducerMessage,
+    RabbitMQService,
+)
+from .sqs import (
+    AsyncSQSConnector,
+    AsyncSQSService,
+    SQSClient,
+    SQSConnector,
+    SQSConsumerConfig,
+    SQSProducerConfig,
+    SQSService,
+)
+from .transaction import Transaction
+
+__all__ = [
+    "AsyncConnector",
+    "AsyncPostgresConnector",
+    "AsyncPostgresService",
+    "AsyncRabbitMQConnector",
+    "AsyncRabbitMQService",
+    "AsyncRabbitmqClient",
+    "AsyncRabbitmqConsumerConfig",
+    "AsyncRabbitmqProducerConfig",
+    "AsyncSQSConnector",
+    "AsyncSQSService",
+    "Connector",
+    "ConnectorConfig",
+    "ExcelConnector",
+    "ExcelFetchConfig",
+    "ExcelRow",
+    "ExcelService",
+    "PostgresClient",
+    "PostgresConsumerConfig",
+    "PostgresProducerConfig",
+    "RabbitMQConnector",
+    "RabbitMQService",
+    "RabbitmqClient",
+    "RabbitmqConsumerMessage",
+    "RabbitmqProducerMessage",
+    "SQSClient",
+    "SQSConnector",
+    "SQSConsumerConfig",
+    "SQSProducerConfig",
+    "SQSService",
+    "Transaction",
+]