data-collection-framework 0.1.0__py3-none-any.whl

dcf/infra/modules/batch_collector/local/airflow/main.tf

@@ -0,0 +1,64 @@
+terraform {
+  required_version = ">= 1.0"
+  required_providers {
+    local = {
+      source  = "hashicorp/local"
+      version = "~> 2.0"
+    }
+    null = {
+      source  = "hashicorp/null"
+      version = "~> 3.0"
+    }
+  }
+}
+
+resource "local_file" "dockerfile" {
+  content  = templatefile("${path.module}/templates/airflow.Dockerfile.tftpl", {
+    target = "local"
+  })
+  filename = "${var.build_context}/Dockerfile"
+}
+
+resource "null_resource" "build" {
+  depends_on = [local_file.dockerfile]
+
+  triggers = {
+    content_hash = var.content_hash
+  }
+
+  provisioner "local-exec" {
+    command = "docker build -t ${var.image_tag} ${var.build_context}"
+  }
+}
+
+resource "local_file" "compose" {
+  content = templatefile("${path.module}/templates/docker-compose.yml.tftpl", {
+    image_tag      = var.image_tag
+    dag_dir        = var.dag_dir
+    docker_socket  = var.docker_socket
+    db_password    = var.db_password
+    admin_password = var.admin_password
+    fernet_key     = var.fernet_key
+    webserver_port = var.webserver_port
+  })
+  filename = var.compose_file_path
+}
+
+resource "null_resource" "up" {
+  depends_on = [local_file.compose, null_resource.build]
+
+  triggers = {
+    content_hash      = var.content_hash
+    compose_hash      = sha256(local_file.compose.content)
+    compose_file_path = var.compose_file_path
+  }
+
+  provisioner "local-exec" {
+    command = "docker compose -f ${var.compose_file_path} up -d"
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "docker compose -f ${self.triggers.compose_file_path} down --volumes"
+  }
+}

dcf/infra/modules/batch_collector/local/airflow/outputs.tf

@@ -0,0 +1,9 @@
+output "webserver_url" {
+  description = "URL of the local Airflow webserver"
+  value       = "http://localhost:${var.webserver_port}"
+}
+
+output "compose_file" {
+  description = "Absolute path to the generated docker-compose.yml"
+  value       = var.compose_file_path
+}

dcf/infra/modules/batch_collector/local/airflow/variables.tf

@@ -0,0 +1,59 @@
+variable "image_tag" {
+  type        = string
+  description = "Docker image tag for the Airflow image (e.g. dcf-airflow-local:latest)"
+}
+
+variable "build_context" {
+  type        = string
+  description = "Absolute path to the Airflow build context directory"
+}
+
+variable "content_hash" {
+  type        = string
+  description = "SHA256 of Airflow Dockerfile template — triggers rebuild when template changes"
+}
+
+variable "dag_dir" {
+  type        = string
+  description = "Absolute host path to the DAGs directory (mounted read-only into scheduler)"
+}
+
+variable "warehouse_path" {
+  type        = string
+  description = "Absolute host path to the warehouse (for DockerOperator volume mounts)"
+}
+
+variable "docker_socket" {
+  type        = string
+  default     = "/var/run/docker.sock"
+  description = "Host Docker socket path"
+}
+
+variable "db_password" {
+  type        = string
+  sensitive   = true
+  description = "PostgreSQL password for the local Airflow database"
+}
+
+variable "admin_password" {
+  type        = string
+  sensitive   = true
+  description = "Airflow webserver admin password"
+}
+
+variable "fernet_key" {
+  type        = string
+  sensitive   = true
+  description = "Airflow fernet key for encrypting connection passwords"
+}
+
+variable "compose_file_path" {
+  type        = string
+  description = "Absolute path where the generated docker-compose.yml will be written"
+}
+
+variable "webserver_port" {
+  type        = number
+  default     = 8090
+  description = "Host port to expose the Airflow webserver on"
+}

dcf/infra/modules/batch_collector/local/main.tf

@@ -0,0 +1,32 @@
+terraform {
+  required_version = ">= 1.0"
+  required_providers {
+    local = {
+      source  = "hashicorp/local"
+      version = "~> 2.0"
+    }
+    null = {
+      source  = "hashicorp/null"
+      version = "~> 3.0"
+    }
+  }
+}
+
+resource "local_file" "dockerfile" {
+  content  = templatefile("${path.module}/templates/batch_collector.Dockerfile.tftpl", {
+    java_enabled = var.java_enabled
+  })
+  filename = "${var.build_context}/Dockerfile"
+}
+
+resource "null_resource" "build" {
+  depends_on = [local_file.dockerfile]
+
+  triggers = {
+    content_hash = var.content_hash
+  }
+
+  provisioner "local-exec" {
+    command = "docker build -t ${var.image_tag} ${var.build_context}"
+  }
+}

dcf/infra/modules/batch_collector/local/outputs.tf

@@ -0,0 +1,4 @@
+output "image_tag" {
+  description = "Docker image tag that was built"
+  value       = var.image_tag
+}

dcf/infra/modules/batch_collector/local/variables.tf

@@ -0,0 +1,25 @@
+variable "collector_name" {
+  type        = string
+  description = "dcf collector name (e.g. github_repos)"
+}
+
+variable "build_context" {
+  type        = string
+  description = "Absolute path to the stable build context directory"
+}
+
+variable "image_tag" {
+  type        = string
+  description = "Docker image tag (e.g. dcf-local/github_repos:latest)"
+}
+
+variable "content_hash" {
+  type        = string
+  description = "SHA256 of build context files — triggers rebuild when changed"
+}
+
+variable "java_enabled" {
+  type        = bool
+  default     = true
+  description = "Install OpenJDK in the container (required for local Iceberg/Spark)"
+}

dcf/infra/templates/airflow.Dockerfile.tftpl

@@ -0,0 +1,6 @@
+FROM apache/airflow:2.10.4-python3.12
+%{ if target == "local" ~}
+RUN pip install --no-cache-dir apache-airflow-providers-docker
+%{ else ~}
+RUN pip install --no-cache-dir apache-airflow-providers-google
+%{ endif ~}

dcf/infra/templates/batch_collector.Dockerfile.tftpl

@@ -0,0 +1,14 @@
+FROM python:3.12-slim
+%{ if java_enabled ~}
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    openjdk-21-jre-headless && rm -rf /var/lib/apt/lists/*
+%{ endif ~}
+WORKDIR /app
+COPY pyproject.toml .
+COPY ddt/ ./ddt/
+RUN pip install --no-cache-dir -e .
+COPY pipelines/ ./pipelines/
+COPY connectors/ ./connectors/
+COPY project.yml .
+ENV PIPELINE_NAME=""
+CMD ["sh", "-c", "ddt run $PIPELINE_NAME"]

dcf/infra/templates/docker-compose.yml.tftpl

@@ -0,0 +1,76 @@
+version: "3.8"
+
+x-airflow-env: &airflow-env
+  AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:${db_password}@postgres/airflow
+  AIRFLOW__CORE__EXECUTOR: LocalExecutor
+  AIRFLOW__CORE__FERNET_KEY: ${fernet_key}
+  AIRFLOW__WEBSERVER__SECRET_KEY: ${fernet_key}
+  AIRFLOW__CORE__LOAD_EXAMPLES: "false"
+  AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: "30"
+  AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth"
+
+services:
+  postgres:
+    image: postgres:15
+    environment:
+      POSTGRES_USER: airflow
+      POSTGRES_PASSWORD: ${db_password}
+      POSTGRES_DB: airflow
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "airflow"]
+      interval: 5s
+      retries: 10
+
+  airflow-init:
+    image: ${image_tag}
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      <<: *airflow-env
+    entrypoint:
+      - bash
+      - -c
+      - >-
+        airflow db migrate &&
+        airflow users create
+        --username admin
+        --password ${admin_password}
+        --firstname Admin
+        --lastname User
+        --role Admin
+        --email admin@ddt.local
+
+  airflow-scheduler:
+    image: ${image_tag}
+    depends_on:
+      airflow-init:
+        condition: service_completed_successfully
+    environment:
+      <<: *airflow-env
+    volumes:
+      - ${dag_dir}:/opt/airflow/dags:ro
+      - ${docker_socket}:/var/run/docker.sock
+    command: scheduler
+    restart: on-failure
+
+  airflow-webserver:
+    image: ${image_tag}
+    depends_on:
+      airflow-init:
+        condition: service_completed_successfully
+    environment:
+      <<: *airflow-env
+    ports:
+      - "${webserver_port}:8080"
+    command: webserver
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
+      interval: 10s
+      retries: 15
+    restart: on-failure
+
+volumes:
+  postgres_data: