data-ingestor 0.2.0__tar.gz

data_ingestor-0.2.0/.github/dependabot.yml

@@ -0,0 +1,23 @@
+version: 2
+updates:
+  - package-ecosystem: "uv"
+    directory: "/"
+    target-branch: "develop"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "chore"
+    groups:
+      python:
+        patterns: ["*"]
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    target-branch: "develop"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "ci"
+    groups:
+      actions:
+        patterns: ["*"]

data_ingestor-0.2.0/.github/workflows/checks.yml

@@ -0,0 +1,72 @@
+name: Checks
+
+on:
+  push:
+    branches: [main, develop, "v*"]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: checks-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          enable-cache: true
+      - run: uv python install 3.14
+      - name: Ruff lint
+        run: uv run --group dev ruff check .
+      - name: Ruff format
+        run: uv run --group dev ruff format --check .
+
+  types:
+    name: types
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          enable-cache: true
+      - run: uv python install 3.14
+      - name: Mypy
+        run: uv run --group dev --extra sse --extra ws mypy src/ingestor
+
+  build:
+    name: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          enable-cache: true
+      - run: uv python install 3.14
+      - name: Lock is in sync
+        run: uv lock --check
+      - name: Build sdist and wheel
+        run: uv build
+
+  docs:
+    name: docs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          enable-cache: true
+      - run: uv python install 3.14
+      - name: MkDocs strict build
+        run: uv run --group docs mkdocs build --strict

data_ingestor-0.2.0/.github/workflows/codeql.yml

@@ -0,0 +1,30 @@
+name: CodeQL
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    branches: [main, develop]
+  schedule:
+    - cron: "0 3 * * 1"
+
+concurrency:
+  group: codeql-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: CodeQL
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@24c54180a607b1449ed407dd24f251e4e9147c8d # v4
+        with:
+          languages: python
+      - name: Analyze
+        uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4

data_ingestor-0.2.0/.github/workflows/publish.yml

@@ -0,0 +1,73 @@
+name: Publish
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch: {} # manual runs publish to TestPyPI only
+
+concurrency:
+  group: publish-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          fetch-depth: 0 # full history and tags so hatch-vcs derives the version
+      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          enable-cache: true
+      - run: uv python install 3.14
+      - name: Lint
+        run: |
+          uv run --group dev ruff check .
+          uv run --group dev ruff format --check .
+      - name: Types
+        run: uv run --group dev --extra sse --extra ws mypy src/ingestor
+      - name: Tests
+        run: uv run --group test --extra sse --extra ws pytest
+      - name: Build
+        run: uv build
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
+        with:
+          name: dist
+          path: dist/
+
+  pypi:
+    name: pypi
+    needs: build
+    if: github.event_name == 'release'
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/data-ingestor
+    permissions:
+      id-token: write # OIDC token for PyPI trusted publishing
+    steps:
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1 # Docker action: pin by release tag, not SHA
+
+  testpypi:
+    name: testpypi
+    needs: build
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/data-ingestor
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1 # Docker action: pin by release tag, not SHA
+        with:
+          repository-url: https://test.pypi.org/legacy/

data_ingestor-0.2.0/.github/workflows/tests.yml

@@ -0,0 +1,32 @@
+name: Tests
+
+on:
+  push:
+    branches: [main, develop, "v*"]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  tests:
+    name: pytest + coverage
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          fetch-depth: 0
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          enable-cache: true
+
+      - name: Install Python 3.14
+        run: uv python install 3.14
+
+      - name: Run tests with coverage (fails under 80%)
+        run: uv run --group test --extra sse --extra ws pytest

data_ingestor-0.2.0/.gitignore

@@ -0,0 +1,29 @@
+# Python Files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual Environment
+.venv
+
+# Environment Variables
+.env
+
+# Archify Working Folders
+.archify/
+
+# MkDocs Build Output
+site/
+
+# Tests and coverage
+.coverage
+.coverage.*
+htmlcov/
+.pytest_cache/
+
+# Tool caches
+.mypy_cache/
+.ruff_cache/

data_ingestor-0.2.0/.python-version

@@ -0,0 +1 @@
+3.14

data_ingestor-0.2.0/.vscode/settings.json

@@ -0,0 +1,39 @@
+{
+  "editor.fontFamily": "'Fira Code', 'JetBrains Mono', Menlo, Monaco, 'Courier New', monospace",
+  "editor.fontLigatures": true,
+  "explorer.compactFolders": false,
+  "makefile.configureOnOpen": false,
+  "cSpell.words": [
+    "Ccircle",
+    "Cpath",
+    "Crect",
+    "Csvg",
+    "enwiktionary",
+    "Etampes",
+    "explorable",
+    "frwiki",
+    "fsrc",
+    "glightbox",
+    "httpx",
+    "inlinehilite",
+    "kout",
+    "ksink",
+    "ksrc",
+    "messagebus",
+    "mkdocstrings",
+    "neighbouring",
+    "pannable",
+    "Pudox",
+    "rasterizes",
+    "rescoping",
+    "sandboxed",
+    "SASL",
+    "Songti",
+    "sublabel",
+    "tasklist",
+    "timeslice",
+    "unimportable",
+    "Victar",
+    "wikidatawiki"
+  ]
+}

data_ingestor-0.2.0/Makefile

@@ -0,0 +1,65 @@
+.DEFAULT_GOAL := help
+.PHONY: help install sync reinstall run test up stop logs ps wiki btc batch custom micro-batch bridge consume-btc docs-serve docs-build docs-deploy clean
+
+help:
+	@grep -E '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) \
+		| awk 'BEGIN{FS=":.*?## "}{printf "  \033[36m%-16s\033[0m %s\n", $$1, $$2}'
+
+install:
+	uv sync
+
+sync: install
+
+reinstall:
+	uv sync --reinstall-package data-ingestor
+
+run:
+	PYTHONPATH=src uv run python -m ingestor
+
+test:
+	PYTHONPATH=src uv run python -m unittest discover -v
+
+up:
+	docker compose up -d
+
+stop:
+	docker compose stop
+
+logs:
+	docker compose logs -f kafka
+
+ps:
+	docker compose ps
+
+wiki:
+	PYTHONPATH=src uv run --extra sse python examples/wikimedia_stream.py
+
+btc:
+	PYTHONPATH=src uv run --extra ws python examples/bitcoin_stream.py
+
+batch:
+	PYTHONPATH=src uv run python examples/batch_file.py $(FILE)
+
+custom:
+	PYTHONPATH=src uv run python examples/custom_adapter.py
+
+micro-batch:
+	PYTHONPATH=src uv run python examples/micro_batch.py
+
+bridge:
+	PYTHONPATH=src uv run --extra ws python examples/stream_to_kafka.py $(N)
+
+consume-btc:
+	PYTHONPATH=src uv run python examples/consume_btc_ticks.py
+
+docs-serve:
+	uv run --group docs mkdocs serve
+
+docs-build:
+	uv run --group docs mkdocs build --strict
+
+docs-deploy:
+	uv run --group docs mkdocs gh-deploy --force
+
+clean:
+	find . -type d -name __pycache__ -prune -exec rm -rf {} + 2>/dev/null || true

data_ingestor-0.2.0/PKG-INFO

@@ -0,0 +1,162 @@
+Metadata-Version: 2.5
+Name: data-ingestor
+Version: 0.2.0
+Summary: Data ingestion (streaming, batch, micro-batch) with a ports-and-adapters architecture
+Requires-Python: >=3.14
+Requires-Dist: confluent-kafka>=2.15.0
+Provides-Extra: sse
+Requires-Dist: httpx>=0.27; extra == 'sse'
+Provides-Extra: ws
+Requires-Dist: websocket-client>=1.8; extra == 'ws'
+Description-Content-Type: text/markdown
+
+# Ingestor
+
+A Python **data ingestion** app that reads records from a source (**Source**) and writes them to a destination (**Sink**). The same logic serves several modes: **streaming** (Kafka), **batch** (file) and **micro-batch** (by limit/window).
+
+The architecture is **ports and adapters**: the logic depends only on interfaces (`ingestor.ports`), and each tool lives isolated in its own adapter (`ingestor.adapters.*`). Swapping source or destination - or using in-memory fakes in tests - does not touch the logic.
+
+## Requirements
+
+- Python 3.14 or newer.
+- `uv` installed to manage dependencies.
+- Docker with Docker Compose and Docker Desktop running (only for the Kafka mode).
+- Port `9092` free for local Kafka.
+
+## How to run
+
+### 1. Clone the project
+
+```bash
+git clone https://github.com/rikemorais/ingestor.git
+cd ingestor
+```
+
+### 2. Install dependencies
+
+```bash
+uv sync
+```
+
+### 3. Start Kafka
+
+```bash
+docker compose up -d
+```
+
+Wait for the broker to start. To follow the logs:
+
+```bash
+docker compose logs -f kafka
+```
+
+Press `Ctrl+C` to leave the logs. Kafka keeps running.
+
+### 4. Run the app
+
+```bash
+uv run ingestor
+```
+
+> `ingestor` is the command defined in `pyproject.toml` (`[project.scripts]`). Equivalent alternative: `uv run python -m ingestor`.
+
+The demo seeds a few records into the topic and then **ingests from Kafka to the console**. With local Kafka available, the expected output is:
+
+```text
+Connected to Kafka: 1 broker(s).
+Ingested: origin=demo-messages key=1 value=Hello Kafka #1
+Ingested: origin=demo-messages key=2 value=Hello Kafka #2
+Ingested: origin=demo-messages key=3 value=Hello Kafka #3
+Ingested: origin=demo-messages key=4 value=Hello Kafka #4
+Ingested: origin=demo-messages key=5 value=Hello Kafka #5
+Ingested 5 records.
+```
+
+The connection uses `localhost:9092` by default and waits up to 10 seconds for a response.
+
+> Note: `KafkaSource` consumes in the `ingestor` group and, after a few consecutive empty polls (`max_empty_polls`), stops - that's why the demo terminates instead of waiting forever. Running again with the same group won't re-read already-read messages (the offset is saved). To re-consume everything, change the group in `src/ingestor/adapters/kafka/connection.py` or recreate the topic.
+
+## Use another address
+
+Set `KAFKA_BOOTSTRAP_SERVERS` when running:
+
+```bash
+KAFKA_BOOTSTRAP_SERVERS=server:9092 uv run ingestor
+```
+
+Replace `server:9092` with your broker's address. The current setup uses an unauthenticated connection; SASL/TLS clusters need adjustments in the adapter.
+
+## Run the tests (no Kafka)
+
+The tests use in-memory implementations (fakes) and a file source, so they **need neither Docker nor a broker**:
+
+```bash
+uv run python -m unittest discover -v
+```
+
+They exercise the same `ingest` function with a `FakeConnection` and with a `FileSource` (batch) - proving in practice that the logic does not depend on the Kafka lib.
+
+> Tip: when adding/moving files under `src/`, if `uv run` complains about a module not found, run once `uv sync --reinstall-package data-ingestor`.
+
+## Stop Kafka
+
+```bash
+docker compose stop
+```
+
+To start the same container again:
+
+```bash
+docker compose start
+```
+
+## If the connection fails
+
+Check that Docker is running and inspect the broker's state and logs:
+
+```bash
+docker compose ps
+docker compose logs --tail=50 kafka
+```
+
+If Kafka is still starting up, wait and run again. Also check the address and that port `9092` is free. On failure, the app prints the error and exits with code `1`.
+
+## Structure
+
+`src/` layout, with the code in an `ingestor` package organized as ports × adapters:
+
+```text
+ingestor/
+├── src/
+│   └── ingestor/
+│       ├── __init__.py
+│       ├── app.py              # use case: ingest(source, sink, limit)
+│       ├── cli.py              # Composition Root: picks and injects the adapters
+│       ├── __main__.py         # enables `python -m ingestor`
+│       ├── ports/              # interfaces + domain (tool-agnostic)
+│       │   ├── record.py       # Record (value, key, origin)
+│       │   ├── source.py       # Source: read() -> Iterator[Record]
+│       │   ├── sink.py         # Sink: write(record)
+│       │   └── connection.py   # Connection: source() / sink()
+│       └── adapters/           # concrete implementations (one folder per tool)
+│           ├── kafka/          # streaming: KafkaSource, KafkaSink, KafkaConnection, KafkaInfoCluster
+│           ├── console/        # generic destination: ConsoleSink
+│           └── batch/          # batch: FileSource (reads a file line by line)
+├── tests/
+│   ├── __init__.py
+│   ├── fakes.py                # ListSource, MemorySink, FakeConnection
+│   └── test_ingestor.py        # unittest tests
+├── docker-compose.yml          # local Kafka
+├── pyproject.toml              # dependencies, build and the `ingestor` command
+└── uv.lock
+```
+
+**How the layers relate:** `app.py` depends only on `ingestor.ports` (Source, Sink, Connection); the adapters in `adapters/` implement those ports; and only `cli.py` knows the concrete classes, wiring everything together. This way no specific lib (like `confluent_kafka`) "leaks" into the logic or the tests.
+
+**The three modes, the same logic:**
+
+| Mode | Source | Does `ingest` change? |
+|---|---|---|
+| Streaming | `KafkaSource` | no |
+| Batch | `FileSource` | no |
+| Micro-batch | any Source + `limit` | no |

data_ingestor-0.2.0/README.md

@@ -0,0 +1,150 @@
+# Ingestor
+
+A Python **data ingestion** app that reads records from a source (**Source**) and writes them to a destination (**Sink**). The same logic serves several modes: **streaming** (Kafka), **batch** (file) and **micro-batch** (by limit/window).
+
+The architecture is **ports and adapters**: the logic depends only on interfaces (`ingestor.ports`), and each tool lives isolated in its own adapter (`ingestor.adapters.*`). Swapping source or destination - or using in-memory fakes in tests - does not touch the logic.
+
+## Requirements
+
+- Python 3.14 or newer.
+- `uv` installed to manage dependencies.
+- Docker with Docker Compose and Docker Desktop running (only for the Kafka mode).
+- Port `9092` free for local Kafka.
+
+## How to run
+
+### 1. Clone the project
+
+```bash
+git clone https://github.com/rikemorais/ingestor.git
+cd ingestor
+```
+
+### 2. Install dependencies
+
+```bash
+uv sync
+```
+
+### 3. Start Kafka
+
+```bash
+docker compose up -d
+```
+
+Wait for the broker to start. To follow the logs:
+
+```bash
+docker compose logs -f kafka
+```
+
+Press `Ctrl+C` to leave the logs. Kafka keeps running.
+
+### 4. Run the app
+
+```bash
+uv run ingestor
+```
+
+> `ingestor` is the command defined in `pyproject.toml` (`[project.scripts]`). Equivalent alternative: `uv run python -m ingestor`.
+
+The demo seeds a few records into the topic and then **ingests from Kafka to the console**. With local Kafka available, the expected output is:
+
+```text
+Connected to Kafka: 1 broker(s).
+Ingested: origin=demo-messages key=1 value=Hello Kafka #1
+Ingested: origin=demo-messages key=2 value=Hello Kafka #2
+Ingested: origin=demo-messages key=3 value=Hello Kafka #3
+Ingested: origin=demo-messages key=4 value=Hello Kafka #4
+Ingested: origin=demo-messages key=5 value=Hello Kafka #5
+Ingested 5 records.
+```
+
+The connection uses `localhost:9092` by default and waits up to 10 seconds for a response.
+
+> Note: `KafkaSource` consumes in the `ingestor` group and, after a few consecutive empty polls (`max_empty_polls`), stops - that's why the demo terminates instead of waiting forever. Running again with the same group won't re-read already-read messages (the offset is saved). To re-consume everything, change the group in `src/ingestor/adapters/kafka/connection.py` or recreate the topic.
+
+## Use another address
+
+Set `KAFKA_BOOTSTRAP_SERVERS` when running:
+
+```bash
+KAFKA_BOOTSTRAP_SERVERS=server:9092 uv run ingestor
+```
+
+Replace `server:9092` with your broker's address. The current setup uses an unauthenticated connection; SASL/TLS clusters need adjustments in the adapter.
+
+## Run the tests (no Kafka)
+
+The tests use in-memory implementations (fakes) and a file source, so they **need neither Docker nor a broker**:
+
+```bash
+uv run python -m unittest discover -v
+```
+
+They exercise the same `ingest` function with a `FakeConnection` and with a `FileSource` (batch) - proving in practice that the logic does not depend on the Kafka lib.
+
+> Tip: when adding/moving files under `src/`, if `uv run` complains about a module not found, run once `uv sync --reinstall-package data-ingestor`.
+
+## Stop Kafka
+
+```bash
+docker compose stop
+```
+
+To start the same container again:
+
+```bash
+docker compose start
+```
+
+## If the connection fails
+
+Check that Docker is running and inspect the broker's state and logs:
+
+```bash
+docker compose ps
+docker compose logs --tail=50 kafka
+```
+
+If Kafka is still starting up, wait and run again. Also check the address and that port `9092` is free. On failure, the app prints the error and exits with code `1`.
+
+## Structure
+
+`src/` layout, with the code in an `ingestor` package organized as ports × adapters:
+
+```text
+ingestor/
+├── src/
+│   └── ingestor/
+│       ├── __init__.py
+│       ├── app.py              # use case: ingest(source, sink, limit)
+│       ├── cli.py              # Composition Root: picks and injects the adapters
+│       ├── __main__.py         # enables `python -m ingestor`
+│       ├── ports/              # interfaces + domain (tool-agnostic)
+│       │   ├── record.py       # Record (value, key, origin)
+│       │   ├── source.py       # Source: read() -> Iterator[Record]
+│       │   ├── sink.py         # Sink: write(record)
+│       │   └── connection.py   # Connection: source() / sink()
+│       └── adapters/           # concrete implementations (one folder per tool)
+│           ├── kafka/          # streaming: KafkaSource, KafkaSink, KafkaConnection, KafkaInfoCluster
+│           ├── console/        # generic destination: ConsoleSink
+│           └── batch/          # batch: FileSource (reads a file line by line)
+├── tests/
+│   ├── __init__.py
+│   ├── fakes.py                # ListSource, MemorySink, FakeConnection
+│   └── test_ingestor.py        # unittest tests
+├── docker-compose.yml          # local Kafka
+├── pyproject.toml              # dependencies, build and the `ingestor` command
+└── uv.lock
+```
+
+**How the layers relate:** `app.py` depends only on `ingestor.ports` (Source, Sink, Connection); the adapters in `adapters/` implement those ports; and only `cli.py` knows the concrete classes, wiring everything together. This way no specific lib (like `confluent_kafka`) "leaks" into the logic or the tests.
+
+**The three modes, the same logic:**
+
+| Mode | Source | Does `ingest` change? |
+|---|---|---|
+| Streaming | `KafkaSource` | no |
+| Batch | `FileSource` | no |
+| Micro-batch | any Source + `limit` | no |