pyintake 0.0.1__tar.gz

pyintake-0.0.1/.github/workflows/publish.yml

@@ -0,0 +1,70 @@
+# Build and publish to PyPI when a version tag (v*) is pushed.
+# Also supports manual run (workflow_dispatch).
+# Version is derived from the git tag by setuptools-scm.
+
+name: Build and Publish
+
+on:
+  push:
+    tags: ['v*']
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+  contents: write
+
+jobs:
+  build-and-publish:
+    name: Build and Publish to PyPI
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event_name == 'release' && github.event.release.tag_name || github.ref }}
+
+      - name: Set version from tag
+        if: startsWith(github.ref, 'refs/tags/')
+        run: echo "SETUPTOOLS_SCM_PRETEND_VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_ENV
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+          cache-dependency-path: pyproject.toml
+
+      - name: Install dependencies and run tests
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+          pytest
+
+      - name: Install build tools
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Verify package
+        run: |
+          python -m zipfile -l dist/pyintake-*.whl | head -30
+          echo "Package built successfully"
+
+      # Only publish to PyPI when building from a tag (official release). Skip for workflow_dispatch so we don't publish dev versions as pre-release.
+      - name: Publish to PyPI
+        if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'release'
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/
+
+      - name: Create GitHub Release
+        if: startsWith(github.ref, 'refs/tags/') && github.event_name != 'release'
+        uses: softprops/action-gh-release@v2.4.2
+        with:
+          tag_name: ${{ github.ref_name }}
+          name: ${{ github.ref_name }}
+          generate_release_notes: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

pyintake-0.0.1/.github/workflows/test.yml

@@ -0,0 +1,39 @@
+name: Test
+
+on:
+  pull_request:
+    branches: [main, develop]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.11', '3.12', '3.13']
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: pyproject.toml
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Run tests
+        run: pytest
+
+      - name: Run linting
+        run: |
+          ruff format --check src/pyintake tests
+          ruff check src/pyintake tests
+
+      - name: Run type checking
+        run: mypy src/pyintake || true

pyintake-0.0.1/Makefile

@@ -0,0 +1,110 @@
+.PHONY: help install install-dev test test-verbose test-cov test-fast test-unit test-integration clean build format lint type-check check publish-test publish docs docs-serve
+
+# Detect if venv exists and set PYTHON/PIP accordingly
+ifeq ($(wildcard venv/bin/python),)
+    PYTHON := python3
+    PIP := pip3
+    PYTHON_CMD := python3 -m
+else
+    PYTHON := venv/bin/python
+    PIP := venv/bin/pip
+    PYTHON_CMD := venv/bin/python -m
+endif
+
+help: ## Show this help message
+	@echo "Available commands:"
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-18s\033[0m %s\n", $$1, $$2}'
+	@echo ""
+	@echo "Publishing:"
+	@echo "  For production: Create a GitHub Release (automatically publishes via Trusted Publishing)"
+
+install: ## Install the package in development mode
+	python3 -m venv venv
+	. venv/bin/activate && pip install --upgrade pip && pip install -e .
+
+install-dev: ## Install package + all development dependencies
+	python3 -m venv venv
+	. venv/bin/activate && pip install --upgrade pip && pip install -e ".[dev]"
+	@echo "To activate: source venv/bin/activate"
+
+test: ## Run tests
+	$(PYTHON_CMD) pytest
+
+test-verbose: ## Run tests with verbose output
+	$(PYTHON_CMD) pytest -v
+
+test-cov: ## Run tests with coverage
+	$(PYTHON_CMD) pytest --cov=pyintake --cov-report=html --cov-report=term
+
+test-fast: ## Run fast tests only (exclude slow markers)
+	$(PYTHON_CMD) pytest -m "not slow"
+
+test-unit: ## Run unit tests only
+	$(PYTHON_CMD) pytest -m "unit"
+
+test-integration: ## Run integration tests only
+	$(PYTHON_CMD) pytest -m "integration"
+
+clean: ## Clean build artifacts
+	rm -rf build/ dist/ *.egg-info src/*.egg-info src/pyintake.egg-info
+	rm -rf .pytest_cache .mypy_cache .ruff_cache htmlcov/ .coverage
+	find . -type d -name __pycache__ -exec rm -r {} + 2>/dev/null || true
+	find . -type f -name "*.pyc" -delete 2>/dev/null || true
+
+build: ## Build distribution packages
+	@if ! $(PYTHON) -c "import build" 2>/dev/null; then \
+		echo "Installing build..."; \
+		$(PIP) install build; \
+	fi
+	$(PYTHON_CMD) build
+
+format: ## Format code with ruff
+	$(PYTHON_CMD) ruff format src/pyintake tests
+	$(PYTHON_CMD) ruff check --fix src/pyintake tests
+
+lint: ## Run linting checks (same as CI)
+	$(PYTHON_CMD) ruff format --check src/pyintake tests
+	$(PYTHON_CMD) ruff check src/pyintake tests
+
+type-check: ## Run type checking with mypy
+	$(PYTHON_CMD) mypy src/pyintake
+
+check: format lint type-check test ## Run all checks (format, lint, type-check, test)
+
+docs: ## Build documentation
+	$(PYTHON_CMD) mkdocs build
+
+docs-serve: ## Serve documentation locally
+	$(PYTHON_CMD) mkdocs serve
+
+publish-test: clean build ## Build and upload to TestPyPI (requires TWINE_PASSWORD)
+	@echo "Uploading to TestPyPI..."
+	@if ! $(PYTHON) -c "import twine" 2>/dev/null; then \
+		echo "Installing twine..."; \
+		$(PIP) install twine; \
+	fi
+	@if [ -z "$$TWINE_PASSWORD" ]; then \
+		echo "Warning: TWINE_PASSWORD not set. Set it with:"; \
+		echo "   export TWINE_PASSWORD=your-testpypi-token"; \
+		exit 1; \
+	fi
+	$(PYTHON_CMD) twine upload --repository testpypi dist/*
+	@echo "Published to TestPyPI"
+	@echo "Test install: pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pyintake"
+
+publish: clean build ## Build and upload to PyPI (use GitHub Releases for production)
+	@echo "WARNING: This will publish to PyPI!"
+	@echo "For production releases, use GitHub Releases instead (automatic via Trusted Publishing)"
+	@echo "Press Ctrl+C to cancel, or Enter to continue..."
+	@read dummy
+	@if ! $(PYTHON) -c "import twine" 2>/dev/null; then \
+		echo "Installing twine..."; \
+		$(PIP) install twine; \
+	fi
+	@if [ -z "$$TWINE_PASSWORD" ]; then \
+		echo "Warning: TWINE_PASSWORD not set. Set it with:"; \
+		echo "   export TWINE_PASSWORD=your-pypi-token"; \
+		exit 1; \
+	fi
+	$(PYTHON_CMD) twine upload dist/*
+	@echo "Published to PyPI"

pyintake-0.0.1/PKG-INFO

@@ -0,0 +1,47 @@
+Metadata-Version: 2.4
+Name: pyintake
+Version: 0.0.1
+Summary: Transport-agnostic data ingestion system with priority buffering and backpressure
+Author-email: StatFYI <contact@statfyi.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/optophi/pyintake
+Project-URL: Repository, https://github.com/optophi/pyintake
+Project-URL: Issues, https://github.com/optophi/pyintake/issues
+Keywords: ingestion,data-pipeline,buffering,backpressure,event-driven
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Typing :: Typed
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Provides-Extra: http
+Requires-Dist: fastapi>=0.104.0; extra == "http"
+Requires-Dist: uvicorn[standard]>=0.24.0; extra == "http"
+Requires-Dist: pydantic>=2.0.0; extra == "http"
+Provides-Extra: kafka
+Requires-Dist: aiokafka>=0.10.0; extra == "kafka"
+Provides-Extra: websocket
+Requires-Dist: websockets>=12.0; extra == "websocket"
+Provides-Extra: api
+Requires-Dist: pyintake[http]; extra == "api"
+Requires-Dist: pydantic-settings>=2.0.0; extra == "api"
+Provides-Extra: config
+Requires-Dist: pyyaml>=6.0.0; extra == "config"
+Requires-Dist: pydantic>=2.0.0; extra == "config"
+Provides-Extra: all
+Requires-Dist: pyintake[api,config,kafka,websocket]; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pyintake[all]; extra == "dev"
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: ruff>=0.9.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: pre-commit>=3.0.0; extra == "dev"
+Requires-Dist: httpx>=0.24.0; extra == "dev"
+Requires-Dist: build>=0.10.0; extra == "dev"

pyintake-0.0.1/pyproject.toml

@@ -0,0 +1,176 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel", "setuptools-scm>=8.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pyintake"
+dynamic = ["version"]
+description = "Transport-agnostic data ingestion system with priority buffering and backpressure"
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+dependencies = []
+authors = [
+    { name = "StatFYI", email = "contact@statfyi.com" },
+]
+keywords = [
+    "ingestion",
+    "data-pipeline",
+    "buffering",
+    "backpressure",
+    "event-driven",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Typing :: Typed",
+    "Topic :: Software Development :: Libraries :: Application Frameworks",
+]
+
+[project.urls]
+Homepage = "https://github.com/optophi/pyintake"
+Repository = "https://github.com/optophi/pyintake"
+Issues = "https://github.com/optophi/pyintake/issues"
+
+[tool.setuptools_scm]
+local_scheme = "no-local-version"
+
+[project.optional-dependencies]
+# ---------------------------------------------------------------------------
+# Install extras:
+#   pip install pyintake              -> core library only (zero deps)
+#   pip install pyintake[http]        -> + FastAPI HTTP adapter
+#   pip install pyintake[kafka]       -> + aiokafka Kafka adapter
+#   pip install pyintake[websocket]   -> + websockets adapter
+#   pip install pyintake[api]         -> + REST API server
+#   pip install pyintake[all]         -> api + kafka + websocket
+#   pip install pyintake[dev]         -> all + testing & dev tools
+# ---------------------------------------------------------------------------
+
+http = [
+    "fastapi>=0.104.0",
+    "uvicorn[standard]>=0.24.0",
+    "pydantic>=2.0.0",
+]
+kafka = [
+    "aiokafka>=0.10.0",
+]
+websocket = [
+    "websockets>=12.0",
+]
+api = [
+    "pyintake[http]",
+    "pydantic-settings>=2.0.0",
+]
+config = [
+    "pyyaml>=6.0.0",
+    "pydantic>=2.0.0",
+]
+all = [
+    "pyintake[api,kafka,websocket,config]",
+]
+dev = [
+    "pyintake[all]",
+    "pytest>=7.0.0",
+    "pytest-asyncio>=0.21.0",
+    "pytest-cov>=4.0.0",
+    "ruff>=0.9.0",
+    "mypy>=1.0.0",
+    "pre-commit>=3.0.0",
+    "httpx>=0.24.0",
+    "build>=0.10.0",
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["pyintake*"]
+
+[tool.setuptools.package-data]
+pyintake = [
+    "py.typed",
+    "data/seed/*.yaml",
+]
+
+[project.scripts]
+pyintake = "pyintake.cli:main"
+
+# =============================================================================
+# Tool Configuration
+# =============================================================================
+
+[tool.ruff]
+target-version = "py311"
+line-length = 88
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "W", "F", "I", "B", "C4", "UP"]
+ignore = [
+    "E501",
+    "B008",
+    "B904",
+    "B028",
+    "B007",
+    "B017",
+    "E402",
+    "E731",
+    "B027",
+    "B905",
+]
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]
+"tests/**" = ["B904", "B017", "E402"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["pyintake"]
+
+[tool.ruff.format]
+quote-style = "double"
+
+[tool.mypy]
+python_version = "3.11"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = false
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+minversion = "7.0"
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+addopts = [
+    "--strict-markers",
+    "--tb=short",
+]
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "integration: marks tests as integration tests",
+    "unit: marks tests as unit tests",
+    "asyncio: mark test as async (pytest-asyncio)",
+]
+filterwarnings = [
+    "error",
+    "ignore::UserWarning",
+]
+
+[tool.coverage.run]
+source = ["pyintake"]
+
+[tool.coverage.report]
+show_missing = true
+skip_empty = true
+exclude_lines = [
+    "pragma: no cover",
+    "if TYPE_CHECKING:",
+    "if __name__ == .__main__.",
+]

pyintake-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

pyintake-0.0.1/src/pyintake/__init__.py

@@ -0,0 +1,83 @@
+"""PyIntake — Transport-agnostic data ingestion system.
+
+Collects, validates, deduplicates, and buffers events from any transport,
+then drains them into pluggable downstream consumers with priority ordering
+and backpressure signaling.
+"""
+
+from __future__ import annotations
+
+from pyintake._protocols import Consumer, MetricsCollector, TransportAdapter, Validator
+from pyintake._types import (
+    BackpressureSignal,
+    BufferEntry,
+    ConsumeResult,
+    IngestResult,
+    Priority,
+    ValidationOutcome,
+)
+from pyintake.buffer._dedup import SlidingWindowDedup
+from pyintake.buffer._priority_buffer import PriorityBuffer
+from pyintake.core._drain import AdaptiveDrainLoop
+from pyintake.core._engine import IngestionEngine
+from pyintake.core._rate_limiter import TokenBucketRateLimiter
+from pyintake.errors import (
+    AdapterError,
+    BufferFullError,
+    ConfigError,
+    ConsumerError,
+    DrainError,
+    DuplicateEventError,
+    ErrorCode,
+    PyIntakeError,
+    RateLimitedError,
+    ShutdownTimeoutError,
+    ValidationError,
+)
+from pyintake.event import IngestEvent
+from pyintake.metrics._collector import InMemoryMetricsCollector
+from pyintake.metrics._models import MetricsSnapshot, SourceMetrics
+from pyintake.validation._chain import ValidationChain
+
+__all__ = [
+    # Core event type
+    "IngestEvent",
+    # Value types and enums
+    "Priority",
+    "BackpressureSignal",
+    "IngestResult",
+    "ValidationOutcome",
+    "ConsumeResult",
+    "BufferEntry",
+    # Protocols
+    "TransportAdapter",
+    "Validator",
+    "Consumer",
+    "MetricsCollector",
+    # Engine and drain
+    "IngestionEngine",
+    "AdaptiveDrainLoop",
+    # Buffer
+    "PriorityBuffer",
+    "SlidingWindowDedup",
+    # Validation
+    "ValidationChain",
+    # Rate limiting
+    "TokenBucketRateLimiter",
+    # Metrics
+    "InMemoryMetricsCollector",
+    "MetricsSnapshot",
+    "SourceMetrics",
+    # Errors
+    "PyIntakeError",
+    "ErrorCode",
+    "BufferFullError",
+    "ValidationError",
+    "DuplicateEventError",
+    "RateLimitedError",
+    "AdapterError",
+    "DrainError",
+    "ConfigError",
+    "ConsumerError",
+    "ShutdownTimeoutError",
+]

pyintake-0.0.1/src/pyintake/_protocols.py

@@ -0,0 +1,145 @@
+"""Protocol definitions for PyIntake pluggable interfaces.
+
+Protocols define the contracts for transport adapters, validators,
+consumers, and metrics collectors. Implementations can be swapped
+in via dependency injection without coupling to concrete classes.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
+
+if TYPE_CHECKING:
+    from pyintake._types import ConsumeResult, ValidationOutcome
+    from pyintake.event import IngestEvent
+    from pyintake.metrics._models import MetricsSnapshot
+
+
+# ---------------------------------------------------------------------------
+# Transport adapter protocol
+# ---------------------------------------------------------------------------
+
+
+@runtime_checkable
+class TransportAdapter(Protocol):
+    """Receives events from a transport and feeds them into IngestionEngine.
+
+    Implementations handle transport-specific concerns (Kafka consumer,
+    HTTP server, file watcher, etc.) and call engine.ingest() for each event.
+    """
+
+    async def start(self, engine: Any) -> None:
+        """Start the adapter and begin feeding events to the engine.
+
+        Args:
+            engine: The IngestionEngine to feed events into.
+        """
+        ...
+
+    async def stop(self) -> None:
+        """Stop the adapter and release transport resources."""
+        ...
+
+
+# ---------------------------------------------------------------------------
+# Validator protocol
+# ---------------------------------------------------------------------------
+
+
+@runtime_checkable
+class Validator(Protocol):
+    """Validates an IngestEvent before it enters the buffer.
+
+    Validators run in the ingestion path (before buffering), so they
+    must be fast and synchronous. Heavy validation should be done
+    downstream by consumers.
+    """
+
+    @property
+    def name(self) -> str:
+        """Human-readable name for metrics and logging."""
+        ...
+
+    def validate(self, event: IngestEvent) -> ValidationOutcome:
+        """Validate a single event.
+
+        Args:
+            event: The event to validate.
+
+        Returns:
+            Outcome indicating whether the event is valid.
+        """
+        ...
+
+
+# ---------------------------------------------------------------------------
+# Consumer protocol
+# ---------------------------------------------------------------------------
+
+
+@runtime_checkable
+class Consumer(Protocol):
+    """Downstream consumer that receives drained event batches.
+
+    Consumers are the output side of the ingestion pipeline. They
+    receive batches of validated, deduplicated events from the
+    drain loop. Examples: pystator orchestrator, pycharter validator,
+    file collector for offline analysis.
+    """
+
+    async def consume(self, batch: list[IngestEvent]) -> ConsumeResult:
+        """Process a batch of events.
+
+        Args:
+            batch: List of events drained from the buffer.
+
+        Returns:
+            Result indicating success/failure and events processed.
+        """
+        ...
+
+    async def close(self) -> None:
+        """Release any resources held by the consumer."""
+        ...
+
+
+# ---------------------------------------------------------------------------
+# Metrics collector protocol
+# ---------------------------------------------------------------------------
+
+
+@runtime_checkable
+class MetricsCollector(Protocol):
+    """Collects ingestion metrics programmatically.
+
+    All methods are synchronous and must be fast — they are called
+    in the hot ingestion path.
+    """
+
+    def record_accepted(self, event: IngestEvent) -> None:
+        """Record a successfully accepted event."""
+        ...
+
+    def record_rejected(self, event: IngestEvent, reason: str) -> None:
+        """Record a rejected event with the rejection reason."""
+        ...
+
+    def record_drained(self, count: int, sources: list[str] | None = None) -> None:
+        """Record that count events were drained to consumers."""
+        ...
+
+    def record_deduplicated(self, event: IngestEvent) -> None:
+        """Record a deduplicated (duplicate-rejected) event."""
+        ...
+
+    def record_shed(self, event: IngestEvent) -> None:
+        """Record an event shed due to buffer pressure."""
+        ...
+
+    def record_dead_letter(self, event: IngestEvent) -> None:
+        """Record an event that exceeded max retries."""
+        ...
+
+    def snapshot(self) -> MetricsSnapshot:
+        """Return a frozen point-in-time snapshot of all metrics."""
+        ...