etlplus 0.3.16__tar.gz → 0.3.19__tar.gz

{etlplus-0.3.16/etlplus.egg-info → etlplus-0.3.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.3.16
+Version: 0.3.19
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team

{etlplus-0.3.16 → etlplus-0.3.19/etlplus.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.3.16
+Version: 0.3.19
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team

{etlplus-0.3.16 → etlplus-0.3.19}/etlplus.egg-info/SOURCES.txt

@@ -87,9 +87,13 @@ tests/integration/test_i_run_profile_pagination_defaults.py
 tests/integration/test_i_run_profile_rate_limit_defaults.py
 tests/unit/conftest.py
 tests/unit/test_u_cli.py
+tests/unit/test_u_enums.py
 tests/unit/test_u_extract.py
 tests/unit/test_u_file.py
 tests/unit/test_u_load.py
+tests/unit/test_u_mixins.py
+tests/unit/test_u_run.py
+tests/unit/test_u_run_helpers.py
 tests/unit/test_u_transform.py
 tests/unit/test_u_utils.py
 tests/unit/test_u_validate.py
@@ -106,6 +110,7 @@ tests/unit/api/test_u_rate_limiter.py
 tests/unit/api/test_u_request_manager.py
 tests/unit/api/test_u_retry_manager.py
 tests/unit/api/test_u_transport.py
+tests/unit/api/test_u_types.py
 tests/unit/config/test_u_connector.py
 tests/unit/config/test_u_pipeline.py
 tests/unit/validation/test_u_validation_utils.py

etlplus-0.3.19/tests/conftest.py

@@ -0,0 +1,210 @@
+"""
+:mod:`tests.conftest` module.
+
+Global pytest fixtures shared across unit, integration, and end-to-end tests.
+
+Notes
+-----
+- Provides CLI helpers so tests no longer need to monkeypatch ``sys.argv``
+    inline.
+- Supplies JSON file factories that rely on ``tmp_path`` for automatic
+    cleanup.
+- Keeps docstrings NumPy-formatted to satisfy numpydoc linting.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+from typing import Protocol
+
+import pytest
+from requests import PreparedRequest  # type: ignore[import]
+
+from etlplus.cli import main
+
+# SECTION: HELPERS ========================================================== #
+
+
+def _coerce_cli_args(
+    cli_args: tuple[str | Sequence[str], ...],
+) -> tuple[str, ...]:
+    """
+    Normalize CLI arguments into a ``tuple[str, ...]``.
+
+    Parameters
+    ----------
+    cli_args : tuple[str | Sequence[str], ...]
+        Arguments provided to ``cli_runner``/``cli_invoke``.
+
+    Returns
+    -------
+    tuple[str, ...]
+        Normalized argument tuple safe to concatenate with ``sys.argv``.
+    """
+    if (
+        len(cli_args) == 1
+        and isinstance(cli_args[0], Sequence)
+        and not isinstance(cli_args[0], (str, bytes))
+    ):
+        return tuple(str(part) for part in cli_args[0])
+    return tuple(str(part) for part in cli_args)
+
+
+class CliInvoke(Protocol):
+    """Protocol describing the :func:`cli_invoke` fixture."""
+
+    def __call__(
+        self,
+        *cli_args: str | Sequence[str],
+    ) -> tuple[int, str, str]: ...
+
+
+class CliRunner(Protocol):
+    """Protocol describing the ``cli_runner`` fixture."""
+
+    def __call__(self, *cli_args: str | Sequence[str]) -> int: ...
+
+
+class JsonFactory(Protocol):
+    """Protocol describing the :func:`json_file_factory` fixture."""
+
+    def __call__(
+        self,
+        payload: Any,
+        *,
+        filename: str | None = None,
+        ensure_ascii: bool = False,
+    ) -> Path: ...
+
+
+class RequestFactory(Protocol):
+    """Protocol describing prepared-request factories."""
+
+    def __call__(
+        self,
+        url: str | None = None,
+    ) -> PreparedRequest: ...
+
+
+# SECTION: FIXTURES ========================================================= #
+
+
+@pytest.fixture(name='base_url')
+def base_url_fixture() -> str:
+    """Return the canonical base URL shared across tests."""
+
+    return 'https://api.example.com'
+
+
+@pytest.fixture(name='json_file_factory')
+def json_file_factory_fixture(
+    tmp_path: Path,
+) -> JsonFactory:
+    """
+    Create JSON files under ``tmp_path`` and return their paths.
+
+    Parameters
+    ----------
+    tmp_path : Path
+        Temporary directory managed by pytest.
+
+    Returns
+    -------
+    JsonFactory
+        Factory that persists the provided payload as JSON and returns the
+        resulting path.
+
+    Examples
+    --------
+    >>> path = json_file_factory({'name': 'Ada'})
+    >>> path.exists()
+    True
+    """
+
+    def _create(
+        payload: Any,
+        *,
+        filename: str | None = None,
+        ensure_ascii: bool = False,
+    ) -> Path:
+        target = tmp_path / (filename or 'payload.json')
+        data = (
+            payload
+            if isinstance(payload, str)
+            else json.dumps(payload, indent=2, ensure_ascii=ensure_ascii)
+        )
+        target.write_text(data)
+        return target
+
+    return _create
+
+
+@pytest.fixture(name='cli_runner')
+def cli_runner_fixture(
+    monkeypatch: pytest.MonkeyPatch,
+) -> CliRunner:
+    """
+    Invoke ``etlplus`` CLI commands with isolated ``sys.argv`` state.
+
+    Parameters
+    ----------
+    monkeypatch : pytest.MonkeyPatch
+        Built-in pytest fixture used to patch ``sys.argv``.
+
+    Returns
+    -------
+    CliRunner
+        Helper that accepts CLI arguments, runs :func:`etlplus.cli.main`, and
+        returns the exit code.
+
+    Examples
+    --------
+    >>> cli_runner(('extract', 'file', 'data.json'))
+    0
+    """
+
+    def _run(*cli_args: str | Sequence[str]) -> int:
+        args = _coerce_cli_args(cli_args)
+        monkeypatch.setattr(sys, 'argv', ['etlplus', *args])
+        return main()
+
+    return _run
+
+
+@pytest.fixture
+def cli_invoke(
+    cli_runner: CliRunner,
+    capsys: pytest.CaptureFixture[str],
+) -> CliInvoke:
+    """
+    Run CLI commands and return exit code, stdout, and stderr.
+
+    Parameters
+    ----------
+    cli_runner : CliRunner
+        Helper fixture defined above.
+    capsys : pytest.CaptureFixture[str]
+        Pytest fixture for capturing stdout/stderr.
+
+    Returns
+    -------
+    CliInvoke
+        Helper that yields ``(exit_code, stdout, stderr)`` tuples.
+
+    Examples
+    --------
+    >>> code, out, err = cli_invoke(('extract', 'file', 'data.json'))
+    >>> code
+    0
+    """
+
+    def _invoke(*cli_args: str | Sequence[str]) -> tuple[int, str, str]:
+        exit_code = cli_runner(*cli_args)
+        captured = capsys.readouterr()
+        return exit_code, captured.out, captured.err
+
+    return _invoke

{etlplus-0.3.16 → etlplus-0.3.19}/tests/integration/conftest.py

@@ -11,6 +11,7 @@ Notes
 from __future__ import annotations
 
 import importlib
+import json
 import pathlib
 from collections.abc import Callable
 from typing import Any
@@ -34,6 +35,12 @@ from etlplus.config import PipelineConfig
 # SECTION: HELPERS ========================================================== #
 
 
+pytestmark = pytest.mark.integration
+
+
+# SECTION: HELPERS ========================================================== #
+
+
 # Directory-level marker for integration tests.
 pytestmark = pytest.mark.integration
 
@@ -55,8 +62,8 @@ class FakeEndpointClientProtocol(Protocol):
 # SECTION: FIXTURES ========================================================= #
 
 
-@pytest.fixture
-def capture_load_to_api(
+@pytest.fixture(name='capture_load_to_api')
+def capture_load_to_api_fixture(
     monkeypatch: pytest.MonkeyPatch,
 ) -> dict[str, Any]:
     """
@@ -112,8 +119,8 @@ def capture_load_to_api(
     return seen
 
 
-@pytest.fixture
-def fake_endpoint_client() -> tuple[
+@pytest.fixture(name='fake_endpoint_client')
+def fake_endpoint_client_fixture() -> tuple[
     type[FakeEndpointClientProtocol],
     list[FakeEndpointClientProtocol],
 ]:  # noqa: ANN201
@@ -170,9 +177,83 @@ def fake_endpoint_client() -> tuple[
     return FakeClient, created
 
 
-@pytest.fixture
-def pipeline_cfg_factory(
+@pytest.fixture(name='file_to_api_pipeline_factory')
+def file_to_api_pipeline_factory_fixture(
+    tmp_path: pathlib.Path,
+    base_url: str,
+) -> Callable[..., PipelineConfig]:
+    """Build a pipeline wiring a JSON file source to an API target."""
+
+    def _make(
+        *,
+        payload: Any | None = None,
+        base_url: str = base_url,
+        base_path: str | None = '/v1',
+        endpoint_path: str = '/ingest',
+        endpoint_name: str = 'ingest',
+        method: str = 'post',
+        headers: dict[str, str] | None = None,
+        job_name: str = 'send',
+        target_name: str = 'ingest_out',
+    ) -> PipelineConfig:
+        source_path = tmp_path / f'{job_name}_input.json'
+        effective_payload = payload if payload is not None else {'ok': True}
+        text = (
+            effective_payload
+            if isinstance(effective_payload, str)
+            else json.dumps(effective_payload)
+        )
+        source_path.write_text(text, encoding='utf-8')
+
+        profile = ApiProfileConfig(
+            base_url=base_url,
+            headers={},
+            base_path=base_path or '',
+            auth={},
+            rate_limit_defaults=None,
+            pagination_defaults=None,
+        )
+        api = ApiConfig(
+            base_url=base_url,
+            profiles={'default': profile},
+            endpoints={endpoint_name: EndpointConfig(path=endpoint_path)},
+        )
+
+        src = ConnectorFile(
+            name='file_src',
+            type='file',
+            format='json',
+            path=str(source_path),
+        )
+        tgt = ConnectorApi(
+            name=target_name,
+            type='api',
+            api='svc',
+            endpoint=endpoint_name,
+            method=method,
+            headers=headers or {},
+        )
+
+        return PipelineConfig(
+            apis={'svc': api},
+            sources=[src],
+            targets=[tgt],
+            jobs=[
+                JobConfig(
+                    name=job_name,
+                    extract=ExtractRef(source='file_src'),
+                    load=LoadRef(target=target_name),
+                ),
+            ],
+        )
+
+    return _make
+
+
+@pytest.fixture(name='pipeline_cfg_factory')
+def pipeline_cfg_factory_fixture(
     tmp_path: pathlib.Path,
+    base_url: str,
 ) -> Callable[..., PipelineConfig]:
     """
     Factory to build a minimal PipelineConfig for runner tests.
@@ -185,6 +266,8 @@ def pipeline_cfg_factory(
     ----------
     tmp_path : pathlib.Path
         The pytest temporary path fixture.
+    base_url : str
+        Common base URL used across tests.
 
     Returns
     -------
@@ -196,9 +279,10 @@ def pipeline_cfg_factory(
         *,
         pagination_defaults: PaginationConfig | None = None,
         rate_limit_defaults: RateLimitConfig | None = None,
+        extract_options: dict[str, Any] | None = None,
     ) -> PipelineConfig:
         prof = ApiProfileConfig(
-            base_url='https://api.example.com',
+            base_url=base_url,
             headers={},
             base_path='/v1',
             auth={},
@@ -219,24 +303,29 @@ def pipeline_cfg_factory(
             format='json',
             path=str(out_path),
         )
+        job = JobConfig(
+            name='job',
+            extract=ExtractRef(source='s'),
+            load=LoadRef(target='t'),
+        )
+        if extract_options is not None:
+            if job.extract is None:
+                msg = 'job.extract is None; cannot set options'
+                raise ValueError(msg)
+            job.extract.options = extract_options
+
         return PipelineConfig(
             apis={'svc': api},
             sources=[src],
             targets=[tgt],
-            jobs=[
-                JobConfig(
-                    name='job',
-                    extract=ExtractRef(source='s'),
-                    load=LoadRef(target='t'),
-                ),
-            ],
+            jobs=[job],
         )
 
     return _make
 
 
-@pytest.fixture
-def run_patched(
+@pytest.fixture(name='run_patched')
+def run_patched_fixture(
     monkeypatch: pytest.MonkeyPatch,
 ) -> Callable[..., dict[str, Any]]:
     """

etlplus-0.3.19/tests/integration/test_i_cli.py

@@ -0,0 +1,244 @@
+"""
+:mod:`tests.integration.test_i_cli` module.
+
+End-to-end CLI integration test suite that exercises the ``etlplus`` command
+without external dependencies. Tests rely on shared fixtures for CLI
+invocation and filesystem management to maximize reuse.
+
+Notes
+-----
+- Uses ``cli_invoke``/``cli_runner`` fixtures to avoid ad-hoc monkeypatching.
+- Creates JSON files through ``json_file_factory`` for deterministic cleanup.
+- Keeps docstrings NumPy-compliant for automated linting.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:  # pragma: no cover - typing helpers only
+    from tests.conftest import CliInvoke
+    from tests.conftest import JsonFactory
+
+
+# SECTION: HELPERS ========================================================== #
+
+
+pytestmark = pytest.mark.integration
+
+
+# SECTION: TESTS ============================================================ #
+
+
+class TestCliEndToEnd:
+    """Integration test suite for :mod:`etlplus.cli`."""
+
+    @pytest.mark.parametrize(
+        ('extra_flags', 'expected_code', 'expected_message'),
+        [
+            pytest.param(
+                ['--strict-format'],
+                1,
+                'Error:',
+                id='strict-errors',
+            ),
+            pytest.param(
+                [],
+                0,
+                'Warning:',
+                id='warns-default',
+            ),
+        ],
+    )
+    def test_extract_format_feedback(
+        self,
+        json_file_factory: JsonFactory,
+        cli_invoke: CliInvoke,
+        extra_flags: list[str],
+        expected_code: int,
+        expected_message: str,
+    ) -> None:
+        """Verify ``extract`` error/warning flow with optional strict flag."""
+        source = json_file_factory({'x': 1}, filename='payload.json')
+        args: list[str] = [
+            'extract',
+            'file',
+            str(source),
+            '--format',
+            'json',
+            *extra_flags,
+        ]
+        code, _out, err = cli_invoke(args)
+        assert code == expected_code
+        assert expected_message in err
+
+    @pytest.mark.parametrize(
+        (
+            'extra_flags',
+            'expected_code',
+            'expected_message',
+            'expect_output',
+        ),
+        [
+            pytest.param(
+                ['--strict-format'],
+                1,
+                'Error:',
+                False,
+                id='strict-errors',
+            ),
+            pytest.param(
+                [],
+                0,
+                'Warning:',
+                True,
+                id='warns-default',
+            ),
+        ],
+    )
+    def test_load_format_feedback(
+        self,
+        tmp_path: Path,
+        cli_invoke: CliInvoke,
+        extra_flags: list[str],
+        expected_code: int,
+        expected_message: str,
+        expect_output: bool,
+    ) -> None:
+        """
+        Validate ``load`` warnings/errors and resulting output file state.
+        """
+        output_path = tmp_path / 'output.csv'
+        args: list[str] = [
+            'load',
+            '{"name": "John"}',
+            'file',
+            str(output_path),
+            '--format',
+            'csv',
+            *extra_flags,
+        ]
+        code, _out, err = cli_invoke(args)
+        assert code == expected_code
+        assert expected_message in err
+        assert output_path.exists() is expect_output
+
+    def test_main_no_command(self, cli_invoke: CliInvoke) -> None:
+        """Test that running :func:`main` with no command shows usage."""
+        code, out, _err = cli_invoke()
+        assert code == 0
+        assert 'usage:' in out.lower()
+
+    def test_main_extract_file(
+        self,
+        json_file_factory: JsonFactory,
+        cli_invoke: CliInvoke,
+    ) -> None:
+        """Test that ``extract file`` prints the serialized payload."""
+        payload = {'name': 'John', 'age': 30}
+        source = json_file_factory(payload, filename='input.json')
+        code, out, _err = cli_invoke(('extract', 'file', str(source)))
+        assert code == 0
+        assert json.loads(out) == payload
+
+    def test_main_validate_data(
+        self,
+        cli_invoke: CliInvoke,
+    ) -> None:
+        """
+        Test that running :func:`main` with the ``validate`` command works.
+        """
+        json_data = '{"name": "John", "age": 30}'
+        code, out, _err = cli_invoke(('validate', json_data))
+        assert code == 0
+        assert json.loads(out)['valid'] is True
+
+    def test_main_transform_data(
+        self,
+        cli_invoke: CliInvoke,
+    ) -> None:
+        """
+        Test that running :func:`main` with the ``transform`` command works.
+        """
+        json_data = '[{"name": "John", "age": 30}]'
+        operations = '{"select": ["name"]}'
+        code, out, _err = cli_invoke(
+            ('transform', json_data, '--operations', operations),
+        )
+        assert code == 0
+        output = json.loads(out)
+        assert len(output) == 1 and 'age' not in output[0]
+
+    def test_main_load_file(
+        self,
+        tmp_path: Path,
+        cli_invoke: CliInvoke,
+    ) -> None:
+        """
+        Test that running :func:`main` with the ``load`` file command works.
+        """
+        output_path = tmp_path / 'output.json'
+        json_data = '{"name": "John", "age": 30}'
+        code, _out, _err = cli_invoke(
+            ('load', json_data, 'file', str(output_path)),
+        )
+        assert code == 0
+        assert output_path.exists()
+
+    def test_main_extract_with_output(
+        self,
+        tmp_path: Path,
+        json_file_factory: JsonFactory,
+        cli_invoke: CliInvoke,
+    ) -> None:
+        """Test extract command with ``-o`` output persistence."""
+        test_data = {'name': 'John', 'age': 30}
+        source = json_file_factory(test_data, filename='input.json')
+        output_path = tmp_path / 'output.json'
+        code, _out, _err = cli_invoke(
+            (
+                'extract',
+                'file',
+                str(source),
+                '-o',
+                str(output_path),
+            ),
+        )
+        assert code == 0
+        assert output_path.exists()
+        assert json.loads(output_path.read_text()) == test_data
+
+    def test_main_error_handling(
+        self,
+        cli_invoke: CliInvoke,
+    ) -> None:
+        """Test that running :func:`main` with an invalid command errors."""
+        code, _out, err = cli_invoke(
+            ('extract', 'file', '/nonexistent/file.json'),
+        )
+        assert code == 1
+        assert 'Error:' in err
+
+    def test_main_strict_format_error(
+        self,
+        cli_invoke: CliInvoke,
+    ) -> None:
+        """
+        Test ``extract`` with ``--strict-format`` rejects mismatched args.
+        """
+        code, _out, err = cli_invoke(
+            (
+                'extract',
+                'file',
+                'data.csv',
+                '--format',
+                'csv',
+                '--strict-format',
+            ),
+        )
+        assert code == 1
+        assert 'Error:' in err

{etlplus-0.3.16 → etlplus-0.3.19}/tests/integration/test_i_examples_data_parity.py

@@ -13,11 +13,16 @@ Notes
 from pathlib import Path
 from typing import Any
 
+import pytest
+
 from etlplus.file import File
 
 # SECTION: HELPERS ========================================================== #
 
 
+pytestmark = pytest.mark.integration
+
+
 def _norm_record(
     rec: dict[str, Any],
 ) -> dict[str, Any]: