etlplus 0.12.10__py3-none-any.whl → 0.14.3__py3-none-any.whl

etlplus/{load.py → ops/load.py}

@@ -1,5 +1,5 @@
 """
-:mod:`etlplus.load` module.
+:mod:`etlplus.ops.load` module.
 
 Helpers to load data into files, databases, and REST APIs.
 """
@@ -12,17 +12,16 @@ from pathlib import Path
 from typing import Any
 from typing import cast
 
-import requests  # type: ignore[import]
-
-from .enums import DataConnectorType
-from .enums import HttpMethod
-from .file import File
-from .file import FileFormat
-from .types import JSONData
-from .types import JSONDict
-from .types import JSONList
-from .types import StrPath
-from .utils import count_records
+from ..api import HttpMethod
+from ..api.utils import resolve_request
+from ..enums import DataConnectorType
+from ..file import File
+from ..file import FileFormat
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import JSONList
+from ..types import StrPath
+from ..utils import count_records
 
 # SECTION: INTERNAL FUNCTIONS ============================================== #
 
@@ -69,7 +68,7 @@ def _parse_json_string(
 # SECTION: FUNCTIONS ======================================================== #
 
 
-# -- Data Loading -- #
+# -- Helpers -- #
 
 
 def load_data(
@@ -119,58 +118,59 @@ def load_data(
     )
 
 
-# -- File Loading -- #
-
-
-def load_to_file(
+def load_to_api(
     data: JSONData,
-    file_path: StrPath,
-    file_format: FileFormat | str | None = None,
+    url: str,
+    method: HttpMethod | str,
+    **kwargs: Any,
 ) -> JSONDict:
     """
-    Persist data to a local file.
+    Load data to a REST API.
 
     Parameters
     ----------
     data : JSONData
-        Data to write.
-    file_path : StrPath
-        Target file path.
-    file_format : FileFormat | str | None, optional
-        Output format. If omitted (None), the format is inferred from the
-        filename extension.
+        Data to send as JSON.
+    url : str
+        API endpoint URL.
+    method : HttpMethod | str
+        HTTP method to use.
+    **kwargs : Any
+        Extra arguments forwarded to ``requests`` (e.g., ``timeout``).
+        When omitted, ``timeout`` defaults to 10 seconds.
 
     Returns
     -------
     JSONDict
-        Result dictionary with status and record count.
+        Result dictionary including response payload or text.
     """
-    path = Path(file_path)
-    path.parent.mkdir(parents=True, exist_ok=True)
+    # Apply a conservative timeout to guard against hanging requests.
+    timeout = kwargs.pop('timeout', 10.0)
+    session = kwargs.pop('session', None)
+    request_callable, timeout, http_method = resolve_request(
+        method,
+        session=session,
+        timeout=timeout,
+    )
+    response = request_callable(url, json=data, timeout=timeout, **kwargs)
+    response.raise_for_status()
 
-    # If no explicit format is provided, let File infer from extension.
-    if file_format is None:
-        records = File(path).write(data)
-        ext = path.suffix.lstrip('.').lower()
-        fmt = FileFormat.coerce(ext) if ext else FileFormat.JSON
-    else:
-        fmt = FileFormat.coerce(file_format)
-        records = File(path, fmt).write(data)
-    if fmt is FileFormat.CSV and records == 0:
-        message = 'No data to write'
-    else:
-        message = f'Data loaded to {path}'
+    # Try JSON first, fall back to text.
+    try:
+        payload: Any = response.json()
+    except ValueError:
+        payload = response.text
 
     return {
         'status': 'success',
-        'message': message,
-        'records': records,
+        'status_code': response.status_code,
+        'message': f'Data loaded to {url}',
+        'response': payload,
+        'records': count_records(data),
+        'method': http_method.value.upper(),
     }
 
 
-# -- Database Loading (Placeholder) -- #
-
-
 def load_to_database(
     data: JSONData,
     connection_string: str,
@@ -206,69 +206,49 @@ def load_to_database(
     }
 
 
-# -- REST API Loading -- #
-
-
-def load_to_api(
+def load_to_file(
     data: JSONData,
-    url: str,
-    method: HttpMethod | str,
-    **kwargs: Any,
+    file_path: StrPath,
+    file_format: FileFormat | str | None = None,
 ) -> JSONDict:
     """
-    Load data to a REST API.
+    Persist data to a local file.
 
     Parameters
     ----------
     data : JSONData
-        Data to send as JSON.
-    url : str
-        API endpoint URL.
-    method : HttpMethod | str
-        HTTP method to use.
-    **kwargs : Any
-        Extra arguments forwarded to ``requests`` (e.g., ``timeout``).
+        Data to write.
+    file_path : StrPath
+        Target file path.
+    file_format : FileFormat | str | None, optional
+        Output format. If omitted (None), the format is inferred from the
+        filename extension.
 
     Returns
     -------
     JSONDict
-        Result dictionary including response payload or text.
-
-    Raises
-    ------
-    TypeError
-        If the session object is not valid.
+        Result dictionary with status and record count.
     """
-    http_method = HttpMethod.coerce(method)
-
-    # Apply a conservative timeout to guard against hanging requests.
-    timeout = kwargs.pop('timeout', 10.0)
-    session = kwargs.pop('session', None)
-    requester = session or requests
-
-    request_callable = getattr(requester, http_method.value, None)
-    if not callable(request_callable):
-        raise TypeError(
-            'Session object must supply a '
-            f'callable "{http_method.value}" method',
-        )
-
-    response = request_callable(url, json=data, timeout=timeout, **kwargs)
-    response.raise_for_status()
+    path = Path(file_path)
+    path.parent.mkdir(parents=True, exist_ok=True)
 
-    # Try JSON first, fall back to text.
-    try:
-        payload: Any = response.json()
-    except ValueError:
-        payload = response.text
+    # If no explicit format is provided, let File infer from extension.
+    if file_format is None:
+        records = File(path).write(data)
+        ext = path.suffix.lstrip('.').lower()
+        fmt = FileFormat.coerce(ext) if ext else FileFormat.JSON
+    else:
+        fmt = FileFormat.coerce(file_format)
+        records = File(path, fmt).write(data)
+    if fmt is FileFormat.CSV and records == 0:
+        message = 'No data to write'
+    else:
+        message = f'Data loaded to {path}'
 
     return {
         'status': 'success',
-        'status_code': response.status_code,
-        'message': f'Data loaded to {url}',
-        'response': payload,
-        'records': count_records(data),
-        'method': http_method.value.upper(),
+        'message': message,
+        'records': records,
     }
 
 

etlplus/{run.py → ops/run.py}

@@ -1,5 +1,5 @@
 """
-:mod:`etlplus.run` module.
+:mod:`etlplus.ops.run` module.
 
 A module for running ETL jobs defined in YAML configurations.
 """
@@ -9,126 +9,78 @@ from __future__ import annotations
 from collections.abc import Mapping
 from typing import Any
 from typing import Final
-from typing import TypedDict
 from typing import cast
 from urllib.parse import urlsplit
 from urllib.parse import urlunsplit
 
-import requests  # type: ignore[import]
-
-from .api import EndpointClient  # noqa: F401 (re-exported for tests)
-from .api import PaginationConfigMap
-from .api import RequestOptions
-from .api import RetryPolicy
-from .api import Url
-from .config import load_pipeline_config
-from .enums import DataConnectorType
+from ..api import EndpointClient  # noqa: F401 (re-exported for tests)
+from ..api import HttpMethod
+from ..api import PaginationConfigMap
+from ..api import RequestOptions
+from ..api import compose_api_request_env
+from ..api import compose_api_target_env
+from ..api import paginate_with_client
+from ..config import load_pipeline_config
+from ..enums import DataConnectorType
+from ..file import FileFormat
+from ..types import JSONData
+from ..types import JSONDict
+from ..types import PipelineConfig
+from ..types import StrPath
+from ..types import Timeout
+from ..utils import print_json
 from .extract import extract
 from .load import load
-from .run_helpers import compose_api_request_env
-from .run_helpers import compose_api_target_env
-from .run_helpers import paginate_with_client
 from .transform import transform
-from .types import JSONDict
-from .types import Timeout
-from .utils import print_json
+from .utils import maybe_validate
 from .validate import validate
-from .validation.utils import maybe_validate
 
 # SECTION: EXPORTS ========================================================== #
 
 
-__all__ = ['run']
-
-
-# SECTION: TYPED DICTS ====================================================== #
-
+__all__ = [
+    # Functions
+    'run',
+    'run_pipeline',
+]
 
-class BaseApiHttpEnv(TypedDict, total=False):
-    """
-    Common HTTP request environment for API interactions.
-
-    Fields shared by both source-side and target-side API operations.
-    """
 
-    # Request details
-    url: Url | None
-    headers: dict[str, str]
-    timeout: Timeout
-
-    # Session
-    session: requests.Session | None
-
-
-class ApiRequestEnv(BaseApiHttpEnv, total=False):
-    """
-    Composed request environment for API sources.
+# SECTION: CONSTANTS ======================================================== #
 
-    Returned by ``compose_api_request_env`` (run_helpers) and consumed by the
-    API extract branch. Values are fully merged with endpoint/API defaults and
-    job-level overrides, preserving the original precedence and behavior.
-    """
 
-    # Client
-    use_endpoints: bool
-    base_url: str | None
-    base_path: str | None
-    endpoints_map: dict[str, str] | None
-    endpoint_key: str | None
+DEFAULT_CONFIG_PATH: Final[str] = 'in/pipeline.yml'
 
-    # Request
-    params: dict[str, Any]
-    pagination: PaginationConfigMap | None
-    sleep_seconds: float
 
-    # Reliability
-    retry: RetryPolicy | None
-    retry_network_errors: bool
+# SECTION: INTERNAL FUNCTIONS =============================================== #
 
 
-class ApiTargetEnv(BaseApiHttpEnv, total=False):
-    """
-    Composed request environment for API targets.
-
-    Returned by ``compose_api_target_env`` (run_helpers) and consumed by the
-    API load branch. Values are merged from the target object, optional
-    API/endpoint reference, and job-level overrides, preserving original
-    precedence and behavior.
-
-    Notes
-    -----
-    - Precedence for inherited values matches original logic:
-        overrides -> target -> API profile defaults.
-    - Target composition does not include pagination/rate-limit/retry since
-        loads are single-request operations; only headers/timeout/session
-        apply.
+def _resolve_validation_config(
+    job_obj: Any,
+    cfg: Any,
+) -> tuple[bool, dict[str, Any], str, str]:
     """
+    Resolve validation settings for a job with safe defaults.
 
-    # Request
-    method: str | None
-
-
-class SessionConfig(TypedDict, total=False):
-    """
-    Minimal session configuration schema accepted by this runner.
+    Parameters
+    ----------
+    job_obj : Any
+        Job configuration object.
+    cfg : Any
+        Pipeline configuration object with validations.
 
-    Keys mirror common requests.Session options; all are optional.
+    Returns
+    -------
+    tuple[bool, dict[str, Any], str, str]
+        Tuple of (enabled, rules, severity, phase).
     """
+    val_ref = job_obj.validate
+    if val_ref is None:
+        return False, {}, 'error', 'before_transform'
 
-    headers: Mapping[str, Any]
-    params: Mapping[str, Any]
-    auth: Any  # (user, pass) tuple or requests-compatible auth object
-    verify: bool | str
-    cert: Any  # str or (cert, key)
-    proxies: Mapping[str, Any]
-    cookies: Mapping[str, Any]
-    trust_env: bool
-
-
-# SECTION: CONSTANTS ======================================================== #
-
-
-DEFAULT_CONFIG_PATH: Final[str] = 'in/pipeline.yml'
+    rules = cfg.validations.get(val_ref.ruleset, {})
+    severity = (val_ref.severity or 'error').lower()
+    phase = (val_ref.phase or 'before_transform').lower()
+    return True, rules, severity, phase
 
 
 # SECTION: FUNCTIONS ======================================================== #
@@ -207,7 +159,7 @@ def run(
                 and env.get('endpoint_key')
             ):
                 # Construct client using module-level EndpointClient so tests
-                # can monkeypatch this class on etlplus.run.
+                # can monkeypatch this class on etlplus.ops.run.
                 ClientClass = EndpointClient  # noqa: N806
                 client = ClientClass(
                     base_url=cast(str, env['base_url']),
@@ -263,19 +215,10 @@ def run(
             # keep explicit guard for defensive programming.
             raise ValueError(f'Unsupported source type: {stype_raw}')
 
-    # DRY: unified validation helper (pre/post transform)
-    val_ref = job_obj.validate
-    enabled_validation = val_ref is not None
-    if enabled_validation:
-        # Type narrowing for static checkers
-        assert val_ref is not None
-        rules = cfg.validations.get(val_ref.ruleset, {})
-        severity = (val_ref.severity or 'error').lower()
-        phase = (val_ref.phase or 'before_transform').lower()
-    else:
-        rules = {}
-        severity = 'error'
-        phase = 'before_transform'
+    enabled_validation, rules, severity, phase = _resolve_validation_config(
+        job_obj,
+        cfg,
+    )
 
     # Pre-transform validation (if configured).
     data = maybe_validate(
@@ -361,3 +304,90 @@ def run(
     # Return the terminal load result directly; callers (e.g., CLI) can wrap
     # it in their own envelope when needed.
     return cast(JSONDict, result)
+
+
+def run_pipeline(
+    *,
+    source_type: DataConnectorType | str | None = None,
+    source: StrPath | JSONData | None = None,
+    operations: PipelineConfig | None = None,
+    target_type: DataConnectorType | str | None = None,
+    target: StrPath | None = None,
+    file_format: FileFormat | str | None = None,
+    method: HttpMethod | str | None = None,
+    **kwargs: Any,
+) -> JSONData:
+    """
+    Run a single extract-transform-load flow without a YAML config.
+
+    Parameters
+    ----------
+    source_type : DataConnectorType | str | None, optional
+        Connector type for extraction. When ``None``, ``source`` is assumed
+        to be pre-loaded data and extraction is skipped.
+    source : StrPath | JSONData | None, optional
+        Data source for extraction or the pre-loaded payload when
+        ``source_type`` is ``None``.
+    operations : PipelineConfig | None, optional
+        Transform configuration passed to :func:`etlplus.ops.transform`.
+    target_type : DataConnectorType | str | None, optional
+        Connector type for loading. When ``None``, load is skipped and the
+        transformed data is returned.
+    target : StrPath | None, optional
+        Target for loading (file path, connection string, or API URL).
+    file_format : FileFormat | str | None, optional
+        File format for file sources/targets (forwarded to extract/load).
+    method : HttpMethod | str | None, optional
+        HTTP method for API loads (forwarded to :func:`etlplus.ops.load`).
+    **kwargs : Any
+        Extra keyword arguments forwarded to extract/load for API options
+        (headers, timeout, session, etc.).
+
+    Returns
+    -------
+    JSONData
+        Transformed data or the load result payload.
+
+    Raises
+    ------
+    TypeError
+        Raised when extracted data is not a dict or list of dicts and no
+        target is specified.
+    ValueError
+        Raised when required source/target inputs are missing.
+    """
+    if source_type is None:
+        if source is None:
+            raise ValueError('source or source_type is required')
+        data = source
+    else:
+        if source is None:
+            raise ValueError('source is required when source_type is set')
+        data = extract(
+            source_type,
+            cast(StrPath, source),
+            file_format=file_format,
+            **kwargs,
+        )
+
+    if operations:
+        data = transform(data, operations)
+
+    if target_type is None:
+        if not isinstance(data, (dict, list)):
+            raise TypeError(
+                f'Expected data to be dict or list of dicts, '
+                f'got {type(data).__name__}',
+            )
+        return data
+    if target is None:
+        raise ValueError('target is required when target_type is set')
+
+    return load(
+        data,
+        target_type,
+        target,
+        file_format=file_format,
+        method=method,
+        **kwargs,
+    )