etlplus 0.9.2__py3-none-any.whl → 0.10.2__py3-none-any.whl

etlplus/{ops/load.py → load.py}

@@ -1,5 +1,5 @@
 """
-:mod:`etlplus.ops.load` module.
+:mod:`etlplus.load` module.
 
 Helpers to load data into files, databases, and REST APIs.
 """
@@ -12,16 +12,20 @@ from pathlib import Path
 from typing import Any
 from typing import cast
 
-from ..api import HttpMethod
-from ..api.utils import resolve_request
-from ..enums import DataConnectorType
-from ..file import File
-from ..file import FileFormat
-from ..types import JSONData
-from ..types import JSONDict
-from ..types import JSONList
-from ..types import StrPath
-from ..utils import count_records
+import requests  # type: ignore[import]
+
+from .enums import DataConnectorType
+from .enums import FileFormat
+from .enums import HttpMethod
+from .enums import coerce_data_connector_type
+from .enums import coerce_file_format
+from .enums import coerce_http_method
+from .file import File
+from .types import JSONData
+from .types import JSONDict
+from .types import JSONList
+from .types import StrPath
+from .utils import count_records
 
 # SECTION: INTERNAL FUNCTIONS ============================================== #
 
@@ -68,7 +72,7 @@ def _parse_json_string(
 # SECTION: FUNCTIONS ======================================================== #
 
 
-# -- Helpers -- #
+# -- Data Loading -- #
 
 
 def load_data(
@@ -97,7 +101,7 @@ def load_data(
         return cast(JSONData, source)
 
     if isinstance(source, Path):
-        return File(source, FileFormat.JSON).read()
+        return File(source, FileFormat.JSON).read_json()
 
     if isinstance(source, str):
         # Special case: '-' means read JSON from STDIN (Unix convention).
@@ -107,7 +111,7 @@ def load_data(
         candidate = Path(source)
         if candidate.exists():
             try:
-                return File(candidate, FileFormat.JSON).read()
+                return File(candidate, FileFormat.JSON).read_json()
             except (OSError, json.JSONDecodeError, ValueError):
                 # Fall back to treating the string as raw JSON content.
                 pass
@@ -118,59 +122,58 @@ def load_data(
     )
 
 
-def load_to_api(
+# -- File Loading -- #
+
+
+def load_to_file(
     data: JSONData,
-    url: str,
-    method: HttpMethod | str,
-    **kwargs: Any,
+    file_path: StrPath,
+    file_format: FileFormat | str | None = None,
 ) -> JSONDict:
     """
-    Load data to a REST API.
+    Persist data to a local file.
 
     Parameters
     ----------
     data : JSONData
-        Data to send as JSON.
-    url : str
-        API endpoint URL.
-    method : HttpMethod | str
-        HTTP method to use.
-    **kwargs : Any
-        Extra arguments forwarded to ``requests`` (e.g., ``timeout``).
-        When omitted, ``timeout`` defaults to 10 seconds.
+        Data to write.
+    file_path : StrPath
+        Target file path.
+    file_format : FileFormat | str | None, optional
+        Output format. If omitted (None), the format is inferred from the
+        filename extension.
 
     Returns
     -------
     JSONDict
-        Result dictionary including response payload or text.
+        Result dictionary with status and record count.
     """
-    # Apply a conservative timeout to guard against hanging requests.
-    timeout = kwargs.pop('timeout', 10.0)
-    session = kwargs.pop('session', None)
-    request_callable, timeout, http_method = resolve_request(
-        method,
-        session=session,
-        timeout=timeout,
-    )
-    response = request_callable(url, json=data, timeout=timeout, **kwargs)
-    response.raise_for_status()
+    path = Path(file_path)
+    path.parent.mkdir(parents=True, exist_ok=True)
 
-    # Try JSON first, fall back to text.
-    try:
-        payload: Any = response.json()
-    except ValueError:
-        payload = response.text
+    # If no explicit format is provided, let File infer from extension.
+    if file_format is None:
+        records = File(path).write(data)
+        ext = path.suffix.lstrip('.').lower()
+        fmt = coerce_file_format(ext) if ext else FileFormat.JSON
+    else:
+        fmt = coerce_file_format(file_format)
+        records = File(path, fmt).write(data)
+    if fmt is FileFormat.CSV and records == 0:
+        message = 'No data to write'
+    else:
+        message = f'Data loaded to {path}'
 
     return {
         'status': 'success',
-        'status_code': response.status_code,
-        'message': f'Data loaded to {url}',
-        'response': payload,
-        'records': count_records(data),
-        'method': http_method.value.upper(),
+        'message': message,
+        'records': records,
     }
 
 
+# -- Database Loading (Placeholder) -- #
+
+
 def load_to_database(
     data: JSONData,
     connection_string: str,
@@ -206,49 +209,69 @@ def load_to_database(
     }
 
 
-def load_to_file(
+# -- REST API Loading -- #
+
+
+def load_to_api(
     data: JSONData,
-    file_path: StrPath,
-    file_format: FileFormat | str | None = None,
+    url: str,
+    method: HttpMethod | str,
+    **kwargs: Any,
 ) -> JSONDict:
     """
-    Persist data to a local file.
+    Load data to a REST API.
 
     Parameters
     ----------
     data : JSONData
-        Data to write.
-    file_path : StrPath
-        Target file path.
-    file_format : FileFormat | str | None, optional
-        Output format. If omitted (None), the format is inferred from the
-        filename extension.
+        Data to send as JSON.
+    url : str
+        API endpoint URL.
+    method : HttpMethod | str
+        HTTP method to use.
+    **kwargs : Any
+        Extra arguments forwarded to ``requests`` (e.g., ``timeout``).
 
     Returns
     -------
     JSONDict
-        Result dictionary with status and record count.
+        Result dictionary including response payload or text.
+
+    Raises
+    ------
+    TypeError
+        If the session object is not valid.
     """
-    path = Path(file_path)
-    path.parent.mkdir(parents=True, exist_ok=True)
+    http_method = coerce_http_method(method)
 
-    # If no explicit format is provided, let File infer from extension.
-    if file_format is None:
-        records = File(path).write(data)
-        ext = path.suffix.lstrip('.').lower()
-        fmt = FileFormat.coerce(ext) if ext else FileFormat.JSON
-    else:
-        fmt = FileFormat.coerce(file_format)
-        records = File(path, fmt).write(data)
-    if fmt is FileFormat.CSV and records == 0:
-        message = 'No data to write'
-    else:
-        message = f'Data loaded to {path}'
+    # Apply a conservative timeout to guard against hanging requests.
+    timeout = kwargs.pop('timeout', 10.0)
+    session = kwargs.pop('session', None)
+    requester = session or requests
+
+    request_callable = getattr(requester, http_method.value, None)
+    if not callable(request_callable):
+        raise TypeError(
+            'Session object must supply a '
+            f'callable "{http_method.value}" method',
+        )
+
+    response = request_callable(url, json=data, timeout=timeout, **kwargs)
+    response.raise_for_status()
+
+    # Try JSON first, fall back to text.
+    try:
+        payload: Any = response.json()
+    except ValueError:
+        payload = response.text
 
     return {
         'status': 'success',
-        'message': message,
-        'records': records,
+        'status_code': response.status_code,
+        'message': f'Data loaded to {url}',
+        'response': payload,
+        'records': count_records(data),
+        'method': http_method.value.upper(),
     }
 
 
@@ -293,7 +316,7 @@ def load(
     """
     data = load_data(source)
 
-    match DataConnectorType.coerce(target_type):
+    match coerce_data_connector_type(target_type):
         case DataConnectorType.FILE:
             # Prefer explicit format if provided, else infer from filename.
             return load_to_file(data, target, file_format)
@@ -308,6 +331,6 @@ def load(
                 **kwargs,
             )
         case _:
-            # :meth:`coerce` already raises for invalid connector types, but
-            # keep explicit guard for defensive programming.
+            # `coerce_data_connector_type` covers invalid entries, but keep
+            # explicit guard.
             raise ValueError(f'Invalid target type: {target_type}')

etlplus/{ops/run.py → run.py}

@@ -1,5 +1,5 @@
 """
-:mod:`etlplus.ops.run` module.
+:mod:`etlplus.run` module.
 
 A module for running ETL jobs defined in YAML configurations.
 """
@@ -9,78 +9,127 @@ from __future__ import annotations
 from collections.abc import Mapping
 from typing import Any
 from typing import Final
+from typing import TypedDict
 from typing import cast
 from urllib.parse import urlsplit
 from urllib.parse import urlunsplit
 
-from ..api import EndpointClient  # noqa: F401 (re-exported for tests)
-from ..api import HttpMethod
-from ..api import PaginationConfigMap
-from ..api import RequestOptions
-from ..api import compose_api_request_env
-from ..api import compose_api_target_env
-from ..api import paginate_with_client
-from ..enums import DataConnectorType
-from ..file import FileFormat
-from ..types import JSONData
-from ..types import JSONDict
-from ..types import PipelineConfig
-from ..types import StrPath
-from ..types import Timeout
-from ..utils import print_json
-from ..workflow import load_pipeline_config
+import requests  # type: ignore[import]
+
+from .api import EndpointClient  # noqa: F401 (re-exported for tests)
+from .api import PaginationConfigMap
+from .api import RequestOptions
+from .api import RetryPolicy
+from .api import Url
+from .config import load_pipeline_config
+from .enums import DataConnectorType
+from .enums import coerce_data_connector_type
 from .extract import extract
 from .load import load
+from .run_helpers import compose_api_request_env
+from .run_helpers import compose_api_target_env
+from .run_helpers import paginate_with_client
 from .transform import transform
-from .utils import maybe_validate
+from .types import JSONDict
+from .types import Timeout
+from .utils import print_json
 from .validate import validate
+from .validation.utils import maybe_validate
 
 # SECTION: EXPORTS ========================================================== #
 
 
-__all__ = [
-    # Functions
-    'run',
-    'run_pipeline',
-]
+__all__ = ['run']
 
 
-# SECTION: CONSTANTS ======================================================== #
+# SECTION: TYPED DICTS ====================================================== #
 
 
-DEFAULT_CONFIG_PATH: Final[str] = 'in/pipeline.yml'
+class BaseApiHttpEnv(TypedDict, total=False):
+    """
+    Common HTTP request environment for API interactions.
 
+    Fields shared by both source-side and target-side API operations.
+    """
+
+    # Request details
+    url: Url | None
+    headers: dict[str, str]
+    timeout: Timeout
 
-# SECTION: INTERNAL FUNCTIONS =============================================== #
+    # Session
+    session: requests.Session | None
 
 
-def _resolve_validation_config(
-    job_obj: Any,
-    cfg: Any,
-) -> tuple[bool, dict[str, Any], str, str]:
+class ApiRequestEnv(BaseApiHttpEnv, total=False):
     """
-    Resolve validation settings for a job with safe defaults.
+    Composed request environment for API sources.
 
-    Parameters
-    ----------
-    job_obj : Any
-        Job configuration object.
-    cfg : Any
-        Pipeline configuration object with validations.
+    Returned by ``compose_api_request_env`` (run_helpers) and consumed by the
+    API extract branch. Values are fully merged with endpoint/API defaults and
+    job-level overrides, preserving the original precedence and behavior.
+    """
 
-    Returns
-    -------
-    tuple[bool, dict[str, Any], str, str]
-        Tuple of (enabled, rules, severity, phase).
+    # Client
+    use_endpoints: bool
+    base_url: str | None
+    base_path: str | None
+    endpoints_map: dict[str, str] | None
+    endpoint_key: str | None
+
+    # Request
+    params: dict[str, Any]
+    pagination: PaginationConfigMap | None
+    sleep_seconds: float
+
+    # Reliability
+    retry: RetryPolicy | None
+    retry_network_errors: bool
+
+
+class ApiTargetEnv(BaseApiHttpEnv, total=False):
     """
-    val_ref = job_obj.validate
-    if val_ref is None:
-        return False, {}, 'error', 'before_transform'
+    Composed request environment for API targets.
+
+    Returned by ``compose_api_target_env`` (run_helpers) and consumed by the
+    API load branch. Values are merged from the target object, optional
+    API/endpoint reference, and job-level overrides, preserving original
+    precedence and behavior.
+
+    Notes
+    -----
+    - Precedence for inherited values matches original logic:
+        overrides -> target -> API profile defaults.
+    - Target composition does not include pagination/rate-limit/retry since
+        loads are single-request operations; only headers/timeout/session
+        apply.
+    """
+
+    # Request
+    method: str | None
+
+
+class SessionConfig(TypedDict, total=False):
+    """
+    Minimal session configuration schema accepted by this runner.
+
+    Keys mirror common requests.Session options; all are optional.
+    """
+
+    headers: Mapping[str, Any]
+    params: Mapping[str, Any]
+    auth: Any  # (user, pass) tuple or requests-compatible auth object
+    verify: bool | str
+    cert: Any  # str or (cert, key)
+    proxies: Mapping[str, Any]
+    cookies: Mapping[str, Any]
+    trust_env: bool
+
+
+# SECTION: CONSTANTS ======================================================== #
+
 
-    rules = cfg.validations.get(val_ref.ruleset, {})
-    severity = (val_ref.severity or 'error').lower()
-    phase = (val_ref.phase or 'before_transform').lower()
-    return True, rules, severity, phase
+DEFAULT_CONFIG_PATH: Final[str] = 'in/pipeline.yml'
 
 
 # SECTION: FUNCTIONS ======================================================== #
@@ -136,7 +185,8 @@ def run(
 
     data: Any
     stype_raw = getattr(source_obj, 'type', None)
-    match DataConnectorType.coerce(stype_raw or ''):
+    stype = coerce_data_connector_type(stype_raw or '')
+    match stype:
         case DataConnectorType.FILE:
             path = getattr(source_obj, 'path', None)
             fmt = ex_opts.get('format') or getattr(
@@ -159,15 +209,12 @@ def run(
                 and env.get('endpoint_key')
             ):
                 # Construct client using module-level EndpointClient so tests
-                # can monkeypatch this class on etlplus.ops.run.
+                # can monkeypatch this class on etlplus.run.
                 ClientClass = EndpointClient  # noqa: N806
                 client = ClientClass(
-                    base_url=cast(str, env.get('base_url')),
+                    base_url=cast(str, env['base_url']),
                     base_path=cast(str | None, env.get('base_path')),
-                    endpoints=cast(
-                        dict[str, str],
-                        env.get('endpoints_map', {}),
-                    ),
+                    endpoints=cast(dict[str, str], env['endpoints_map']),
                     retry=env.get('retry'),
                     retry_network_errors=bool(
                         env.get('retry_network_errors', False),
@@ -176,7 +223,7 @@ def run(
                 )
                 data = paginate_with_client(
                     client,
-                    cast(str, env.get('endpoint_key')),
+                    cast(str, env['endpoint_key']),
                     env.get('params'),
                     env.get('headers'),
                     env.get('timeout'),
@@ -214,14 +261,23 @@ def run(
                     sleep_seconds=cast(float, env.get('sleep_seconds', 0.0)),
                 )
         case _:
-            # :meth:`coerce` already raises for invalid connector types, but
-            # keep explicit guard for defensive programming.
+            # ``coerce_data_connector_type`` already raises for invalid
+            # connector types; this branch is defensive only.
             raise ValueError(f'Unsupported source type: {stype_raw}')
 
-    enabled_validation, rules, severity, phase = _resolve_validation_config(
-        job_obj,
-        cfg,
-    )
+    # DRY: unified validation helper (pre/post transform)
+    val_ref = job_obj.validate
+    enabled_validation = val_ref is not None
+    if enabled_validation:
+        # Type narrowing for static checkers
+        assert val_ref is not None
+        rules = cfg.validations.get(val_ref.ruleset, {})
+        severity = (val_ref.severity or 'error').lower()
+        phase = (val_ref.phase or 'before_transform').lower()
+    else:
+        rules = {}
+        severity = 'error'
+        phase = 'before_transform'
 
     # Pre-transform validation (if configured).
     data = maybe_validate(
@@ -262,7 +318,8 @@ def run(
     overrides = job_obj.load.overrides or {}
 
     ttype_raw = getattr(target_obj, 'type', None)
-    match DataConnectorType.coerce(ttype_raw or ''):
+    ttype = coerce_data_connector_type(ttype_raw or '')
+    match ttype:
         case DataConnectorType.FILE:
             path = overrides.get('path') or getattr(target_obj, 'path', None)
             fmt = overrides.get('format') or getattr(
@@ -279,14 +336,12 @@ def run(
             if not url_t:
                 raise ValueError('API target missing "url"')
             kwargs_t: dict[str, Any] = {}
-            headers = env_t.get('headers')
-            if headers:
-                kwargs_t['headers'] = cast(dict[str, str], headers)
+            if env_t.get('headers'):
+                kwargs_t['headers'] = cast(dict[str, str], env_t['headers'])
             if env_t.get('timeout') is not None:
-                kwargs_t['timeout'] = env_t.get('timeout')
-            session = env_t.get('session')
-            if session is not None:
-                kwargs_t['session'] = session
+                kwargs_t['timeout'] = env_t['timeout']
+            if env_t.get('session') is not None:
+                kwargs_t['session'] = env_t['session']
             result = load(
                 data,
                 'api',
@@ -302,97 +357,10 @@ def run(
             )
             result = load(data, 'database', str(conn))
         case _:
-            # :meth:`coerce` already raises for invalid connector types, but
-            # keep explicit guard for defensive programming.
+            # ``coerce_data_connector_type`` already raises for invalid
+            # connector types; this branch is defensive only.
             raise ValueError(f'Unsupported target type: {ttype_raw}')
 
     # Return the terminal load result directly; callers (e.g., CLI) can wrap
     # it in their own envelope when needed.
     return cast(JSONDict, result)
-
-
-def run_pipeline(
-    *,
-    source_type: DataConnectorType | str | None = None,
-    source: StrPath | JSONData | None = None,
-    operations: PipelineConfig | None = None,
-    target_type: DataConnectorType | str | None = None,
-    target: StrPath | None = None,
-    file_format: FileFormat | str | None = None,
-    method: HttpMethod | str | None = None,
-    **kwargs: Any,
-) -> JSONData:
-    """
-    Run a single extract-transform-load flow without a YAML config.
-
-    Parameters
-    ----------
-    source_type : DataConnectorType | str | None, optional
-        Connector type for extraction. When ``None``, ``source`` is assumed
-        to be pre-loaded data and extraction is skipped.
-    source : StrPath | JSONData | None, optional
-        Data source for extraction or the pre-loaded payload when
-        ``source_type`` is ``None``.
-    operations : PipelineConfig | None, optional
-        Transform configuration passed to :func:`etlplus.ops.transform`.
-    target_type : DataConnectorType | str | None, optional
-        Connector type for loading. When ``None``, load is skipped and the
-        transformed data is returned.
-    target : StrPath | None, optional
-        Target for loading (file path, connection string, or API URL).
-    file_format : FileFormat | str | None, optional
-        File format for file sources/targets (forwarded to extract/load).
-    method : HttpMethod | str | None, optional
-        HTTP method for API loads (forwarded to :func:`etlplus.ops.load`).
-    **kwargs : Any
-        Extra keyword arguments forwarded to extract/load for API options
-        (headers, timeout, session, etc.).
-
-    Returns
-    -------
-    JSONData
-        Transformed data or the load result payload.
-
-    Raises
-    ------
-    TypeError
-        Raised when extracted data is not a dict or list of dicts and no
-        target is specified.
-    ValueError
-        Raised when required source/target inputs are missing.
-    """
-    if source_type is None:
-        if source is None:
-            raise ValueError('source or source_type is required')
-        data = source
-    else:
-        if source is None:
-            raise ValueError('source is required when source_type is set')
-        data = extract(
-            source_type,
-            cast(StrPath, source),
-            file_format=file_format,
-            **kwargs,
-        )
-
-    if operations:
-        data = transform(data, operations)
-
-    if target_type is None:
-        if not isinstance(data, (dict, list)):
-            raise TypeError(
-                f'Expected data to be dict or list of dicts, '
-                f'got {type(data).__name__}',
-            )
-        return data
-    if target is None:
-        raise ValueError('target is required when target_type is set')
-
-    return load(
-        data,
-        target_type,
-        target,
-        file_format=file_format,
-        method=method,
-        **kwargs,
-    )