rdetoolkit 1.5.1__cp312-cp312-win_amd64.whl → 1.5.3__cp312-cp312-win_amd64.whl

rdetoolkit/__init__.py

@@ -1,9 +1,21 @@
 from __future__ import annotations
 
+import sys
+import warnings
+
+# Python 3.9 deprecation warning (Issue #360)
+if sys.version_info < (3, 10):
+    warnings.warn(
+        "Python 3.9 support is deprecated and will be removed in rdetoolkit v2.0. "
+        "Please upgrade to Python 3.10 or later.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
 from importlib import import_module
 from typing import Any
 
-__version__ = "1.5.1"
+__version__ = "1.5.3"
 
 _LAZY_ATTRS: dict[str, tuple[str, str]] = {
     "DirectoryOps": ("rdetoolkit.core", "DirectoryOps"),
@@ -26,6 +38,9 @@ _LAZY_MODULES: dict[str, str] = {
     "rde2util": "rdetoolkit.rde2util",
     "rdelogger": "rdetoolkit.rdelogger",
     "workflows": "rdetoolkit.workflows",
+    "processing": "rdetoolkit.processing",
+    "storage": "rdetoolkit.storage",
+    "traceback": "rdetoolkit.traceback",
     "config": "rdetoolkit.models.config",
     "invoice": "rdetoolkit.models.invoice",
     "invoice_schema": "rdetoolkit.models.invoice_schema",
@@ -55,6 +70,9 @@ __all__ = [
     "rde2util",
     "rdelogger",
     "workflows",
+    "processing",
+    "storage",
+    "traceback",
     "config",
     "invoice",
     "invoice_schema",

rdetoolkit/cli/validate.py

@@ -13,7 +13,13 @@ if TYPE_CHECKING:
 
 app = typer.Typer(
     name="validate",
-    help="Validate RDE schema and data files",
+    help="""Validate RDE schema and data files.
+
+All validate commands use standardized exit codes:
+  0 = Success (validation passed)
+  1 = Validation failure (data/schema issues)
+  2 = Usage error (invalid arguments, missing files)
+""",
     no_args_is_help=True,
 )
 
@@ -87,10 +93,11 @@ def handle_validation_error(error: Exception, error_type: str) -> None:
         typer.echo(str(error), err=True)
         raise typer.Exit(code=1) from error
 
-    # Unknown errors get exit code 3 (internal error)
-    msg = f"Internal error during {error_type}: {error}"
+    # All other errors (including internal errors) map to exit code 2
+    # This includes unexpected exceptions that indicate configuration or usage issues
+    msg = f"Error during {error_type}: {error}"
     typer.echo(msg, err=True)
-    raise typer.Exit(code=3) from error
+    raise typer.Exit(code=2) from error
 
 
 # Common options for validation commands
@@ -141,6 +148,11 @@ def invoice_schema(
     Validates that the invoice schema file conforms to the expected structure
     and contains valid field definitions.
 
+    Exit Codes:
+        0: Success - validation passed
+        1: Validation failure - schema structure issues found
+        2: Usage error - file not found or invalid arguments
+
     Examples:
         rdetoolkit validate invoice-schema tasksupport/invoice.schema.json
         rdetoolkit validate invoice-schema schema.json --format json
@@ -151,6 +163,9 @@ def invoice_schema(
         command = InvoiceSchemaCommand(schema_path)
         result = command.execute()
         handle_validation_result(result, format_type, strict, quiet)
+    except typer.Exit:
+        # Re-raise typer.Exit to let Typer handle it properly
+        raise
     except FileNotFoundError:
         handle_file_not_found(schema_path, "Schema")
     except Exception as e:
@@ -177,6 +192,11 @@ def metadata_def(
     Validates that the metadata definition file conforms to the expected
     structure defined by the MetadataItem schema.
 
+    Exit Codes:
+        0: Success - validation passed
+        1: Validation failure - definition structure issues found
+        2: Usage error - file not found or invalid arguments
+
     Examples:
         rdetoolkit validate metadata-def tasksupport/metadata_def.json
         rdetoolkit validate metadata-def metadata_def.json --format json
@@ -187,6 +207,9 @@ def metadata_def(
         command = MetadataDefCommand(metadata_def_path)
         result = command.execute()
         handle_validation_result(result, format_type, strict, quiet)
+    except typer.Exit:
+        # Re-raise typer.Exit to let Typer handle it properly
+        raise
     except FileNotFoundError:
         handle_file_not_found(metadata_def_path, "Metadata definition")
     except Exception as e:
@@ -195,26 +218,22 @@ def metadata_def(
 
 @app.command("invoice")
 def invoice(
-    invoice_path: Annotated[
-        Path,
-        typer.Argument(
-            help="Path to invoice.json file",
-            exists=True,
-            dir_okay=False,
-            resolve_path=True,
-        ),
-    ],
-    schema_path: Annotated[
-        Path,
-        typer.Option(
-            "--schema",
-            "-s",
-            help="Path to invoice.schema.json file",
-            exists=True,
-            dir_okay=False,
-            resolve_path=True,
-        ),
-    ],
+    invoice_path: Path = typer.Argument(
+        ...,
+        help="Path to invoice.json file",
+        exists=True,
+        dir_okay=False,
+        resolve_path=True,
+    ),
+    schema_path: Path = typer.Option(
+        ...,
+        "--schema",
+        "-s",
+        help="Path to invoice.schema.json file",
+        exists=True,
+        dir_okay=False,
+        resolve_path=True,
+    ),
     format_type: FormatOption = OutputFormat.TEXT,
     strict: StrictOption = False,
     quiet: QuietOption = False,
@@ -224,6 +243,11 @@ def invoice(
     Validates that the invoice.json file conforms to the structure and
     constraints defined in the invoice.schema.json file.
 
+    Exit Codes:
+        0: Success - validation passed
+        1: Validation failure - data violates schema constraints
+        2: Usage error - files not found or invalid arguments
+
     Examples:
         rdetoolkit validate invoice raw/invoice.json --schema tasksupport/invoice.schema.json
         rdetoolkit validate invoice invoice.json -s schema.json --format json
@@ -251,26 +275,22 @@ def invoice(
 
 @app.command("metadata")
 def metadata(
-    metadata_path: Annotated[
-        Path,
-        typer.Argument(
-            help="Path to metadata.json file",
-            exists=True,
-            dir_okay=False,
-            resolve_path=True,
-        ),
-    ],
-    schema_path: Annotated[
-        Path,
-        typer.Option(
-            "--schema",
-            "-s",
-            help="Path to metadata definition JSON file",
-            exists=True,
-            dir_okay=False,
-            resolve_path=True,
-        ),
-    ],
+    metadata_path: Path = typer.Argument(
+        ...,
+        help="Path to metadata.json file",
+        exists=True,
+        dir_okay=False,
+        resolve_path=True,
+    ),
+    schema_path: Path = typer.Option(
+        ...,
+        "--schema",
+        "-s",
+        help="Path to metadata definition JSON file",
+        exists=True,
+        dir_okay=False,
+        resolve_path=True,
+    ),
     format_type: FormatOption = OutputFormat.TEXT,
     strict: StrictOption = False,
     quiet: QuietOption = False,
@@ -280,6 +300,11 @@ def metadata(
     Validates that the metadata.json file conforms to the structure
     defined in the metadata definition file.
 
+    Exit Codes:
+        0: Success - validation passed
+        1: Validation failure - data violates definition constraints
+        2: Usage error - files not found or invalid arguments
+
     Examples:
         rdetoolkit validate metadata raw/metadata.json --schema tasksupport/metadata_def.json
         rdetoolkit validate metadata metadata.json -s metadata_def.json --format json
@@ -290,6 +315,9 @@ def metadata(
         command = MetadataCommand(metadata_path, schema_path)
         result = command.execute()
         handle_validation_result(result, format_type, strict, quiet)
+    except typer.Exit:
+        # Re-raise typer.Exit to let Typer handle it properly
+        raise
     except FileNotFoundError:
         # Determine which file is missing
         if not metadata_path.exists():
@@ -304,16 +332,14 @@ def metadata(
 
 @app.command("all")
 def validate_all(
-    project_dir: Annotated[
-        Optional[Path],
-        typer.Argument(
-            help="Root directory of RDE project (defaults to current directory)",
-            exists=True,
-            file_okay=False,
-            dir_okay=True,
-            resolve_path=True,
-        ),
-    ] = None,
+    project_dir: Optional[Path] = typer.Argument(
+        None,
+        help="Root directory of RDE project (defaults to current directory)",
+        exists=True,
+        file_okay=False,
+        dir_okay=True,
+        resolve_path=True,
+    ),
     format_type: FormatOption = OutputFormat.TEXT,
     strict: StrictOption = False,
     quiet: QuietOption = False,
@@ -329,7 +355,10 @@ def validate_all(
     - input/invoice/invoice.json (with schema)
     - input/metadata/metadata.json (if exists, with schema)
 
-    Exit code 0 only if ALL validations pass.
+    Exit Codes:
+        0: Success - all validations passed
+        1: Validation failure - one or more validations failed
+        2: Usage error - project directory not found or invalid arguments
 
     Examples:
         rdetoolkit validate all

rdetoolkit/cmd/validate.py

@@ -8,7 +8,7 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Optional, Union
 
-from rdetoolkit.validation import InvoiceValidator, MetadataValidator
+from rdetoolkit.validation import InvoiceValidator, MetadataDefinitionValidator, MetadataValidator
 
 
 @dataclass
@@ -181,8 +181,7 @@ def determine_exit_code(result: ValidationResult, strict: bool = False) -> int:
     Exit codes:
     - 0: Validation passed
     - 1: Validation failed (errors or warnings in strict mode)
-    - 2: Usage/argument errors (handled by Typer)
-    - 3: Internal errors (handled by exception handlers)
+    - 2: Usage/argument errors (handled by CLI layer)
 
     Args:
         result: Validation result
@@ -428,7 +427,11 @@ class MetadataDefCommand(_ValidationErrorParser):
     """Command to validate metadata definition JSON files.
 
     This command validates metadata definition files using the
-    MetadataValidator's schema (MetadataItem pydantic model).
+    MetadataDefinitionValidator (for metadata-def.json structure).
+
+    Note:
+        This is separate from MetadataCommand which validates metadata.json
+        data files against metadata definition schemas.
     """
 
     def __init__(self, metadata_def_path: str | Path) -> None:
@@ -460,7 +463,8 @@ class MetadataDefCommand(_ValidationErrorParser):
         warnings: list[ValidationWarning] = []
 
         try:
-            validator = MetadataValidator()
+            # Use MetadataDefinitionValidator for metadata-def.json
+            validator = MetadataDefinitionValidator()
             # validate() with path parameter validates the file
             _ = validator.validate(path=self.metadata_def_path)
 
@@ -520,16 +524,15 @@ class MetadataCommand(_ValidationErrorParser):
         warnings: list[ValidationWarning] = []
 
         try:
-            # First validate the schema/definition itself
-            validator = MetadataValidator()
-            _ = validator.validate(path=self.schema_path)
-
-            # Then validate the metadata data against the schema
-            # Note: Current MetadataValidator validates individual items,
-            # not data against a separate schema. This is a structural difference
-            # from invoice validation. We validate that the metadata file
-            # conforms to MetadataItem structure.
-            _ = validator.validate(path=self.metadata_path)
+            # First validate the schema/definition file (metadata-def.json)
+            # Uses MetadataDefinitionValidator for dict[str, MetadataDefEntry] structure
+            def_validator = MetadataDefinitionValidator()
+            _ = def_validator.validate(path=self.schema_path)
+
+            # Then validate the metadata data file (metadata.json)
+            # Uses MetadataValidator for MetadataItem structure (constant/variable)
+            data_validator = MetadataValidator()
+            _ = data_validator.validate(path=self.metadata_path)
 
         except Exception as e:
             # Parse validation errors from exception message

rdetoolkit/config.py

@@ -6,8 +6,11 @@ from typing import Any, Final
 
 import yaml
 from pydantic import ValidationError
+from tomlkit.exceptions import TOMLKitError
 from tomlkit.toml_file import TOMLFile
+from yaml import YAMLError
 
+from rdetoolkit.exceptions import ConfigError
 from rdetoolkit.models.config import Config, TracebackSettings, MultiDataTileSettings, SystemSettings, SmartTableSettings
 from rdetoolkit.models.rde2types import RdeFsPath
 
@@ -16,6 +19,69 @@ PYPROJECT_CONFIG_FILES: Final = ["pyproject.toml"]
 CONFIG_FILES = CONFIG_FILE + PYPROJECT_CONFIG_FILES
 
 
+def _format_validation_error(
+    validation_error: ValidationError,
+    file_path: str,
+) -> ConfigError:
+    """Format pydantic ValidationError into user-friendly ConfigError.
+
+    Args:
+        validation_error: The pydantic ValidationError
+        file_path: Path to the configuration file
+
+    Returns:
+        ConfigError with detailed field-level information
+    """
+    errors = validation_error.errors()
+
+    if not errors:
+        return ConfigError(
+            "Configuration validation failed",
+            file_path=file_path,
+            error_type="validation_error",
+        )
+
+    # Take the first error for the main message
+    first_error = errors[0]
+    field_path = ".".join(str(loc) for loc in first_error["loc"])
+    error_msg = first_error["msg"]
+    error_type_detail = first_error["type"]
+
+    # Build detailed message
+    message_parts = [f"Invalid configuration in '{file_path}'"]
+
+    if field_path:
+        message_parts.append(f"Field '{field_path}' validation failed: {error_msg}")
+    else:
+        message_parts.append(f"Validation failed: {error_msg}")
+
+    # Add information about expected values if available
+    if "input" in first_error:
+        input_value = first_error["input"]
+        message_parts.append(f"Provided value: {input_value!r}")
+
+    # For extended_mode, provide specific guidance
+    if "extended_mode" in field_path and "enum" in error_type_detail.lower():
+        message_parts.append(
+            "Valid values for 'extended_mode': ['rdeformat', 'MultiDataTile']",
+        )
+
+    # Add validation error context if multiple errors exist
+    if len(errors) > 1:
+        message_parts.append(
+            f"Note: {len(errors)} validation error(s) found. Showing the first one.",
+        )
+
+    full_message = "\n".join(message_parts)
+
+    return ConfigError(
+        full_message,
+        file_path=file_path,
+        error_type="validation_error",
+        field_name=field_path,
+    )
+
+
 def parse_config_file(*, path: str | None = None) -> Config:
     """Parse the configuration file and return a Config object.
 
@@ -26,7 +92,7 @@ def parse_config_file(*, path: str | None = None) -> Config:
         Config: The parsed configuration object.
 
     Raises:
-        FileNotFoundError: If the specified configuration file does not exist.
+        ConfigError: If the specified configuration file does not exist or cannot be parsed.
 
     File Loading Priority:
         1. If `path` is provided and the file extension is ".toml", the function will attempt to read the file as a TOML file.
@@ -39,10 +105,14 @@ def parse_config_file(*, path: str | None = None) -> Config:
         - "pyproject.toml"
 
     Note:
-        - If the specified configuration file does not exist or is not in the correct format, an empty Config object will be returned.
+        - If the specified file is not a recognized config file name (not in CONFIG_FILES),
+          an empty Config object will be returned.
+        - If the specified file does not exist, ConfigError is raised.
+        - If the file contains invalid YAML/TOML syntax, ConfigError is raised with line/column info.
+        - If the configuration fails validation, ConfigError is raised with field details.
 
     Example:
-        parse_config_file(path="config.yaml")
+        parse_config_file(path="rdeconfig.yaml")
 
     """
     config_data: dict[str, Any] = {
@@ -50,14 +120,57 @@ def parse_config_file(*, path: str | None = None) -> Config:
         "multidata_tile": MultiDataTileSettings().model_dump(),
         "smarttable": SmartTableSettings().model_dump(),
     }
+
+    # Check file existence when path is provided
+    if path is not None:
+        path_obj = Path(path)
+        if not path_obj.exists():
+            msg = (
+                f"Configuration file not found: '{path}'. "
+                f"Create a configuration file or use 'rdetoolkit gen-config' to generate one."
+            )
+            raise ConfigError(
+                msg,
+                file_path=path,
+                error_type="file_not_found",
+            )
+
     if path is not None and Path(path).name not in CONFIG_FILES:
         return Config(system=SystemSettings(), multidata_tile=MultiDataTileSettings(), smarttable=SmartTableSettings())
 
     if path is not None and is_toml(path):
         config_data = __read_pyproject_toml(path)
     elif path is not None and is_yaml(path):
-        with open(path, encoding="utf-8") as f:
-            config_data = yaml.safe_load(f)
+        try:
+            with open(path, encoding="utf-8") as f:
+                config_data = yaml.safe_load(f)
+        except YAMLError as e:
+            # Extract line and column information from YAMLError
+            line_number = None
+            column_number = None
+
+            if hasattr(e, "problem_mark") and e.problem_mark is not None:
+                line_number = e.problem_mark.line + 1  # YAML uses 0-indexed lines
+                column_number = e.problem_mark.column + 1
+
+            error_msg = "Failed to parse YAML file: invalid syntax"
+            if hasattr(e, "problem"):
+                error_msg = f"Failed to parse YAML file: {e.problem}"
+
+            raise ConfigError(
+                error_msg,
+                file_path=path,
+                error_type="parse_error",
+                line_number=line_number,
+                column_number=column_number,
+            ) from e
+        except OSError as e:
+            msg = f"Failed to read YAML file: {e}"
+            raise ConfigError(
+                msg,
+                file_path=path,
+                error_type="io_error",
+            ) from e
     elif path is None:
         project_path = Path.cwd()
         pyproject_toml = project_path.joinpath(PYPROJECT_CONFIG_FILES[0])
@@ -68,19 +181,81 @@ def parse_config_file(*, path: str | None = None) -> Config:
     if config_data is None:
         return Config(system=SystemSettings(), multidata_tile=MultiDataTileSettings(), smarttable=SmartTableSettings())
 
-    return Config(**config_data)
+    try:
+        return Config(**config_data)
+    except ValidationError as e:
+        # Use helper to format validation error
+        if path:
+            raise _format_validation_error(e, path) from e
+        # Fallback for when path is None
+        msg = f"Configuration validation failed: {str(e)}"
+        raise ConfigError(
+            msg,
+            error_type="validation_error",
+        ) from e
+    except TypeError as e:
+        # This occurs when config_data is not a mapping (e.g., list or string),
+        # which makes Config(**config_data) invalid. Normalize it to ConfigError.
+        base_msg = "Configuration data must be a mapping (key-value structure)"
+        msg = f"{base_msg} in file '{path}'" if path else base_msg
+        raise ConfigError(
+            msg,
+            file_path=path,
+            error_type="validation_error",
+        ) from e
 
 
 def __read_pyproject_toml(path: str) -> dict[str, Any]:
     """Read the pyproject.toml file and return the contents as a dictionary.
 
+    Args:
+        path: Path to the pyproject.toml file
+
     Returns:
         dict[str, Any]: The contents of the pyproject.toml file.
+
+    Raises:
+        ConfigError: If the file does not exist or cannot be parsed
     """
-    toml = TOMLFile(path)
-    obj = toml.read()
-    _obj = obj.unwrap()
-    return _obj.get("tool", {}).get("rdetoolkit", {})
+    # Check file existence first
+    path_obj = Path(path)
+    if not path_obj.exists():
+        msg = (
+            f"Configuration file not found: '{path}'. "
+            f"Create a pyproject.toml file with [tool.rdetoolkit] section."
+        )
+        raise ConfigError(
+            msg,
+            file_path=path,
+            error_type="file_not_found",
+        )
+
+    try:
+        toml = TOMLFile(path)
+        obj = toml.read()
+        _obj = obj.unwrap()
+        return _obj.get("tool", {}).get("rdetoolkit", {})
+    except TOMLKitError as e:
+        # Extract line information if available
+        line_number = None
+        if hasattr(e, "line"):
+            line_number = e.line
+
+        error_msg = f"Failed to parse TOML file: {str(e)}"
+
+        raise ConfigError(
+            error_msg,
+            file_path=path,
+            error_type="parse_error",
+            line_number=line_number,
+        ) from e
+    except OSError as e:
+        msg = f"Failed to read TOML file: {e}"
+        raise ConfigError(
+            msg,
+            file_path=path,
+            error_type="io_error",
+        ) from e
 
 
 def is_toml(filename: str) -> bool:
@@ -153,27 +328,32 @@ def get_config(target_dir_path: RdeFsPath) -> Config | None:
 
     Returns:
         Optional[Config]: The first valid configuration found, or None if no valid configuration is found.
+
+    Raises:
+        ConfigError: If the target directory does not exist.
     """
     if isinstance(target_dir_path, str):
         target_dir_path = Path(target_dir_path)
     if not target_dir_path.exists():
-        return None
+        msg = (
+            f"Configuration directory not found: '{target_dir_path}'. "
+            f"Ensure the directory exists and contains a valid configuration file."
+        )
+        raise ConfigError(
+            msg,
+            file_path=str(target_dir_path),
+            error_type="directory_not_found",
+        )
     for cfg_file in find_config_files(target_dir_path):
-        try:
-            __config = parse_config_file(path=cfg_file)
-        except ValidationError as e:
-            emsg = f"Invalid configuration file: {cfg_file}"
-            raise ValueError(emsg) from e
+        # parse_config_file now converts ValidationError to ConfigError internally
+        __config = parse_config_file(path=cfg_file)
         if __config is not None:
             return __config
 
     pyproject_toml_path = get_pyproject_toml()
     if pyproject_toml_path is not None:
-        try:
-            __config = parse_config_file(path=str(pyproject_toml_path))
-        except ValidationError as e:
-            emsg = f"Invalid configuration file: {pyproject_toml_path}"
-            raise ValueError(emsg) from e
+        # parse_config_file now converts ValidationError to ConfigError internally
+        __config = parse_config_file(path=str(pyproject_toml_path))
         if __config is not None:
             return __config
     return None
@@ -194,8 +374,16 @@ def load_config(tasksupport_path: RdeFsPath, *, config: Config | None = None) ->
     if config is not None:
         __config = config
     else:
-        __rtn_config = get_config(tasksupport_path)
-        __config = Config() if __rtn_config is None else __rtn_config
+        try:
+            __rtn_config = get_config(tasksupport_path)
+            __config = Config() if __rtn_config is None else __rtn_config
+        except ConfigError as e:
+            # Only swallow directory_not_found errors for backward compatibility
+            # Re-raise other ConfigError types (parse_error, validation_error, etc.)
+            if getattr(e, "error_type", None) == "directory_not_found":
+                __config = Config()
+            else:
+                raise
     return __config