manualforge 0.1.1__py3-none-any.whl

manualforge/__init__.py

@@ -0,0 +1,3 @@
+"""ManualForge — A configuration-driven management manual generation framework."""
+
+__version__ = "0.1.1"

manualforge/__main__.py

@@ -0,0 +1,25 @@
+"""manualforge file for ensuring the package is executable
+as `manualforge` and `python -m manualforge`
+"""
+
+import sys
+from pathlib import Path
+from typing import Any
+
+from kedro.framework.cli.utils import find_run_command
+from kedro.framework.project import configure_project
+
+
+def main(*args, **kwargs) -> Any:
+    package_name = Path(__file__).parent.name
+    configure_project(package_name)
+
+    interactive = hasattr(sys, "ps1")
+    kwargs["standalone_mode"] = not interactive
+
+    run = find_run_command(package_name)
+    return run(*args, **kwargs)
+
+
+if __name__ == "__main__":
+    main()

manualforge/config.py

@@ -0,0 +1,73 @@
+"""ManualForge configuration helpers.
+
+Utilities for safely reading project configuration and providing sensible
+defaults, so node functions stay clean when operating in config-driven mode.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+# Sentinel for "not set" to distinguish from explicit None.
+_UNSET = object()
+
+
+def get_datasource(config: dict, source_id: str) -> dict:
+    """Return the datasource sub-config for *source_id*."""
+    sources = config.get("datasources", {})
+    if source_id not in sources:
+        raise KeyError(f"Datasource '{source_id}' not found in config.datasources")
+    return sources[source_id]
+
+
+def get_standardization(config: dict) -> dict:
+    """Return the standardization config section."""
+    return config.get("standardization", {})
+
+
+def get_standardization_fields(config: dict) -> list[dict]:
+    """Return the list of field-standardization definitions."""
+    return get_standardization(config).get("fields", [])
+
+
+def get_summary(config: dict, summary_id: str) -> dict:
+    """Return the summary sub-config for *summary_id*."""
+    summaries = config.get("summaries", {})
+    if summary_id not in summaries:
+        raise KeyError(f"Summary '{summary_id}' not found in config.summaries")
+    return summaries[summary_id]
+
+
+def get_sort_order(config: dict, order_name: str) -> list[str]:
+    """Return a named sort-order list."""
+    return config.get("sort_orders", {}).get(order_name, [])
+
+
+def _resolve_sort_ref(config: dict, ref: str | list) -> list[str]:
+    """Resolve a sort_by value which is either a sort-order name or an inline list."""
+    if isinstance(ref, list):
+        return ref
+    if isinstance(ref, str):
+        return get_sort_order(config, ref)
+    return []
+
+
+def get_sort_list(config: dict, sort_by: dict) -> dict[str, list[str]]:
+    """Resolve a sort_by dict {column: order_name_or_list} → {column: [values]}."""
+    return {col: _resolve_sort_ref(config, ref) for col, ref in sort_by.items()}
+
+
+def get_report(config: dict, report_id: str) -> dict:
+    """Return the report sub-config for *report_id*."""
+    reports = config.get("reports", {})
+    if report_id not in reports:
+        raise KeyError(f"Report '{report_id}' not found in config.reports")
+    return reports[report_id]
+
+
+def get_project(config: dict) -> dict:
+    """Return the project metadata section."""
+    return config.get("project", {})

manualforge/hooks.py

@@ -0,0 +1,425 @@
+"""
+Custom hooks for manualforge project.
+Execute shell commands before the first node and after the last node.
+"""
+
+import logging
+import os
+import subprocess
+from typing import Any
+
+from kedro.framework.hooks import hook_impl
+
+logger = logging.getLogger(__name__)
+
+
+class PipelineHooks:
+    """
+    Hooks for executing shell commands before first node and after last node.
+
+    Commands are loaded from configuration file (conf/base/hooks.yml).
+    """
+
+    def __init__(self):
+        self.before_commands: list[str] = []
+        self.after_commands: list[str] = []
+        self.before_node_commands: dict[str, list[str]] = {}
+        self.after_node_commands: dict[str, list[str]] = {}
+        self._config_loaded = False
+
+    @hook_impl
+    def after_context_created(self, context) -> None:
+        """
+        Load configuration after context is created.
+
+        Args:
+            context: Kedro context object with access to configuration
+        """
+        try:
+            # Get configuration from context
+            # The hooks configuration should be in conf/base/hooks.yml
+            # and will be loaded as part of parameters
+            config = context.params
+
+            # Look for hooks configuration
+            hooks_config = config.get("hooks", {})
+
+            # Load pipeline-level commands
+            before_cmds = hooks_config.get("before_pipeline", [])
+            after_cmds = hooks_config.get("after_pipeline", [])
+            logger.info(before_cmds)
+            # Validate and store pipeline-level commands
+            if isinstance(before_cmds, list):
+                self.before_commands = [
+                    cmd for cmd in before_cmds if isinstance(cmd, dict)
+                ]
+                logger.info(
+                    f"Loaded {len(self.before_commands)} before-pipeline commands"
+                )
+
+            if isinstance(after_cmds, list):
+                self.after_commands = [
+                    cmd for cmd in after_cmds if isinstance(cmd, dict)
+                ]
+                logger.info(
+                    f"Loaded {len(self.after_commands)} after-pipeline commands"
+                )
+
+            # Load node-level commands
+            before_nodes_config = hooks_config.get("before_nodes", {})
+            after_nodes_config = hooks_config.get("after_nodes", {})
+
+            if isinstance(before_nodes_config, dict):
+                for node_name, cmds in before_nodes_config.items():
+                    if isinstance(cmds, list):
+                        valid_cmds = [cmd for cmd in cmds if isinstance(cmd, dict)]
+                        if valid_cmds:
+                            self.before_node_commands[node_name] = valid_cmds
+                            logger.info(
+                                f"Loaded {len(valid_cmds)} before-node commands for '{node_name}'"
+                            )
+
+            if isinstance(after_nodes_config, dict):
+                for node_name, cmds in after_nodes_config.items():
+                    if isinstance(cmds, list):
+                        valid_cmds = [cmd for cmd in cmds if isinstance(cmd, dict)]
+                        if valid_cmds:
+                            self.after_node_commands[node_name] = valid_cmds
+                            logger.info(
+                                f"Loaded {len(valid_cmds)} after-node commands for '{node_name}'"
+                            )
+
+            self._config_loaded = True
+
+            # Log summary
+            if self.before_commands:
+                logger.debug(f"Before-pipeline commands: {self.before_commands}")
+            if self.after_commands:
+                logger.debug(f"After-pipeline commands: {self.after_commands}")
+            if self.before_node_commands:
+                logger.debug(
+                    f"Before-node commands configured for nodes: {list(self.before_node_commands.keys())}"
+                )
+            if self.after_node_commands:
+                logger.debug(
+                    f"After-node commands configured for nodes: {list(self.after_node_commands.keys())}"
+                )
+
+        except Exception as e:
+            logger.error(f"Failed to load hook configuration: {e}")
+            self._config_loaded = False
+
+    @hook_impl
+    def before_pipeline_run(self, run_params: dict[str, Any]) -> None:
+        """
+        Execute commands before the first node runs.
+
+        Args:
+            run_params: Dictionary of pipeline run parameters
+        """
+        if not self._config_loaded:
+            logger.warning(
+                "Hook configuration not loaded, skipping before-pipeline commands"
+            )
+            return
+
+        if not self.before_commands:
+            logger.info("No before-pipeline commands configured")
+            return
+
+        logger.info(f"Executing {len(self.before_commands)} before-pipeline commands")
+        logger.info(f"Current working directory: {os.getcwd()}")
+
+        for i, cmd in enumerate(self.before_commands, 1):
+            try:
+                logger.info(f"Running command {i}/{len(self.before_commands)}: {cmd}")
+                logger.info(f"{i}")
+                logger.info(f"{cmd}")
+                result = subprocess.run(
+                    cmd["cmd"],
+                    shell=True,
+                    check=True,
+                    cwd=cmd["cwd"],
+                    capture_output=True,
+                    text=True,
+                    timeout=300,  # 5 minutes timeout
+                )
+
+                logger.info(f"Command {i}/{len(self.before_commands)} succeeded")
+
+                # Log output if not empty
+                if result.stdout and result.stdout.strip():
+                    logger.info(f"Command output: {result.stdout.strip()}")
+                if result.stderr and result.stderr.strip():
+                    logger.warning(f"Command stderr: {result.stderr.strip()}")
+
+            except subprocess.CalledProcessError as e:
+                logger.error(f"Command failed with exit code {e.returncode}: {cmd}")
+                logger.error(
+                    f"Error output: {e.stderr[:500] if e.stderr else 'No error output'}"
+                )
+
+                # Re-raise to stop pipeline execution
+                raise RuntimeError(f"Before-pipeline hook command failed: {cmd}") from e
+
+            except subprocess.TimeoutExpired:
+                logger.error(f"Command timed out after 300 seconds: {cmd}")
+                raise RuntimeError(f"Before-pipeline hook command timed out: {cmd}")
+
+            except Exception as e:
+                logger.error(f"Unexpected error executing command: {cmd}")
+                logger.error(f"Error: {e}")
+                raise
+
+    @hook_impl
+    def after_pipeline_run(self, run_params: dict[str, Any]) -> None:
+        """
+        Execute commands after the last node runs.
+
+        Args:
+            run_params: Dictionary of pipeline run parameters
+        """
+        if not self._config_loaded:
+            logger.warning(
+                "Hook configuration not loaded, skipping after-pipeline commands"
+            )
+            return
+
+        if not self.after_commands:
+            logger.info("No after-pipeline commands configured")
+            return
+
+        logger.info(f"Executing {len(self.after_commands)} after-pipeline commands")
+        logger.info(f"Current working directory: {os.getcwd()}")
+
+        for i, cmd in enumerate(self.after_commands, 1):
+            try:
+                logger.info(f"Running command {i}/{len(self.after_commands)}: {cmd}")
+                logger.info(f"{i}")
+                logger.info(f"{cmd}")
+
+                # Execute shell command
+                result = subprocess.run(
+                    cmd["cmd"],
+                    shell=True,
+                    check=True,
+                    cwd=cmd["cwd"],
+                    capture_output=True,
+                    text=True,
+                    timeout=300,  # 5 minutes timeout
+                )
+
+                logger.info(f"Command {i}/{len(self.after_commands)} succeeded")
+
+                # Log output if not empty
+                if result.stdout and result.stdout.strip():
+                    logger.info(f"Command output: {result.stdout.strip()}")
+                if result.stderr and result.stderr.strip():
+                    logger.warning(f"Command stderr: {result.stderr.strip()}")
+
+            except subprocess.CalledProcessError as e:
+                logger.error(f"Command failed with exit code {e.returncode}: {cmd}")
+                logger.error(
+                    f"Error output: {e.stderr[:500] if e.stderr else 'No error output'}"
+                )
+
+                # Re-raise to ensure pipeline failure is known
+                raise RuntimeError(f"After-pipeline hook command failed: {cmd}") from e
+
+            except subprocess.TimeoutExpired:
+                logger.error(f"Command timed out after 300 seconds: {cmd}")
+                raise RuntimeError(f"After-pipeline hook command timed out: {cmd}")
+
+            except Exception as e:
+                logger.error(f"Unexpected error executing command: {cmd}")
+                logger.error(f"Error: {e}")
+                raise
+
+    @hook_impl
+    def before_node_run(self, node, catalog, inputs, is_async):
+        """
+        Execute commands before a specific node runs.
+
+        Args:
+            node: The node about to be executed
+            catalog: Data catalog
+            inputs: Inputs to the node
+            is_async: Whether node execution is async
+        """
+        if not self._config_loaded:
+            logger.warning(
+                "Hook configuration not loaded, skipping before-node commands"
+            )
+            return
+
+        node_name = node.name
+        if node_name not in self.before_node_commands:
+            return
+
+        commands = self.before_node_commands[node_name]
+        if not commands:
+            return
+
+        logger.info(f"Executing {len(commands)} before-node commands for '{node_name}'")
+        logger.info(f"Current working directory: {os.getcwd()}")
+
+        for i, cmd in enumerate(commands, 1):
+            try:
+                logger.info(
+                    f"Running before-node command {i}/{len(commands)} for '{node_name}': {cmd}"
+                )
+
+                # Execute shell command
+                result = subprocess.run(
+                    cmd["cmd"],
+                    shell=True,
+                    check=True,
+                    cwd=cmd["cwd"],
+                    capture_output=True,
+                    text=True,
+                    timeout=300,  # 5 minutes timeout
+                )
+
+                logger.info(
+                    f"Before-node command {i}/{len(commands)} for '{node_name}' succeeded"
+                )
+
+                # Log output if not empty
+                if result.stdout and result.stdout.strip():
+                    logger.info(
+                        f"Command output for node '{node_name}': {result.stdout.strip()}"
+                    )
+                if result.stderr and result.stderr.strip():
+                    logger.warning(
+                        f"Command stderr for node '{node_name}': {result.stderr.strip()}"
+                    )
+
+            except subprocess.CalledProcessError as e:
+                logger.error(
+                    f"Before-node command failed with exit code {e.returncode} for '{node_name}': {cmd}"
+                )
+                logger.error(
+                    f"Error output: {e.stderr[:500] if e.stderr else 'No error output'}"
+                )
+
+                # Re-raise to stop node execution
+                raise RuntimeError(
+                    f"Before-node hook command failed for '{node_name}': {cmd}"
+                ) from e
+
+            except subprocess.TimeoutExpired:
+                logger.error(
+                    f"Before-node command timed out after 300 seconds for '{node_name}': {cmd}"
+                )
+                raise RuntimeError(
+                    f"Before-node hook command timed out for '{node_name}': {cmd}"
+                )
+
+            except Exception as e:
+                logger.error(
+                    f"Unexpected error executing before-node command for '{node_name}': {cmd}"
+                )
+                logger.error(f"Error: {e}")
+                raise
+
+    @hook_impl
+    def after_node_run(self, node, catalog, inputs, outputs, is_async):
+        """
+        Execute commands after a specific node runs.
+
+        Args:
+            node: The node that was executed
+            catalog: Data catalog
+            inputs: Inputs to the node
+            outputs: Outputs from the node
+            is_async: Whether node execution was async
+        """
+        if not self._config_loaded:
+            logger.warning(
+                "Hook configuration not loaded, skipping after-node commands"
+            )
+            return
+
+        node_name = node.name
+        if node_name not in self.after_node_commands:
+            return
+
+        commands = self.after_node_commands[node_name]
+        if not commands:
+            return
+
+        logger.info(f"Executing {len(commands)} after-node commands for '{node_name}'")
+        logger.info(f"Current working directory: {os.getcwd()}")
+
+        for i, cmd in enumerate(commands, 1):
+            try:
+                logger.info(
+                    f"Running after-node command {i}/{len(commands)} for '{node_name}': {cmd}"
+                )
+
+                # Execute shell command
+                result = subprocess.run(
+                    cmd["cmd"],
+                    shell=True,
+                    check=True,
+                    cwd=cmd["cwd"],
+                    capture_output=True,
+                    text=True,
+                    timeout=300,  # 5 minutes timeout
+                )
+
+                logger.info(
+                    f"After-node command {i}/{len(commands)} for '{node_name}' succeeded"
+                )
+
+                # Log output if not empty
+                if result.stdout and result.stdout.strip():
+                    logger.info(
+                        f"Command output for node '{node_name}': {result.stdout.strip()}"
+                    )
+                if result.stderr and result.stderr.strip():
+                    logger.warning(
+                        f"Command stderr for node '{node_name}': {result.stderr.strip()}"
+                    )
+
+            except subprocess.CalledProcessError as e:
+                logger.error(
+                    f"After-node command failed with exit code {e.returncode} for '{node_name}': {cmd}"
+                )
+                logger.error(
+                    f"Error output: {e.stderr[:500] if e.stderr else 'No error output'}"
+                )
+
+                # Re-raise to mark pipeline as failed
+                raise RuntimeError(
+                    f"After-node hook command failed for '{node_name}': {cmd}"
+                ) from e
+
+            except subprocess.TimeoutExpired:
+                logger.error(
+                    f"After-node command timed out after 300 seconds for '{node_name}': {cmd}"
+                )
+                raise RuntimeError(
+                    f"After-node hook command timed out for '{node_name}': {cmd}"
+                )
+
+            except Exception as e:
+                logger.error(
+                    f"Unexpected error executing after-node command for '{node_name}': {cmd}"
+                )
+                logger.error(f"Error: {e}")
+                raise
+
+    @hook_impl
+    def on_pipeline_error(self, error: Exception, run_params: dict[str, Any]) -> None:
+        """
+        Handle pipeline errors.
+
+        Args:
+            error: Exception that caused pipeline to fail
+            run_params: Dictionary of pipeline run parameters
+        """
+        logger.error(f"Pipeline failed with error: {error}")
+
+        # You could add error handling commands here if needed
+        # For example, cleanup commands even on failure

manualforge/io/__init__.py

@@ -0,0 +1,7 @@
+"""
+自定义数据集实现。
+"""
+
+from .polars_excel_dataset import PolarsExcelDataset
+
+__all__ = ["PolarsExcelDataset"]

manualforge/io/polars_excel_dataset.py

@@ -0,0 +1,144 @@
+"""
+自定义Polars Excel数据集，支持使用polars读取Excel文件。
+支持读取单个工作表或多个工作表。
+"""
+
+from pathlib import Path
+from typing import Any, Union
+
+import polars as pl
+from kedro.io import AbstractDataset
+from kedro.io.core import DatasetError
+
+
+class PolarsExcelDataset(
+    AbstractDataset[
+        Union[pl.DataFrame, dict[str, pl.DataFrame]],
+        Union[pl.DataFrame, dict[str, pl.DataFrame]],
+    ]
+):
+    """``PolarsExcelDataset`` 使用polars库加载和保存Excel文件。
+
+    根据`sheet_name`参数，可以返回单个DataFrame或工作表名称到DataFrame的字典。
+
+    示例:
+    ::
+
+        >>> # 读取单个工作表
+        >>> dataset = PolarsExcelDataset(
+        >>>     filepath="/path/to/file.xlsx",
+        >>>     load_args={"has_header": True, "sheet_name": "Sheet1"}
+        >>> )
+        >>> dataframe = dataset.load()  # 返回单个DataFrame
+        >>>
+        >>> # 读取所有工作表
+        >>> dataset = PolarsExcelDataset(
+        >>>     filepath="/path/to/file.xlsx",
+        >>>     load_args={"has_header": True, "sheet_name": None}
+        >>> )
+        >>> sheets_dict = dataset.load()  # 返回Dict[str, DataFrame]
+
+    """
+
+    def __init__(
+        self,
+        filepath: str,
+        load_args: dict[str, Any] | None = None,
+        save_args: dict[str, Any] | None = None,
+    ) -> None:
+        """创建PolarsExcelDataset的新实例。
+
+        Args:
+            filepath: Excel文件的路径。
+            load_args: 传递给polars.read_excel()的选项。
+                支持的参数包括：sheet_id, sheet_name, engine, has_header,
+                xlsx2csv_options, read_options, raise_if_empty,
+                infer_schema_length, schema_overrides等。
+                参见 https://pola-rs.github.io/polars/py-polars/html/reference/io.html
+            save_args: 传递给polars.DataFrame.write_excel()的选项。
+                参见 https://pola-rs.github.io/polars/py-polars/html/reference/dataframe/io.html
+
+        Raises:
+            DatasetError: 当提供无效参数时。
+        """
+        self._filepath = Path(filepath)
+        self._load_args = load_args if load_args is not None else {}
+        self._save_args = save_args if save_args is not None else {}
+
+        # 设置默认引擎为calamine（fastexcel），如果未指定
+        if "engine" not in self._load_args:
+            self._load_args["engine"] = "calamine"
+
+    def _describe(self) -> dict[str, Any]:
+        return {
+            "filepath": str(self._filepath),
+            "load_args": self._load_args,
+            "save_args": self._save_args,
+        }
+
+    def _load(self) -> pl.DataFrame | dict[str, pl.DataFrame]:
+        """从Excel文件加载数据。
+
+        Returns:
+            如果sheet_id=0或sheet_name是列表/None，返回工作表名称到DataFrame的字典；
+            否则返回单个DataFrame。
+
+        Raises:
+            DatasetError: 当加载失败时。
+        """
+        try:
+            result = pl.read_excel(self._filepath, **self._load_args)
+
+            # 帮助类型检查器理解返回类型
+            # polars.read_excel在sheet_id=0或sheet_name为列表/None时返回Dict[str, DataFrame]
+            # 否则返回DataFrame
+            return result
+        except Exception as exc:
+            raise DatasetError(f"从 {self._filepath} 加载Excel文件失败。") from exc
+
+    def _save(self, data: pl.DataFrame | dict[str, pl.DataFrame]) -> None:
+        """将数据保存到Excel文件。
+
+        Args:
+            data: 要保存的Polars DataFrame或工作表名称到DataFrame的字典。
+
+        Raises:
+            DatasetError: 当保存失败时，或当数据格式不支持时。
+        """
+        try:
+            # 确保目录存在
+            self._filepath.parent.mkdir(parents=True, exist_ok=True)
+
+            if isinstance(data, pl.DataFrame):
+                # 单个DataFrame
+                data.write_excel(self._filepath, **self._save_args)
+            elif isinstance(data, dict):
+                # 多个工作表
+                # 注意：polars的write_excel可以通过worksheet_name参数支持多个工作表
+                # 但需要检查是否所有值都是DataFrame
+                for sheet_name, df in data.items():
+                    if not isinstance(df, pl.DataFrame):
+                        raise DatasetError(
+                            f"字典值必须是DataFrame，但 '{sheet_name}' 的类型是 {type(df)}"
+                        )
+                # 目前polars的write_excel不支持直接写入多个工作表到单个文件
+                # 我们可以使用第一个DataFrame写入，然后添加其他工作表
+                # 这是一个简化实现：只保存第一个工作表
+                # 更完整的实现需要使用其他库如openpyxl或xlsxwriter
+                first_sheet_name = next(iter(data))
+                first_df = data[first_sheet_name]
+                first_df.write_excel(self._filepath, **self._save_args)
+                # 记录警告或抛出不支持的错误
+                if len(data) > 1:
+                    raise DatasetError(
+                        "当前版本不支持保存多个工作表。请提供一个DataFrame。"
+                    )
+            else:
+                raise DatasetError(
+                    f"不支持的数据类型：{type(data)}。必须是DataFrame或Dict[str, DataFrame]。"
+                )
+        except Exception as exc:
+            raise DatasetError(f"保存到 {self._filepath} 失败。") from exc
+
+    def _exists(self) -> bool:
+        return self._filepath.exists()