lograder 0.0.2__py3-none-any.whl

lograder/__init__.py

@@ -0,0 +1,2 @@
+def hello_world() -> str:
+    return "Hello world! ~from `lograder`!"

lograder/_core_exceptions.py

@@ -0,0 +1,7 @@
+class LograderError(Exception):
+    """
+    This is the base exception class for all exceptions raised
+    by the `lograder` module, for easy error handling.
+    """
+
+    pass

lograder/common/types.py

@@ -0,0 +1,5 @@
+from os import PathLike
+from pathlib import Path
+from typing import Union
+
+FilePath = Union[str, PathLike[str], Path]

lograder/common/utils.py

@@ -0,0 +1,6 @@
+import random
+import string
+
+
+def random_name(length=25) -> str:
+    return "".join(random.choices(string.ascii_uppercase + string.digits, k=length))

lograder/dispatch/__init__.py

@@ -0,0 +1,9 @@
+from . import common, cpp, misc
+from .common import *  # noqa
+from .cpp import *  # noqa
+from .misc import *  # noqa
+
+__all__ = []
+for _mod in (common, cpp, misc):
+    if hasattr(_mod, "__all__"):
+        __all__.extend(_mod.__all__)

lograder/dispatch/_core_exceptions.py

@@ -0,0 +1,10 @@
+from .._core_exceptions import LograderError
+
+
+class LograderStudentBuildError(LograderError):
+    """
+    This is the base exception class for all exceptions raised
+    by the `lograder.dispatch` module, for easy error handling.
+    """
+
+    pass

lograder/dispatch/common/__init__.py

@@ -0,0 +1,30 @@
+from .assignment import AssignmentSummary, BuilderOutput, PreprocessorOutput
+from .interface import (
+    BuilderInterface,
+    DispatcherInterface,
+    ExecutableBuildResults,
+    PreprocessorInterface,
+    PreprocessorResults,
+    RunnerInterface,
+    RuntimePrepResults,
+    RuntimeResults,
+)
+from .templates import CLIBuilder, ExecutableRunner, TrivialBuilder, TrivialPreprocessor
+
+__all__ = [
+    "TrivialBuilder",
+    "TrivialPreprocessor",
+    "AssignmentSummary",
+    "CLIBuilder",
+    "ExecutableRunner",
+    "BuilderInterface",
+    "PreprocessorInterface",
+    "RunnerInterface",
+    "DispatcherInterface",
+    "BuilderOutput",
+    "ExecutableBuildResults",
+    "PreprocessorResults",
+    "PreprocessorOutput",
+    "RuntimePrepResults",
+    "RuntimeResults",
+]

lograder/dispatch/common/assignment.py

@@ -0,0 +1,99 @@
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
+
+from ...output.formatters.default import (
+    DefaultBuildOutputFormatter,
+    DefaultExecutableTestCaseFormatter,
+    DefaultMetadataFormatter,
+    DefaultPreprocessorOutputFormatter,
+    DefaultRuntimeSummaryFormatter,
+)
+from ...output.formatters.interfaces import (
+    BuildOutputFormatterInterface,
+    ExecutableTestFormatterInterface,
+    MetadataFormatterInterface,
+    PreprocessorOutputFormatterInterface,
+    RuntimeSummaryFormatterInterface,
+)
+from ...output.raw_json.assignment import AssignmentJSON
+from ...output.raw_json.test_case import TestCaseJSON
+from ...tests.test import ExecutableTestInterface
+from .types import AssignmentMetadata, BuilderOutput, PreprocessorOutput
+
+
+class AssignmentSummary(BaseModel):
+    model_config = {"arbitrary_types_allowed": True}
+
+    metadata: AssignmentMetadata
+    preprocessor_output: PreprocessorOutput
+    build_output: BuilderOutput
+    test_cases: List[ExecutableTestInterface]
+
+    metadata_fmt: MetadataFormatterInterface = Field(
+        default_factory=DefaultMetadataFormatter, exclude=True
+    )
+    preprocessor_output_fmt: PreprocessorOutputFormatterInterface = Field(
+        default_factory=DefaultPreprocessorOutputFormatter, exclude=True
+    )
+    build_output_fmt: BuildOutputFormatterInterface = Field(
+        default_factory=DefaultBuildOutputFormatter, exclude=True
+    )
+    runtime_summary_fmt: RuntimeSummaryFormatterInterface = Field(
+        default_factory=DefaultRuntimeSummaryFormatter, exclude=True
+    )
+    test_case_fmt: ExecutableTestFormatterInterface = Field(
+        default_factory=DefaultExecutableTestCaseFormatter, exclude=True
+    )
+
+    @classmethod
+    def set_formatters(
+        cls,
+        *,
+        metadata: Optional[MetadataFormatterInterface] = None,
+        preprocessor_output: Optional[PreprocessorOutputFormatterInterface] = None,
+        build_output: Optional[BuildOutputFormatterInterface] = None,
+        runtime_summary: Optional[RuntimeSummaryFormatterInterface] = None,
+        test_case: Optional[ExecutableTestFormatterInterface] = None,
+    ):
+        if metadata is not None:
+            cls.metadata_fmt = metadata
+        if preprocessor_output is not None:
+            cls.preprocessor_output_fmt = preprocessor_output
+        if build_output is not None:
+            cls.build_output_fmt = build_output
+        if runtime_summary is not None:
+            cls.runtime_summary_fmt = runtime_summary
+        if test_case is not None:
+            cls.test_case_fmt = test_case
+
+    def get_assignment_text(self):
+        return (
+            f"\n{self.metadata_fmt.format(self.metadata)}"
+            f"{self.preprocessor_output_fmt.format(self.preprocessor_output)}\n\n"
+            f"{self.build_output_fmt.format(self.build_output)}\n\n"
+            f"{self.runtime_summary_fmt.format(self.test_cases)}\n\n"
+        )
+
+    def get_score_multiplier(self):
+        total_score = sum([test_case.get_weight() for test_case in self.test_cases])
+        return 100.0 / total_score if total_score else 0.0
+
+    def get_raw(self) -> AssignmentJSON:
+        return AssignmentJSON(
+            output=self.get_assignment_text(),
+            visibility="visible",
+            tests=[
+                TestCaseJSON(
+                    name=test_case.get_name(),
+                    output=self.test_case_fmt.format(test_case),
+                    score=self.get_score_multiplier()
+                    * test_case.get_weight()
+                    * test_case.get_successful()
+                    * test_case.get_penalty(),
+                    max_score=self.get_score_multiplier() * test_case.get_weight(),
+                )
+                for test_case in self.test_cases
+            ],
+            leaderboard=None,  # TODO: Add leaderboard support.
+        )

lograder/dispatch/common/exceptions.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+from typing import List
+
+from .._core_exceptions import LograderStudentBuildError
+
+
+class CxxSourceBuildError(LograderStudentBuildError):
+    def __init__(self, sources: List[Path]):
+        super().__init__(
+            f"Was unable to compile C++ sources; found source files: {', '.join([str(path.resolve()) for path in sources])}."
+        )
+
+
+class CMakeListsNotFoundError(LograderStudentBuildError):
+    def __init__(self):
+        super().__init__("Could not find a `CMakeLists.txt` anywhere in the project.")
+
+
+class CMakeTargetNotFoundError(LograderStudentBuildError):
+    def __init__(self, targets: List[str], cmake_path: Path):
+        super().__init__(
+            f"Could not find a valid cmake target anywhere in `{cmake_path.resolve()}`. The targets found were: [{', '.join(targets)}]."
+        )
+
+
+class CMakeExecutableNotFoundError(LograderStudentBuildError):
+    def __init__(self, cmake_path: Path):
+        super().__init__(
+            f"Could not find a valid cmake executable output path anywhere in `{cmake_path.resolve()}`."
+        )
+
+
+class MakefileNotFoundError(LograderStudentBuildError):
+    def __init__(self):
+        super().__init__("Could not find a `Makefile` anywhere in the project.")
+
+
+class MakefileRunNotFoundError(LograderStudentBuildError):
+    def __init__(self, makefile_path: Path):
+        super().__init__(
+            f"Could not find an `run` entrypoint in `{makefile_path.resolve()}` (i.e. the command `make run ARGS=<test-args>` is used to run the project)."
+        )

lograder/dispatch/common/file_operations.py

@@ -0,0 +1,118 @@
+import subprocess
+import sys
+from collections import deque
+from pathlib import Path
+from typing import List, Optional
+
+from ...static.basicconfig import LograderBasicConfig
+
+
+def bfs_walk(root: Path):  # pathlib defaults to dfs; must implement bfs ourselves.
+    queue = deque([root])
+    while queue:
+        current = queue.popleft()
+        if current.is_dir():
+            for child in current.iterdir():
+                queue.append(child)
+        else:
+            yield current
+
+
+def is_cxx_source_file(path: Path) -> bool:
+    return path.exists() and path.suffix in (
+        ".cc",
+        ".cp",
+        ".cxx",
+        ".cpp",
+        ".CPP",
+        ".c++",
+        ".C",
+        ".c",
+    )
+
+
+def is_cmake_file(path: Path) -> bool:
+    return path.exists() and path.name.startswith("CMakeLists.txt")
+
+
+def is_makefile_file(path: Path) -> bool:
+    return path.exists() and path.name == "Makefile"
+
+
+def is_makefile_target(makefile: Path, target: str) -> bool:
+    if not is_makefile_file(makefile):
+        return False
+    proc = subprocess.run(
+        ["make", "-qp"], cwd=makefile.parent, capture_output=True, text=True
+    )
+    for line in proc.stdout.splitlines():
+        if line.strip().startswith(f"{target}:"):
+            return True
+    return False
+
+
+def is_valid_target(target: str) -> bool:
+    if target in (
+        "all",
+        "install",
+        "depend",
+        "test",
+        "package",
+        "package_source",
+        "edit_cache",
+        "rebuild_cache",
+        "clean",
+        "help",
+        "ALL_BUILD",
+        "ZERO_CHECK",
+        "INSTALL",
+        "RUN_TESTS",
+        "PACKAGE",
+    ):
+        return False
+    if target.endswith(".obj") or target.endswith(".i") or target.endswith(".s"):
+        return False
+    return True
+
+
+def do_process(args: List[str | Path], **kwargs) -> subprocess.CompletedProcess:
+    win_prefix: List[str | Path] = ["cmd", "/c"]
+    cmd: List[str | Path] = args
+    if sys.platform.startswith("win"):
+        cmd = win_prefix + cmd
+    return subprocess.run(cmd, **kwargs)
+
+
+def run_cmd(
+    cmd: List[str | Path],
+    commands: Optional[List[List[str | Path]]] = None,
+    stdout: Optional[List[str]] = None,
+    stderr: Optional[List[str]] = None,
+    working_directory: Optional[Path] = None,
+):
+
+    if working_directory is None:
+        result = do_process(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            timeout=LograderBasicConfig.DEFAULT_EXECUTABLE_TIMEOUT,
+        )
+    else:
+        result = do_process(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            timeout=LograderBasicConfig.DEFAULT_EXECUTABLE_TIMEOUT,
+            cwd=working_directory,
+        )
+
+    if commands is not None:
+        commands.append(cmd)
+    if stdout is not None:
+        stdout.append(result.stdout)
+    if stderr is not None:
+        stderr.append(result.stderr)
+    return result

lograder/dispatch/common/interface.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+import json
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import List, Optional, Sequence
+
+from ...common.types import FilePath
+from ...static import LograderBasicConfig, LograderMessageConfig
+from ...tests.common.exceptions import TestNotRunError
+from ...tests.test import ExecutableTestInterface
+from ..common.assignment import AssignmentSummary, BuilderOutput, PreprocessorOutput
+from .types import AssignmentMetadata
+
+
+class PreprocessorResults:
+    def __init__(self, output: PreprocessorOutput):
+        # The reason why this is a class is in case we want to "expand" later.
+        self._output = output
+
+    def get_output(self) -> PreprocessorOutput:
+        return self._output
+
+
+class BuildResults:
+    def __init__(self, output: BuilderOutput):
+        self._output = output
+
+    def get_output(self) -> BuilderOutput:
+        return self._output
+
+
+class ExecutableBuildResults(BuildResults):
+    def __init__(self, executable: FilePath, output: BuilderOutput):
+        super().__init__(output)
+        self._executable = Path(executable)
+
+    def get_executable(self) -> Path:
+        return self._executable
+
+
+class RuntimePrepResults:  # dummy class to allow distinguishing between pre- and post-run.
+    def __init__(
+        self,
+        results: (
+            RuntimePrepResults | RuntimeResults | Sequence[ExecutableTestInterface]
+        ),
+    ):
+        if isinstance(results, RuntimePrepResults):
+            self._results = results.get_test_cases()
+        else:
+            self._results = list(results)
+
+    def get_test_cases(self) -> List[ExecutableTestInterface]:
+        return self._results
+
+
+class RuntimeResults(RuntimePrepResults):
+    def get_test_cases(self) -> List[ExecutableTestInterface]:
+        for result in self._results:
+            if not result.is_executed():
+                raise TestNotRunError(result.get_name())
+        return self._results
+
+
+class ProcessInterface(ABC):
+    _linked_preprocessors: set[PreprocessorInterface] = set()
+    _linked_builders: set[BuilderInterface] = set()
+    _linked_runners: set[RunnerInterface] = set()
+
+    @classmethod
+    def register_builder(cls, builder: BuilderInterface):
+        cls._linked_builders.add(builder)
+
+    @classmethod
+    def register_preprocessor(cls, preprocessor: PreprocessorInterface):
+        cls._linked_preprocessors.add(preprocessor)
+
+    @classmethod
+    def register_runner(cls, runner: RunnerInterface):
+        cls._linked_runners.add(runner)
+
+    @classmethod
+    def is_build_successful(cls) -> bool:
+        for builder in cls._linked_builders:
+            if builder.is_build_error():
+                return False
+        return True
+
+    @classmethod
+    def is_preprocessor_successful(cls) -> bool:
+        return cls.get_validation_multiplier() == 1.0
+
+    @classmethod
+    def get_validation_multiplier(cls) -> float:
+        scores = [prep.get_validation_penalty() for prep in cls._linked_preprocessors]
+        score = 1.0
+        for score in scores:
+            score *= score
+        return score
+
+    def __eq__(self, other):
+        return id(self) == id(
+            other
+        )  # default "is" comparison, but it's here in case it changes.
+
+    def __hash__(self):
+        return hash(id(self))
+
+
+class PreprocessorInterface(ProcessInterface, ABC):
+    def __init__(self):
+        super().__init__()
+        ProcessInterface.register_preprocessor(self)
+        self._validation_penalty: float = 0.0
+
+    def set_validation_penalty(self, penalty: float):
+        self._validation_penalty = penalty
+
+    def get_validation_penalty(self) -> float:
+        return self._validation_penalty
+
+    def get_individual_validation_multiplier(self) -> float:
+        return 1.0 - min(max(self.get_validation_penalty(), 0.0), 1.0)
+
+    def check(self):
+        if self.validate():
+            return
+        self.set_validation_penalty(1.0)
+
+    @abstractmethod
+    def validate(self) -> bool:
+        pass
+
+    @abstractmethod
+    def preprocess(self) -> PreprocessorResults:
+        pass
+
+
+class BuilderInterface(ProcessInterface, ABC):
+    def __init__(self):
+        super().__init__()
+        ProcessInterface.register_builder(self)
+        self._build_error: bool = False
+
+    def set_build_error(self, build_error: bool):
+        self._build_error = build_error
+
+    def is_build_error(self) -> bool:
+        return self._build_error
+
+    @abstractmethod
+    def build(self) -> BuildResults:
+        pass
+
+
+class ExecutableBuilderInterface(BuilderInterface, ABC):
+    @abstractmethod
+    def build(self) -> ExecutableBuildResults:
+        pass
+
+
+class RunnerInterface(ProcessInterface, ABC):
+    def __init__(self):
+        super().__init__()
+        ProcessInterface.register_runner(self)
+        self._wrap_args: bool = False
+        self._cwd: Optional[Path] = None
+
+    def set_cwd(self, cwd: Path):
+        self._cwd = cwd
+
+    def set_wrap_args(self, wrap_args: bool = True):
+        self._wrap_args = wrap_args
+
+    def run_tests_auto(self) -> RuntimeResults:
+        results: RuntimePrepResults = self.prep_tests()
+        for test_case in results.get_test_cases():
+            if not self.is_build_successful():
+                test_case.force_unsuccessful()
+                test_case.override_output(
+                    LograderMessageConfig.DEFAULT_BUILD_ERROR_OVERRIDE_MESSAGE,
+                    LograderMessageConfig.DEFAULT_BUILD_ERROR_OVERRIDE_MESSAGE,
+                )
+            else:
+                test_case.run(wrap_args=self._wrap_args, working_directory=self._cwd)
+        return RuntimeResults(results)
+
+    @abstractmethod
+    def prep_tests(self) -> RuntimePrepResults:
+        pass
+
+
+class DispatcherInterface(ABC):
+    def run(
+        self, out_path: Path = LograderBasicConfig.DEFAULT_RESULT_PATH
+    ) -> AssignmentSummary:
+        metadata = self.metadata()
+        prep = self.preprocess()
+        build = self.build()
+        runtime_results = self.run_tests()
+
+        summary = AssignmentSummary(
+            metadata=metadata,
+            preprocessor_output=prep.get_output(),
+            build_output=build.get_output(),
+            test_cases=runtime_results.get_test_cases(),
+        )
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        out_path.write_text(json.dumps(summary.model_dump()))
+        return summary
+
+    @abstractmethod
+    def metadata(self) -> AssignmentMetadata:
+        pass
+
+    @abstractmethod
+    def preprocess(self) -> PreprocessorResults:
+        pass
+
+    @abstractmethod
+    def build(self) -> ExecutableBuildResults:
+        pass
+
+    @abstractmethod
+    def run_tests(self) -> RuntimeResults:
+        pass

lograder/dispatch/common/templates/__init__.py

@@ -0,0 +1,10 @@
+from .cli_builder import CLIBuilder
+from .executable_runner import ExecutableRunner
+from .trivial import TrivialBuilder, TrivialPreprocessor
+
+__all__ = [
+    "ExecutableRunner",
+    "CLIBuilder",
+    "TrivialPreprocessor",
+    "TrivialBuilder",
+]

lograder/dispatch/common/templates/cli_builder.py

@@ -0,0 +1,59 @@
+from abc import ABC
+from pathlib import Path
+from typing import List, Optional
+
+from ....static import LograderMessageConfig
+from ..file_operations import run_cmd
+from ..interface import BuilderInterface, BuilderOutput, ExecutableBuildResults
+from ..types import ProjectType
+
+
+class CLIBuilder(BuilderInterface, ABC):
+    def __init__(self, build_type: ProjectType):
+        super().__init__()
+        self._commands: List[List[str | Path]] = []
+        self._stdout: List[str] = []
+        self._stderr: List[str] = []
+        self._build_type = build_type
+
+    def get_commands(self) -> List[List[str | Path]]:
+        return self._commands
+
+    def get_build_type(self) -> ProjectType:
+        return self._build_type
+
+    def get_stdout(self) -> List[str]:
+        return self._stdout
+
+    def get_stderr(self) -> List[str]:
+        return self._stderr
+
+    def get_build_error_output(self) -> ExecutableBuildResults:
+        return ExecutableBuildResults(
+            executable=LograderMessageConfig.DEFAULT_BUILD_ERROR_EXECUTABLE_NAME,
+            output=BuilderOutput(
+                commands=self.get_commands(),
+                stdout=self.get_stdout(),
+                stderr=self.get_stderr(),
+                project_type=self.get_build_type(),
+            ),
+        )
+
+    def run_cmd(
+        self, cmd: List[str | Path], working_directory: Optional[Path] = None
+    ) -> BuilderOutput:
+        result = run_cmd(
+            cmd,
+            commands=self._commands,
+            stdout=self._stdout,
+            stderr=self._stderr,
+            working_directory=working_directory,
+        )
+        if result.returncode != 0:
+            self.set_build_error(True)
+        return BuilderOutput(
+            commands=self._commands,
+            stdout=self._stdout,
+            stderr=self._stderr,
+            project_type=self._build_type,
+        )

lograder/dispatch/common/templates/executable_runner.py

@@ -0,0 +1,23 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import List
+
+from ....tests.registry import TestRegistry
+from ....tests.test.interface import ExecutableTestInterface
+from ..interface import RunnerInterface, RuntimePrepResults
+
+
+class ExecutableRunner(RunnerInterface, ABC):
+    @abstractmethod
+    def get_executable(self) -> List[str | Path]:
+        pass
+
+    def prep_tests(self) -> RuntimePrepResults:
+        tests: List[ExecutableTestInterface] = []
+        for test in TestRegistry.iterate():
+            if isinstance(test, ExecutableTestInterface):
+                test.set_target(self.get_executable())
+                tests.append(test)
+            else:
+                raise TypeError
+        return RuntimePrepResults(tests)