sinol-make 1.6.0.dev4__tar.gz → 1.7.0.dev1__tar.gz

{sinol_make-1.6.0.dev4/src/sinol_make.egg-info → sinol_make-1.7.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sinol_make
-Version: 1.6.0.dev4
+Version: 1.7.0.dev1
 Summary: CLI tool for creating sio2 task packages
 Author: Mateusz Masiarz
 Author-email: m.masiarz@fri.edu.pl
@@ -113,6 +113,9 @@ Run `sinol-make ingen --help` to see available flags.
 program to use, what tests to check and how many CPUs to use. Run `sinol-make inwer --help` to see available flags.
 - `sinol-make export` -- Creates archive ready to upload to sio2 or szkopul. Run `sinol-make export --help` to see all available flags.
 - `sinol-make doc` -- Compiles all LaTeX files in doc/ directory to PDF. Run `sinol-make doc --help` to see all available flags.
+- `sinol-make verify` -- Verifies the package. This command runs stress tests (if available), verifies the config,
+generates tests, generates problem statements, runs inwer and run all solutions. Ingen and inwer are compiled with
+address and UB sanitizers. Run `sinol-make verify --help` to see all available flags.
 - `sinol-make init [id]` -- Creates package from template [on github](https://github.com/sio2project/sinol-make/tree/main/example_package) and sets task id to provided `[id]`. Requires an internet connection to run.
 
 You can also run multiple commands at once, for example:

{sinol_make-1.6.0.dev4 → sinol_make-1.7.0.dev1}/README.md

@@ -83,6 +83,9 @@ Run `sinol-make ingen --help` to see available flags.
 program to use, what tests to check and how many CPUs to use. Run `sinol-make inwer --help` to see available flags.
 - `sinol-make export` -- Creates archive ready to upload to sio2 or szkopul. Run `sinol-make export --help` to see all available flags.
 - `sinol-make doc` -- Compiles all LaTeX files in doc/ directory to PDF. Run `sinol-make doc --help` to see all available flags.
+- `sinol-make verify` -- Verifies the package. This command runs stress tests (if available), verifies the config,
+generates tests, generates problem statements, runs inwer and run all solutions. Ingen and inwer are compiled with
+address and UB sanitizers. Run `sinol-make verify --help` to see all available flags.
 - `sinol-make init [id]` -- Creates package from template [on github](https://github.com/sio2project/sinol-make/tree/main/example_package) and sets task id to provided `[id]`. Requires an internet connection to run.
 
 You can also run multiple commands at once, for example:

{sinol_make-1.6.0.dev4 → sinol_make-1.7.0.dev1}/setup.cfg

@@ -44,6 +44,7 @@ tests =
 [options.entry_points]
 console_scripts = 
 	sinol-make = sinol_make:main
+	sm = sinol_make:main
 
 [tool:pytest]
 testpaths = 

{sinol_make-1.6.0.dev4 → sinol_make-1.7.0.dev1}/src/sinol_make/__init__.py

@@ -6,7 +6,8 @@ import argcomplete
 
 from sinol_make import util, oiejq
 
-__version__ = "1.6.0.dev4"
+
+__version__ = "1.7.0.dev1"
 
 
 def configure_parsers():
@@ -33,7 +34,7 @@ def configure_parsers():
 
 def check_oiejq():
     if util.is_linux() and not oiejq.check_oiejq():
-        print(util.warning('`oiejq` in `~/.local/bin/` not found, installing now...'))
+        print(util.warning('Up to date `oiejq` in `~/.local/bin/` not found, installing new version...'))
         try:
             if oiejq.install_oiejq():
                 print(util.info('`oiejq` was successfully installed.'))

{sinol_make-1.6.0.dev4 → sinol_make-1.7.0.dev1}/src/sinol_make/commands/export/__init__.py

@@ -119,7 +119,7 @@ class Command(BaseCommand):
             if num_tests == 0:
                 print(util.warning('No ocen tests found.'))
             elif os.path.exists(dlazaw_dir):
-                print(util.warning('Skipping ocen arhive creation because dlazaw directory exists.'))
+                print(util.warning('Skipping ocen archive creation because dlazaw directory exists.'))
             else:
                 shutil.make_archive(os.path.join(attachments_dir, f'{self.task_id}ocen'), 'zip', tmpdir)
 

{sinol_make-1.6.0.dev4 → sinol_make-1.7.0.dev1}/src/sinol_make/commands/gen/__init__.py

@@ -5,6 +5,7 @@ from sinol_make.commands.ingen import Command as IngenCommand
 from sinol_make.commands.outgen import Command as OutgenCommand
 from sinol_make.helpers import parsers
 from sinol_make.interfaces.BaseCommand import BaseCommand
+from sinol_make.task_type import get_task_type
 
 
 class Command(BaseCommand):
@@ -34,14 +35,13 @@ class Command(BaseCommand):
         parsers.add_cpus_argument(parser, 'number of cpus to use to generate output files')
         parser.add_argument('-n', '--no-validate', default=False, action='store_true',
                             help='do not validate test contents')
-        parser.add_argument('-f', '--fsanitize', default=False, action='store_true',
-                            help='Use -fsanitize=address,undefined for ingen compilation. Warning: this may fail on some '
-                                 'systems. To fix this, run `sudo sysctl vm.mmap_rnd_bits = 28`.')
+        parsers.add_fsanitize_argument(parser)
         parsers.add_compilation_arguments(parser)
         return parser
 
     def run(self, args: argparse.Namespace):
         args = util.init_package_command(args)
+        task_type = get_task_type()
 
         self.args = args
         self.ins = args.only_inputs
@@ -55,6 +55,6 @@ class Command(BaseCommand):
             command = IngenCommand()
             command.run(args)
 
-        if self.outs:
+        if self.outs and task_type.run_outgen():
             command = OutgenCommand()
             command.run(args)

{sinol_make-1.6.0.dev4 → sinol_make-1.7.0.dev1}/src/sinol_make/commands/ingen/__init__.py

@@ -31,19 +31,39 @@ class Command(BaseCommand):
         parser.add_argument('-n', '--no-validate', default=False, action='store_true',
                             help='do not validate test contents')
         parsers.add_cpus_argument(parser, 'number of cpus used for validating tests')
-        parser.add_argument('-f', '--fsanitize', default=False, action='store_true',
-                            help='Use -fsanitize=address,undefined for compilation. Warning: this may fail on some '
-                                 'systems. Tof fix this, run `sudo sysctl vm.mmap_rnd_bits = 28`.')
+        parsers.add_fsanitize_argument(parser)
         parsers.add_compilation_arguments(parser)
         return parser
 
+    def delete_dangling_files(self, dates):
+        to_delete = set()
+        for test in glob.glob(os.path.join(os.getcwd(), "in", f"{self.task_id}*.in")):
+            basename = os.path.basename(test)
+            if basename in dates and dates[basename] == os.path.getmtime(test):
+                to_delete.add(os.path.basename(test))
+        if to_delete:
+            config = package_util.get_config()
+            if 'sinol_static_tests' not in config:
+                print(util.warning('Old input files won\'t be deleted, '
+                                   'because static tests are not defined. '
+                                   'You can define them in config.yml with `sinol_static_tests` key.'))
+            else:
+                static_files = config['sinol_static_tests']
+                if isinstance(static_files, str):
+                    static_files = [static_files]
+                static_files = set([os.path.basename(test) for test in static_files])
+                to_delete = to_delete - static_files
+                if to_delete:
+                    print('Cleaning up old input files.')
+                    for test in to_delete:
+                        os.remove(os.path.join(os.getcwd(), "in", test))
+
     def run(self, args: argparse.Namespace):
         args = util.init_package_command(args)
 
         self.args = args
 
         self.task_id = package_util.get_task_id()
-        package_util.validate_test_names(self.task_id)
         util.change_stack_size_to_unlimited()
         self.ingen = get_ingen(self.task_id, args.ingen_path)
         print(f'Using ingen file {os.path.basename(self.ingen)}')
@@ -65,11 +85,7 @@ class Command(BaseCommand):
         else:
             util.exit_with_error('Failed to generate input files.')
 
-        print('Cleaning up old input files.')
-        for test in glob.glob(os.path.join(os.getcwd(), "in", f"{self.task_id}*.in")):
-            basename = os.path.basename(test)
-            if basename in dates and dates[basename] == os.path.getmtime(test):
-                os.unlink(test)
+        self.delete_dangling_files(dates)
 
         with open(paths.get_cache_path("input_tests"), "w") as f:
             f.write("\n".join(glob.glob(os.path.join(os.getcwd(), "in", f"{self.task_id}*.in"))))

{sinol_make-1.6.0.dev4 → sinol_make-1.7.0.dev1}/src/sinol_make/commands/inwer/__init__.py

@@ -37,9 +37,7 @@ class Command(BaseCommand):
         parser.add_argument('-t', '--tests', type=str, nargs='+',
                             help='test to verify, for example in/abc{0,1}*')
         parsers.add_cpus_argument(parser, 'number of cpus to use when verifying tests')
-        parser.add_argument('-f', '--fsanitize', default=False, action='store_true',
-                            help='Use -fsanitize=address,undefined for compilation. Warning: this may fail on some '
-                                 'systems. Tof fix this, run `sudo sysctl vm.mmap_rnd_bits = 28`.')
+        parsers.add_fsanitize_argument(parser)
         parsers.add_compilation_arguments(parser)
         return parser
 

{sinol_make-1.6.0.dev4 → sinol_make-1.7.0.dev1}/src/sinol_make/commands/outgen/__init__.py

@@ -10,6 +10,7 @@ from sinol_make.commands.outgen.outgen_util import get_correct_solution, compile
 from sinol_make.structs.gen_structs import OutputGenerationArguments
 from sinol_make.helpers import parsers, package_util, cache, paths
 from sinol_make.interfaces.BaseCommand import BaseCommand
+from sinol_make.task_type import get_task_type
 
 
 class Command(BaseCommand):
@@ -106,6 +107,9 @@ class Command(BaseCommand):
 
     def run(self, args: argparse.Namespace):
         args = util.init_package_command(args)
+        task_type = get_task_type()
+        if not task_type.run_outgen():
+            util.exit_with_error('This task type does not support output generation.')
 
         self.args = args
         self.task_id = package_util.get_task_id()

{sinol_make-1.6.0.dev4 → sinol_make-1.7.0.dev1}/src/sinol_make/commands/run/__init__.py

@@ -5,6 +5,7 @@ import subprocess
 import signal
 import threading
 import time
+import traceback
 import psutil
 import glob
 import shutil
@@ -17,7 +18,7 @@ import multiprocessing as mp
 from io import StringIO
 from typing import Dict
 
-from sinol_make import contest_types, oiejq, util
+from sinol_make import contest_types, oiejq, task_type, util
 from sinol_make.structs.run_structs import ExecutionData, PrintData
 from sinol_make.structs.cache_structs import CacheTest, CacheFile
 from sinol_make.interfaces.BaseCommand import BaseCommand
@@ -304,19 +305,11 @@ class Command(BaseCommand):
         parser.add_argument('--ignore-expected', dest='ignore_expected', action='store_true',
                             help='ignore expected scores from config.yml. When this flag is set, '
                                  'the expected scores are not compared with the actual scores.')
+        parser.add_argument('--no-outputs', dest='allow_no_outputs', action='store_true',
+                            help='allow running the script without full outputs')
         parsers.add_compilation_arguments(parser)
         return parser
 
-    def parse_time(self, time_str):
-        if len(time_str) < 3: return -1
-        return int(time_str[:-2])
-
-
-    def parse_memory(self, memory_str):
-        if len(memory_str) < 3: return -1
-        return int(memory_str[:-2])
-
-
     def extract_file_name(self, file_path):
         return os.path.split(file_path)[1]
 
@@ -332,9 +325,6 @@ class Command(BaseCommand):
                 return file + ext
         util.exit_with_error("Source file not found for executable %s" % executable)
 
-    def get_executables(self, args_solutions):
-        return [package_util.get_executable(solution) for solution in package_util.get_solutions(self.ID, args_solutions)]
-
 
     def get_possible_score(self, groups):
         possible_score = 0
@@ -351,17 +341,19 @@ class Command(BaseCommand):
         return sorted(list(set([self.get_group(test) for test in tests])))
 
 
-    def compile_solutions(self, solutions, is_checker=False):
+    def compile_solutions(self, solutions, remove_all_cache=False):
         os.makedirs(paths.get_compilation_log_path(), exist_ok=True)
         os.makedirs(paths.get_executables_path(), exist_ok=True)
         print("Compiling %d solutions..." % len(solutions))
-        args = [(solution, True, is_checker) for solution in solutions]
+        args = [(solution, True, remove_all_cache) for solution in solutions]
         with mp.Pool(self.cpus) as pool:
             compilation_results = pool.starmap(self.compile, args)
         return compilation_results
 
 
-    def compile(self, solution, use_extras = False, is_checker = False):
+    def compile(self, solution, use_extras=False, remove_all_cache=False):
+        os.makedirs(paths.get_compilation_log_path(), exist_ok=True)
+        os.makedirs(paths.get_executables_path(), exist_ok=True)
         compile_log_file = paths.get_compilation_log_path("%s.compile_log" % package_util.get_file_name(solution))
         source_file = os.path.join(os.getcwd(), "prog", self.get_solution_from_exe(solution))
         output = paths.get_executables_path(package_util.get_executable(solution))
@@ -382,7 +374,7 @@ class Command(BaseCommand):
         try:
             with open(compile_log_file, "w") as compile_log:
                 compile.compile(source_file, output, self.compilers, compile_log, self.args.compile_mode,
-                                extra_compilation_args, extra_compilation_files, is_checker=is_checker)
+                                extra_compilation_args, extra_compilation_files, remove_all_cache=remove_all_cache)
             print(util.info("Compilation of file %s was successful."
                             % package_util.get_file_name(solution)))
             return True
@@ -392,262 +384,21 @@ class Command(BaseCommand):
             compile.print_compile_log(compile_log_file)
             return False
 
-    def check_output_diff(self, output_file, answer_file):
-        """
-        Checks whether the output file and the answer file are the same.
-        """
-        return util.file_diff(output_file, answer_file)
-
-    def check_output_checker(self, name, input_file, output_file, answer_file):
-        """
-        Checks if the output file is correct with the checker.
-        Returns True if the output file is correct, False otherwise and number of points.
-        """
-        command = [self.checker_executable, input_file, output_file, answer_file]
-        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
-        process.wait()
-        checker_output = process.communicate()[0].decode("utf-8").splitlines()
-
-        if len(checker_output) == 0:
-            raise CheckerOutputException("Checker output is empty.")
-
-        if checker_output[0].strip() == "OK":
-            points = 100
-            if len(checker_output) >= 3:
-                try:
-                    points = int(checker_output[2].strip())
-                except ValueError:
-                    pass
-
-            return True, points
-        else:
-            return False, 0
-
-
-    def check_output(self, name, input_file, output_file_path, output, answer_file_path):
-        """
-        Checks if the output file is correct.
-        Returns a tuple (is correct, number of points).
-        """
-        try:
-            has_checker = self.checker is not None
-        except AttributeError:
-            has_checker = False
-
-        if not has_checker:
-            with open(answer_file_path, "r") as answer_file:
-                correct = util.lines_diff(output, answer_file.readlines())
-            return correct, 100 if correct else 0
-        else:
-            with open(output_file_path, "w") as output_file:
-                output_file.write("\n".join(output) + "\n")
-            return self.check_output_checker(name, input_file, output_file_path, answer_file_path)
-
-    def execute_oiejq(self, name, timetool_path, executable, result_file_path, input_file_path, output_file_path, answer_file_path,
-                      time_limit, memory_limit, hard_time_limit, execution_dir):
-        command = f'"{timetool_path}" "{executable}"'
-        env = os.environ.copy()
-        env["MEM_LIMIT"] = f'{memory_limit}K'
-        env["MEASURE_MEM"] = "1"
-        env["UNDER_OIEJQ"] = "1"
-
-        timeout = False
-        with open(input_file_path, "r") as input_file, open(output_file_path, "w") as output_file, \
-                open(result_file_path, "w") as result_file:
-            process = subprocess.Popen(command, shell=True, stdin=input_file, stdout=output_file,
-                                       stderr=result_file, env=env, preexec_fn=os.setsid, cwd=execution_dir)
-
-            def sigint_handler(signum, frame):
-                try:
-                    os.killpg(os.getpgid(process.pid), signal.SIGTERM)
-                except ProcessLookupError:
-                    pass
-                sys.exit(1)
-            signal.signal(signal.SIGINT, sigint_handler)
-
-            try:
-                process.wait(timeout=hard_time_limit)
-            except subprocess.TimeoutExpired:
-                timeout = True
-                try:
-                    os.killpg(os.getpgid(process.pid), signal.SIGTERM)
-                except ProcessLookupError:
-                    pass
-                process.communicate()
-
-        with open(result_file_path, "r") as result_file:
-            lines = result_file.read()
-        with open(output_file_path, "r") as output_file:
-            output = output_file.read()
-        result = ExecutionResult()
-
-        if not timeout:
-            lines = lines.splitlines()
-            output = output.splitlines()
-
-            for line in lines:
-                line = line.strip()
-                if ": " in line:
-                    (key, value) = line.split(": ")[:2]
-                    if key == "Time":
-                        result.Time = self.parse_time(value)
-                    elif key == "Memory":
-                        result.Memory = self.parse_memory(value)
-                    else:
-                        setattr(result, key, value)
-
-        if timeout:
-            result.Status = Status.TL
-        elif getattr(result, "Time") is not None and result.Time > time_limit:
-            result.Status = Status.TL
-        elif getattr(result, "Memory") is not None and result.Memory > memory_limit:
-            result.Status = Status.ML
-        elif getattr(result, "Status") is None:
-            result.Status = Status.RE
-        elif result.Status == "OK":  # Here OK is a string, because it is set while parsing oiejq's output.
-            if result.Time > time_limit:
-                result.Status = Status.TL
-            elif result.Memory > memory_limit:
-                result.Status = Status.ML
-            else:
-                try:
-                    correct, result.Points = self.check_output(name, input_file_path, output_file_path, output, answer_file_path)
-                    if not correct:
-                        result.Status = Status.WA
-                except CheckerOutputException as e:
-                    result.Status = Status.CE
-                    result.Error = e.message
-        else:
-            result.Status = result.Status[:2]
-
-        return result
-
-
-    def execute_time(self, name, executable, result_file_path, input_file_path, output_file_path, answer_file_path,
-                      time_limit, memory_limit, hard_time_limit, execution_dir):
-        if sys.platform == 'darwin':
-            time_name = 'gtime'
-        elif sys.platform == 'linux':
-            time_name = 'time'
-        elif sys.platform == 'win32' or sys.platform == 'cygwin':
-            raise Exception("Measuring time with GNU time on Windows is not supported.")
-
-        command = [f'{time_name}', '-f', '%U\\n%M\\n%x', '-o', result_file_path, executable]
-        timeout = False
-        mem_limit_exceeded = False
-        with open(input_file_path, "r") as input_file, open(output_file_path, "w") as output_file:
-            process = subprocess.Popen(command, stdin=input_file, stdout=output_file, stderr=subprocess.DEVNULL,
-                                       preexec_fn=os.setsid, cwd=execution_dir)
-
-            def sigint_handler(signum, frame):
-                try:
-                    os.killpg(os.getpgid(process.pid), signal.SIGTERM)
-                except ProcessLookupError:
-                    pass
-                sys.exit(1)
-            signal.signal(signal.SIGINT, sigint_handler)
-
-            start_time = time.time()
-            while process.poll() is None:
-                try:
-                    time_process = psutil.Process(process.pid)
-                    executable_process = None
-                    for child in time_process.children():
-                        if child.name() == executable:
-                            executable_process = child
-                            break
-                    if executable_process is not None and executable_process.memory_info().rss > memory_limit * 1024:
-                        try:
-                            os.killpg(os.getpgid(process.pid), signal.SIGTERM)
-                        except ProcessLookupError:
-                            pass
-                        mem_limit_exceeded = True
-                        break
-                except psutil.NoSuchProcess:
-                    pass
-
-                if time.time() - start_time > hard_time_limit:
-                    try:
-                        os.killpg(os.getpgid(process.pid), signal.SIGTERM)
-                    except ProcessLookupError:
-                        pass
-                    timeout = True
-                    break
-
-        with open(output_file_path, "r") as output_file:
-            output = output_file.read()
-        result = ExecutionResult()
-        program_exit_code = None
-        if not timeout:
-            output = output.splitlines()
-            with open(result_file_path, "r") as result_file:
-                lines = result_file.readlines()
-            if len(lines) == 3:
-                """
-                If programs runs successfully, the output looks like this:
-                 - first line is CPU time in seconds
-                 - second line is memory in KB
-                 - third line is exit code
-                This format is defined by -f flag in time command.
-                """
-                result.Time = round(float(lines[0].strip()) * 1000)
-                result.Memory = int(lines[1].strip())
-                program_exit_code = int(lines[2].strip())
-            elif len(lines) > 0 and "Command terminated by signal " in lines[0]:
-                """
-                If there was a runtime error, the first line is the error message with signal number.
-                For example:
-                    Command terminated by signal 11
-                """
-                program_exit_code = int(lines[0].strip().split(" ")[-1])
-            elif not mem_limit_exceeded:
-                result.Status = Status.RE
-                result.Error = "Unexpected output from time command: " + "".join(lines)
-                return result
-
-        if program_exit_code is not None and program_exit_code != 0:
-            result.Status = Status.RE
-        elif timeout:
-            result.Status = Status.TL
-        elif mem_limit_exceeded:
-            result.Memory = memory_limit + 1  # Add one so that the memory is red in the table
-            result.Status = Status.ML
-        elif result.Time > time_limit:
-            result.Status = Status.TL
-        elif result.Memory > memory_limit:
-            result.Status = Status.ML
-        else:
-            try:
-                correct, result.Points = self.check_output(name, input_file_path, output_file_path, output,
-                                                           answer_file_path)
-                if correct:
-                    result.Status = Status.OK
-                else:
-                    result.Status = Status.WA
-            except CheckerOutputException as e:
-                result.Status = Status.CE
-                result.Error = e.message
-
-        return result
-
-
     def run_solution(self, data_for_execution: ExecutionData):
         """
         Run an execution and return the result as ExecutionResult object.
         """
 
-        (name, executable, test, time_limit, memory_limit, timetool_path, execution_dir) = data_for_execution
+        (name, executable, test, output_test, time_limit, memory_limit, timetool_path, execution_dir) = data_for_execution
         file_no_ext = paths.get_executions_path(name, package_util.extract_test_id(test, self.ID))
         output_file = file_no_ext + ".out"
         result_file = file_no_ext + ".res"
-        hard_time_limit_in_s = math.ceil(2 * time_limit / 1000.0)
+        hard_time_limit = math.ceil(2 * time_limit / 1000.0)
+
+        oiejq = self.timetool_name == 'oiejq'
+        return self.task_type.run(oiejq, timetool_path, executable, result_file, os.path.join(os.getcwd(), test), output_file,
+                                  output_test, time_limit, memory_limit, hard_time_limit, execution_dir)
 
-        if self.timetool_name == 'oiejq':
-            return self.execute_oiejq(name, timetool_path, executable, result_file, test, output_file, self.get_output_file(test),
-                                      time_limit, memory_limit, hard_time_limit_in_s, execution_dir)
-        elif self.timetool_name == 'time':
-            return self.execute_time(name, executable, result_file, test, output_file, self.get_output_file(test),
-                                     time_limit, memory_limit, hard_time_limit_in_s, execution_dir)
 
     def run_solutions(self, compiled_commands, names, solutions, executables_dir):
         """
@@ -682,8 +433,9 @@ class Command(BaseCommand):
                             test_result.time_tool == self.timetool_name:
                         all_results[name][self.get_group(test)][test] = test_result.result
                     else:
-                        executions.append((name, executable, test, test_time_limit, test_memory_limit,
-                                           self.timetool_path, os.path.dirname(executable)))
+                        executions.append((name, executable, test,
+                                           os.path.join(os.getcwd(), self.get_output_file(test)), test_time_limit,
+                                           test_memory_limit, self.timetool_path, os.path.dirname(executable)))
                         all_results[name][self.get_group(test)][test] = ExecutionResult(Status.PENDING)
                 os.makedirs(paths.get_executions_path(name), exist_ok=True)
             else:
@@ -805,7 +557,7 @@ class Command(BaseCommand):
                 if group not in self.scores:
                     util.exit_with_error(f'Group {group} doesn\'t have points specified in config file.')
 
-        if self.checker is None:
+        if self.task_type.has_checker():
             for solution in results.keys():
                 new_expected_scores[solution] = {
                     "expected": results[solution],
@@ -1030,6 +782,7 @@ class Command(BaseCommand):
         self.ID = package_util.get_task_id()
         self.SOURCE_EXTENSIONS = ['.c', '.cpp', '.py', '.java']
         self.SOLUTIONS_RE = package_util.get_solutions_re(self.ID)
+        self.task_type = task_type.get_task_type()
 
 
     def validate_arguments(self, args):
@@ -1126,9 +879,14 @@ class Command(BaseCommand):
 
             print(util.warning('Missing output files for tests: ' + ', '.join(
                 [self.extract_file_name(test) for test in missing_tests])))
+            if self.args.allow_no_outputs != True:
+                util.exit_with_error('There are tests without outputs. \n'
+                                     'Run outgen to fix this issue or add the --no-outputs flag to ignore the issue.')
             print(util.warning('Running only on tests with output files.'))
             self.tests = valid_input_files
             self.groups = self.get_groups(self.tests)
+            if len(self.groups) < 1:
+                util.exit_with_error('No tests with valid outputs.')
 
     def check_are_any_tests_to_run(self):
         """
@@ -1142,7 +900,7 @@ class Command(BaseCommand):
             if len(example_tests) == len(self.tests):
                 print(util.warning('Running only on example tests.'))
 
-            if not self.has_lib:
+            if self.task_type.require_outputs() and not self.has_lib:
                 self.validate_existence_of_outputs()
         else:
             util.exit_with_error('There are no tests to run.')
@@ -1160,34 +918,13 @@ class Command(BaseCommand):
                     if results[solution][group][test].Error is not None:
                         error_msg += f'Solution {solution} had an error on test {test}: {results[solution][group][test].Error}\n'
         if error_msg != "":
-            util.exit_with_error(error_msg)
-
-    def compile_checker(self):
-        checker_basename = os.path.basename(self.checker)
-        self.checker_executable = paths.get_executables_path(checker_basename + ".e")
-
-        checker_compilation = self.compile_solutions([self.checker], is_checker=True)
-        if not checker_compilation[0]:
-            util.exit_with_error('Checker compilation failed.')
-
-    def check_had_checker(self, has_checker):
-        """
-        Checks if there was a checker and if it is now removed (or the other way around) and if so, removes tests cache.
-        In theory, removing cache after adding a checker is redundant, because during its compilation, the cache is
-        removed.
-        """
-        had_checker = os.path.exists(paths.get_cache_path("checker"))
-        if (had_checker and not has_checker) or (not had_checker and has_checker):
-            cache.remove_results_cache()
-        if has_checker:
-            with open(paths.get_cache_path("checker"), "w") as f:
-                f.write("")
-        else:
-            try:
-                os.remove(paths.get_cache_path("checker"))
-            except FileNotFoundError:
-                pass
+            print(util.error(error_msg))
 
+    def compile_additional_files(self, files):
+        for name, args, kwargs in files:
+            print(f'Compiling {name}...')
+            if not self.compile(*args, **kwargs):
+                util.exit_with_error(f'Compilation of {name} failed.')
 
     def run(self, args):
         args = util.init_package_command(args)
@@ -1213,14 +950,7 @@ class Command(BaseCommand):
         cache.process_extra_execution_files(self.config.get("extra_execution_files", {}), self.ID)
         cache.remove_results_if_contest_type_changed(self.config.get("sinol_contest_type", "default"))
 
-        checker = package_util.get_files_matching_pattern(self.ID, f'{self.ID}chk.*')
-        if len(checker) != 0:
-            print("Checker found: %s" % os.path.basename(checker[0]))
-            self.checker = checker[0]
-            self.compile_checker()
-        else:
-            self.checker = None
-        self.check_had_checker(self.checker is not None)
+        self.compile_additional_files(self.task_type.get_files_to_compile())
 
         lib = package_util.get_files_matching_pattern(self.ID, f'{self.ID}lib.*')
         self.has_lib = len(lib) != 0
@@ -1253,7 +983,8 @@ class Command(BaseCommand):
             self.config = util.try_fix_config(self.config)
             try:
                 validation_results = self.validate_expected_scores(results)
-            except Exception:
+            except Exception as e:
+                print(traceback.format_exc())
                 util.exit_with_error("Validating expected scores failed. "
                                      "This probably means that `sinol_expected_scores` is broken. "
                                      "Delete it and run `sinol-make run --apply-suggestions` again.")