test: add test for segfaulting CI script

The C code leaked out of src/, so I moved .clang-format and some compile
options to the root directory.

Also, I'm starting to hit test execution limits; I'm going to limit the
repositories used for stress testing.

11 files changed, 192 insertions, 47 deletions

diff --git a/src/.clang-format b/.clang-format
index 87ad91d..87ad91d 100644
--- a/src/.clang-format
+++ b/.clang-format
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 95f423e..64efa33 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -4,6 +4,9 @@ project(cimple VERSION 0.0.1 LANGUAGES C)
 
 enable_testing()
 
+add_compile_options(-std=c17 -Wpedantic -Wall -Wextra)
+add_compile_options($<$<NOT:$<CONFIG:Debug>>:-Werror>)
+
 add_subdirectory(src)
 add_subdirectory(test)
 
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index c85ffdf..d994943 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1,5 +1,3 @@
-add_compile_options(-std=c17 -Wpedantic -Wall -Wextra $<$<NOT:$<CONFIG:Debug>>:-Werror>)
-
 # Even w/ _POSIX_C_SOURCE=200809L, we still need accept4, pipe2, execvpe, etc.,
 # which are all _GNU_SOURCE-only (at least for glibc). Try compiling without
 # any features macros defined at all, and prepare to be amazed!
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 3aca9fa..7f4ff11 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -1,3 +1,5 @@
+add_subdirectory(sigsegv)
+
 find_package(Python3 REQUIRED COMPONENTS Interpreter)
 
 set(python_test_args
@@ -7,6 +9,7 @@ set(python_test_args
     --server-binary "$<TARGET_FILE:server>"
     --worker-binary "$<TARGET_FILE:worker>"
     --client-binary "$<TARGET_FILE:client>"
+    --sigsegv-binary "$<TARGET_FILE:sigsegv>"
     --project-version "${PROJECT_VERSION}")
 
 function(add_python_tests name)
diff --git a/test/py/conftest.py b/test/py/conftest.py
index 0131c37..54ef444 100644
--- a/test/py/conftest.py
+++ b/test/py/conftest.py
@@ -8,7 +8,7 @@ import os
 
 from pytest import fixture
 
-from lib import test_repo
+from lib import test_repo as repo
 from lib.db import Database
 from lib.net import random_unused_port
 from lib.process import CmdLine
@@ -45,7 +45,7 @@ class ParamBinary(Param):
 
 
 BINARY_PARAMS = [
-    ParamBinary(name) for name in ('server', 'worker', 'client')
+    ParamBinary(name) for name in ('server', 'worker', 'client', 'sigsegv')
 ]
 
 PARAM_VALGRIND = ParamBinary('valgrind', required=False)
@@ -162,12 +162,17 @@ def worker_cmd(base_cmd_line, paths, server_port):
 
 
 @fixture
-def client_cmd(base_cmd_line, paths, server_port):
+def client(base_cmd_line, paths, server_port):
     args = ['--host', '127.0.0.1', '--port', server_port]
     return CmdLine.wrap(base_cmd_line, CmdLine(paths.client_binary, *args))
 
 
 @fixture
+def sigsegv(paths):
+    return CmdLine(paths.sigsegv_binary)
+
+
+@fixture
 def server(server_cmd):
     with server_cmd.run_async() as server:
         yield server
@@ -184,19 +189,35 @@ def workers(worker_cmd):
 
 
 @fixture
-def client(client_cmd):
-    return client_cmd
-
-
-@fixture(params=[
-             test_repo.TestRepoOutputSimple,
-             test_repo.TestRepoOutputEmpty,
-             test_repo.TestRepoOutputLong,
-             test_repo.TestRepoOutputNull,
-         ],
-         ids=['output_simple', 'output_empty', 'output_long', 'output_null'])
-def test_repo(tmp_path, request):
-    return request.param(os.path.join(tmp_path, 'repo'))
+def repo_path(tmp_path):
+    return os.path.join(tmp_path, 'repo')
+
+
+ALL_REPOS = [
+    repo.TestRepoOutputSimple,
+    repo.TestRepoOutputEmpty,
+    repo.TestRepoOutputLong,
+    repo.TestRepoOutputNull,
+    repo.TestRepoSegfault,
+]
+
+
+def _make_repo(repo_path, paths, cls):
+    args = [repo_path]
+    if cls is repo.TestRepoSegfault:
+        args += [paths.sigsegv_binary]
+    return cls(*args)
+
+
+@fixture(params=ALL_REPOS, ids=[repo.codename() for repo in ALL_REPOS])
+def test_repo(repo_path, paths, request):
+    return _make_repo(repo_path, paths, request.param)
+
+
+@fixture(params=[repo for repo in ALL_REPOS if repo.enabled_for_stress_testing()],
+         ids=[repo.codename() for repo in ALL_REPOS if repo.enabled_for_stress_testing()])
+def stress_test_repo(repo_path, paths, request):
+    return _make_repo(repo_path, paths, request.param)
 
 
 class Env:
diff --git a/test/py/lib/test_repo.py b/test/py/lib/test_repo.py
index 0d722be..aca6dfa 100644
--- a/test/py/lib/test_repo.py
+++ b/test/py/lib/test_repo.py
@@ -40,10 +40,6 @@ class TestRepo(Repo):
     # Prevent Pytest from discovering test cases in this class:
     __test__ = False
 
-    def _format_ci_script(self):
-        runs_dir = shlex.quote(self.runs_dir)
-        return CI_SCRIPT.format(runs_dir=runs_dir)
-
     def __init__(self, path, ci_script='ci.sh'):
         super().__init__(path)
 
@@ -51,13 +47,37 @@ class TestRepo(Repo):
         os.makedirs(self.runs_dir, exist_ok=True)
 
         self.ci_script_path = os.path.join(self.path, ci_script)
-        with open(self.ci_script_path, mode='x') as f:
-            f.write(self._format_ci_script())
-        os.chmod(self.ci_script_path, 0o755)
 
+        self.write_ci_script()
         self.run('git', 'add', '--', ci_script)
         self.run('git', 'commit', '-q', '-m', 'add CI script')
 
+    @staticmethod
+    @abc.abstractmethod
+    def codename():
+        pass
+
+    @staticmethod
+    def enabled_for_stress_testing():
+        return False
+
+    @abc.abstractmethod
+    def run_exit_code_matches(self, ec):
+        pass
+
+    @abc.abstractmethod
+    def run_output_matches(self, output):
+        pass
+
+    def write_ci_script(self):
+        with open(self.ci_script_path, mode='x') as f:
+            f.write(self.format_ci_script())
+        os.chmod(self.ci_script_path, 0o755)
+
+    def format_ci_script(self):
+        runs_dir = shlex.quote(self.runs_dir)
+        return CI_SCRIPT.format(runs_dir=runs_dir)
+
     def count_run_files(self):
         return len([name for name in os.listdir(self.runs_dir) if os.path.isfile(os.path.join(self.runs_dir, name))])
 
@@ -68,32 +88,31 @@ class TestRepoOutput(TestRepo, abc.ABC):
     OUTPUT_SCRIPT_NAME = 'generate-output'
 
     def __init__(self, path):
+        self.output_script_path = os.path.join(path, TestRepoOutput.OUTPUT_SCRIPT_NAME)
         super().__init__(path)
 
-        self.output_script_path = os.path.join(self.path, TestRepoOutput.OUTPUT_SCRIPT_NAME)
-        with open(self.output_script_path, mode='x') as f:
-            f.write(self._format_output_script())
-        os.chmod(self.output_script_path, 0o755)
-
-        with open(self.ci_script_path, mode='a') as f:
-            f.write(self._format_ci_script_addition())
-
+        self.write_output_script()
         self.run('git', 'add', '--', TestRepoOutput.OUTPUT_SCRIPT_NAME)
-        self.run('git', 'add', '-u')
         self.run('git', 'commit', '-q', '-m', 'add output script')
 
-    @abc.abstractmethod
-    def _format_output_script(self):
-        pass
-
-    def _format_ci_script_addition(self):
-        return r'{output_script} | tee -a "$run_output_path"'.format(
+    def format_ci_script(self):
+        script = super().format_ci_script()
+        added = r'{output_script} | tee -a "$run_output_path"'.format(
             output_script=shlex.quote(self.output_script_path))
+        return f'{script}\n{added}\n'
+
+    def write_output_script(self):
+        with open(self.output_script_path, mode='x') as f:
+            f.write(self.format_output_script())
+        os.chmod(self.output_script_path, 0o755)
 
     @abc.abstractmethod
-    def run_output_matches(self, output):
+    def format_output_script(self):
         pass
 
+    def run_exit_code_matches(self, ec):
+        return ec == 0
+
 
 OUTPUT_SCRIPT_SIMPLE = r'''#!/bin/sh -e
 timestamp="$( date --iso-8601=ns )"
@@ -104,7 +123,15 @@ echo "A CI run happened at $timestamp"
 class TestRepoOutputSimple(TestRepoOutput):
     __test__ = False
 
-    def _format_output_script(self):
+    @staticmethod
+    def codename():
+        return 'output_simple'
+
+    @staticmethod
+    def enabled_for_stress_testing():
+        return True
+
+    def format_output_script(self):
         return OUTPUT_SCRIPT_SIMPLE
 
     def run_output_matches(self, output):
@@ -118,7 +145,11 @@ OUTPUT_SCRIPT_EMPTY = r'''#!/bin/sh
 class TestRepoOutputEmpty(TestRepoOutput):
     __test__ = False
 
-    def _format_output_script(self):
+    @staticmethod
+    def codename():
+        return 'output_empty'
+
+    def format_output_script(self):
         return OUTPUT_SCRIPT_EMPTY
 
     def run_output_matches(self, output):
@@ -144,7 +175,15 @@ class TestRepoOutputLong(TestRepoOutput):
         self.output = base64.encodebytes(random.randbytes(nb)).decode()
         super().__init__(*args, **kwargs)
 
-    def _format_output_script(self):
+    @staticmethod
+    def codename():
+        return 'output_long'
+
+    @staticmethod
+    def enabled_for_stress_testing():
+        return True
+
+    def format_output_script(self):
         return OUTPUT_SCRIPT_LONG.format(output=repr(self.output))
 
     def run_output_matches(self, output):
@@ -168,8 +207,40 @@ class TestRepoOutputNull(TestRepoOutput):
         self.output = TestRepoOutputNull.OUTPUT
         super().__init__(*args, **kwargs)
 
-    def _format_output_script(self):
+    @staticmethod
+    def codename():
+        return 'output_null'
+
+    def format_output_script(self):
         return OUTPUT_SCRIPT_NULL.format(output=repr(self.output))
 
     def run_output_matches(self, output):
         return output == self.output
+
+
+class TestRepoSegfault(TestRepo):
+    __test__ = False
+
+    def __init__(self, repo_path, prog_path):
+        self.prog_path = prog_path
+        self.prog_name = os.path.basename(prog_path)
+        super().__init__(repo_path)
+
+        shutil.copy(prog_path, self.path)
+        self.run('git', 'add', '--', self.prog_name)
+        self.run('git', 'commit', '-q', '-m', 'add test program')
+
+    @staticmethod
+    def codename():
+        return 'segfault'
+
+    def format_ci_script(self):
+        script = super().format_ci_script()
+        added = r'exec {prog}'.format(prog=shlex.quote(f'./{self.prog_name}'))
+        return f'{script}\n{added}\n'
+
+    def run_exit_code_matches(self, ec):
+        return ec < 0
+
+    def run_output_matches(self, output):
+        return "Started the test program.\n" == output.decode()
diff --git a/test/py/test_repo.py b/test/py/test_repo.py
index e3a498d..aa89261 100644
--- a/test/py/test_repo.py
+++ b/test/py/test_repo.py
@@ -62,6 +62,7 @@ def _test_repo_internal(env, repo, numof_processes, runs_per_process):
 
     for id, status, ec, output, url, rev in runs:
         assert status == 'finished', f'Invalid status for run {id}: {status}'
+        assert repo.run_exit_code_matches(ec), f"Exit code doesn't match: {ec}"
         assert repo.run_output_matches(output), f"Output doesn't match: {output}"
 
 
@@ -80,6 +81,12 @@ def my_parametrize(names, values, ids=None, **kwargs):
     return pytest.mark.parametrize(names, values, ids=ids, **kwargs)
 
 
+def test_sigsegv(sigsegv):
+    ec, output = sigsegv.try_run()
+    assert ec == -11
+    assert output == 'Started the test program.\n'
+
+
 @my_parametrize('runs_per_client', [1, 5])
 @my_parametrize('numof_clients', [1, 5])
 def test_repo(env, test_repo, numof_clients, runs_per_client):
@@ -89,5 +96,5 @@ def test_repo(env, test_repo, numof_clients, runs_per_client):
 @pytest.mark.stress
 @my_parametrize(('numof_clients', 'runs_per_client'),
                 [(10, 50), (1, 2000), (4, 500)])
-def test_repo_stress(env, test_repo, numof_clients, runs_per_client):
-    _test_repo_internal(env, test_repo, numof_clients, runs_per_client)
+def test_repo_stress(env, stress_test_repo, numof_clients, runs_per_client):
+    _test_repo_internal(env, stress_test_repo, numof_clients, runs_per_client)
diff --git a/test/sigsegv/CMakeLists.txt b/test/sigsegv/CMakeLists.txt
new file mode 100644
index 0000000..b92147d
--- /dev/null
+++ b/test/sigsegv/CMakeLists.txt
@@ -0,0 +1,11 @@
+# Hopefully putting the NULL reference to a separate library should force the
+# sigsegv binary to crash. If I put it in main.c, surprisingly, Clang (with
+# -O2) produces a binary that doesn't crash: https://godbolt.org/z/joeqEEs3Y.
+# To force it to crash, the -fno-delete-null-pointer-checks flag is required:
+# https://godbolt.org/z/o3xEoeG3z. This is a bit hacky though; putting the
+# reference in a separate library should also do the trick.
+
+add_library(null lib.c)
+
+add_executable(sigsegv main.c)
+target_link_libraries(sigsegv PRIVATE null)
diff --git a/test/sigsegv/lib.c b/test/sigsegv/lib.c
new file mode 100644
index 0000000..1e136cc
--- /dev/null
+++ b/test/sigsegv/lib.c
@@ -0,0 +1,5 @@
+#include "lib.h"
+
+#include <stddef.h>
+
+struct data *data = NULL;
diff --git a/test/sigsegv/lib.h b/test/sigsegv/lib.h
new file mode 100644
index 0000000..1f162d5
--- /dev/null
+++ b/test/sigsegv/lib.h
@@ -0,0 +1,10 @@
+#ifndef __LIB_H__
+#define __LIB_H__
+
+struct data {
+	int x;
+};
+
+extern struct data *data;
+
+#endif
diff --git a/test/sigsegv/main.c b/test/sigsegv/main.c
new file mode 100644
index 0000000..428d168
--- /dev/null
+++ b/test/sigsegv/main.c
@@ -0,0 +1,16 @@
+#include "lib.h"
+
+#include <stdio.h>
+
+int main(void)
+{
+	puts("Started the test program.");
+	fflush(stdout);
+
+	printf("This will crash: %d.\n", data->x);
+
+	puts("You shouldn't see this.");
+	fflush(stdout);
+
+	return 0;
+}