aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/test/py/lib
diff options
context:
space:
mode:
authorEgor Tensin <Egor.Tensin@gmail.com>2023-07-11 21:17:10 +0200
committerEgor Tensin <Egor.Tensin@gmail.com>2023-07-11 23:14:52 +0200
commit8619f8934a2762fa35d80b792ad349cd1128c1e9 (patch)
tree0078ea6f8f14c25a8ddbcf573d4e67dfa3dc3d32 /test/py/lib
parentsqlite: make some identifiers more readable (diff)
downloadcimple-8619f8934a2762fa35d80b792ad349cd1128c1e9.tar.gz
cimple-8619f8934a2762fa35d80b792ad349cd1128c1e9.zip
test: add test for segfaulting CI script
The C code leaked out of src/, so I moved .clang-format and some compile options to the root directory. Also, I'm starting to hit test execution limits; I'm going to limit the repositories used for stress testing.
Diffstat (limited to '')
-rw-r--r--test/py/lib/test_repo.py125
1 files changed, 98 insertions, 27 deletions
diff --git a/test/py/lib/test_repo.py b/test/py/lib/test_repo.py
index 0d722be..aca6dfa 100644
--- a/test/py/lib/test_repo.py
+++ b/test/py/lib/test_repo.py
@@ -40,10 +40,6 @@ class TestRepo(Repo):
# Prevent Pytest from discovering test cases in this class:
__test__ = False
- def _format_ci_script(self):
- runs_dir = shlex.quote(self.runs_dir)
- return CI_SCRIPT.format(runs_dir=runs_dir)
-
def __init__(self, path, ci_script='ci.sh'):
super().__init__(path)
@@ -51,13 +47,37 @@ class TestRepo(Repo):
os.makedirs(self.runs_dir, exist_ok=True)
self.ci_script_path = os.path.join(self.path, ci_script)
- with open(self.ci_script_path, mode='x') as f:
- f.write(self._format_ci_script())
- os.chmod(self.ci_script_path, 0o755)
+ self.write_ci_script()
self.run('git', 'add', '--', ci_script)
self.run('git', 'commit', '-q', '-m', 'add CI script')
+ @staticmethod
+ @abc.abstractmethod
+ def codename():
+ pass
+
+ @staticmethod
+ def enabled_for_stress_testing():
+ return False
+
+ @abc.abstractmethod
+ def run_exit_code_matches(self, ec):
+ pass
+
+ @abc.abstractmethod
+ def run_output_matches(self, output):
+ pass
+
+ def write_ci_script(self):
+ with open(self.ci_script_path, mode='x') as f:
+ f.write(self.format_ci_script())
+ os.chmod(self.ci_script_path, 0o755)
+
+ def format_ci_script(self):
+ runs_dir = shlex.quote(self.runs_dir)
+ return CI_SCRIPT.format(runs_dir=runs_dir)
+
def count_run_files(self):
return len([name for name in os.listdir(self.runs_dir) if os.path.isfile(os.path.join(self.runs_dir, name))])
@@ -68,32 +88,31 @@ class TestRepoOutput(TestRepo, abc.ABC):
OUTPUT_SCRIPT_NAME = 'generate-output'
def __init__(self, path):
+ self.output_script_path = os.path.join(path, TestRepoOutput.OUTPUT_SCRIPT_NAME)
super().__init__(path)
- self.output_script_path = os.path.join(self.path, TestRepoOutput.OUTPUT_SCRIPT_NAME)
- with open(self.output_script_path, mode='x') as f:
- f.write(self._format_output_script())
- os.chmod(self.output_script_path, 0o755)
-
- with open(self.ci_script_path, mode='a') as f:
- f.write(self._format_ci_script_addition())
-
+ self.write_output_script()
self.run('git', 'add', '--', TestRepoOutput.OUTPUT_SCRIPT_NAME)
- self.run('git', 'add', '-u')
self.run('git', 'commit', '-q', '-m', 'add output script')
- @abc.abstractmethod
- def _format_output_script(self):
- pass
-
- def _format_ci_script_addition(self):
- return r'{output_script} | tee -a "$run_output_path"'.format(
+ def format_ci_script(self):
+ script = super().format_ci_script()
+ added = r'{output_script} | tee -a "$run_output_path"'.format(
output_script=shlex.quote(self.output_script_path))
+ return f'{script}\n{added}\n'
+
+ def write_output_script(self):
+ with open(self.output_script_path, mode='x') as f:
+ f.write(self.format_output_script())
+ os.chmod(self.output_script_path, 0o755)
@abc.abstractmethod
- def run_output_matches(self, output):
+ def format_output_script(self):
pass
+ def run_exit_code_matches(self, ec):
+ return ec == 0
+
OUTPUT_SCRIPT_SIMPLE = r'''#!/bin/sh -e
timestamp="$( date --iso-8601=ns )"
@@ -104,7 +123,15 @@ echo "A CI run happened at $timestamp"
class TestRepoOutputSimple(TestRepoOutput):
__test__ = False
- def _format_output_script(self):
+ @staticmethod
+ def codename():
+ return 'output_simple'
+
+ @staticmethod
+ def enabled_for_stress_testing():
+ return True
+
+ def format_output_script(self):
return OUTPUT_SCRIPT_SIMPLE
def run_output_matches(self, output):
@@ -118,7 +145,11 @@ OUTPUT_SCRIPT_EMPTY = r'''#!/bin/sh
class TestRepoOutputEmpty(TestRepoOutput):
__test__ = False
- def _format_output_script(self):
+ @staticmethod
+ def codename():
+ return 'output_empty'
+
+ def format_output_script(self):
return OUTPUT_SCRIPT_EMPTY
def run_output_matches(self, output):
@@ -144,7 +175,15 @@ class TestRepoOutputLong(TestRepoOutput):
self.output = base64.encodebytes(random.randbytes(nb)).decode()
super().__init__(*args, **kwargs)
- def _format_output_script(self):
+ @staticmethod
+ def codename():
+ return 'output_long'
+
+ @staticmethod
+ def enabled_for_stress_testing():
+ return True
+
+ def format_output_script(self):
return OUTPUT_SCRIPT_LONG.format(output=repr(self.output))
def run_output_matches(self, output):
@@ -168,8 +207,40 @@ class TestRepoOutputNull(TestRepoOutput):
self.output = TestRepoOutputNull.OUTPUT
super().__init__(*args, **kwargs)
- def _format_output_script(self):
+ @staticmethod
+ def codename():
+ return 'output_null'
+
+ def format_output_script(self):
return OUTPUT_SCRIPT_NULL.format(output=repr(self.output))
def run_output_matches(self, output):
return output == self.output
+
+
+class TestRepoSegfault(TestRepo):
+ __test__ = False
+
+ def __init__(self, repo_path, prog_path):
+ self.prog_path = prog_path
+ self.prog_name = os.path.basename(prog_path)
+ super().__init__(repo_path)
+
+ shutil.copy(prog_path, self.path)
+ self.run('git', 'add', '--', self.prog_name)
+ self.run('git', 'commit', '-q', '-m', 'add test program')
+
+ @staticmethod
+ def codename():
+ return 'segfault'
+
+ def format_ci_script(self):
+ script = super().format_ci_script()
+ added = r'exec {prog}'.format(prog=shlex.quote(f'./{self.prog_name}'))
+ return f'{script}\n{added}\n'
+
+ def run_exit_code_matches(self, ec):
+ return ec < 0
+
+ def run_output_matches(self, output):
+ return "Started the test program.\n" == output.decode()