test: store multiple flame graphs alongside

Previously, it would get stored in build/flame_graph/flame_graphs.svg.
Now, the test repository codename is added to the file name.

Also, some refactoring and simplifying test filtering.

5 files changed, 29 insertions, 32 deletions

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index f9f53e6..edbf8aa 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -21,13 +21,13 @@ function(add_python_tests name)
 endfunction()
 
 add_python_tests(python_tests_sanity
-    Python3::Interpreter -m pytest ${python_test_args} -m "not stress")
+    Python3::Interpreter -m pytest ${python_test_args} -m "not stress and not flame_graph")
 
 add_python_tests(python_tests_stress
     Python3::Interpreter -m pytest ${python_test_args} -m "stress")
 
 add_python_tests(python_tests_valgrind
-    Python3::Interpreter -m pytest ${python_test_args} -m "valgrind"
+    Python3::Interpreter -m pytest ${python_test_args} -m "not stress and not flame_graph"
         --valgrind-binary "${CMAKE_CURRENT_SOURCE_DIR}/../src/valgrind.sh")
 
 if(NOT DEFINED FLAME_GRAPHS_DIR)
diff --git a/test/py/conftest.py b/test/py/conftest.py
index 748b5e5..84163a5 100644
--- a/test/py/conftest.py
+++ b/test/py/conftest.py
@@ -194,25 +194,17 @@ def workers(worker_cmd):
 
 
 @fixture
-def repo_path(tmp_path):
-    return os.path.join(tmp_path, 'repo')
-
-
-@fixture
-def flame_graph_svg(pytestconfig, tmp_path):
+def flame_graph_svg(pytestconfig, tmp_path, flame_graph_repo):
     dir = pytestconfig.getoption(PARAM_FLAME_GRAPHS_DIR.codename)
     if dir is None:
         return os.path.join(tmp_path, 'flame_graph.svg')
     os.makedirs(dir, exist_ok=True)
-    return os.path.join(dir, 'flame_graph.svg')
+    return os.path.join(dir, f'flame_graph_{flame_graph_repo.codename()}.svg')
 
 
 @fixture
 def profiler(pytestconfig, server, workers, flame_graph_svg):
     script = pytestconfig.getoption(PARAM_FLAMEGRAPH.codename)
-    if script is None:
-        yield
-        return
     pids = [server.pid] + [worker.pid for worker in workers]
     pids = map(str, pids)
     cmd_line = CmdLine(script, flame_graph_svg, *pids)
@@ -221,7 +213,12 @@ def profiler(pytestconfig, server, workers, flame_graph_svg):
     assert proc.returncode == 0
 
 
-ALL_REPOS = [
+@fixture
+def repo_path(tmp_path):
+    return os.path.join(tmp_path, 'repo')
+
+
+TEST_REPOS = [
     repo.TestRepoOutputSimple,
     repo.TestRepoOutputEmpty,
     repo.TestRepoOutputLong,
@@ -229,6 +226,11 @@ ALL_REPOS = [
     repo.TestRepoSegfault,
 ]
 
+STRESS_TEST_REPOS = [
+    repo.TestRepoOutputSimple,
+    repo.TestRepoOutputLong,
+]
+
 
 def _make_repo(repo_path, paths, cls):
     args = [repo_path]
@@ -237,20 +239,24 @@ def _make_repo(repo_path, paths, cls):
     return cls(*args)
 
 
-@fixture(params=ALL_REPOS, ids=[repo.codename() for repo in ALL_REPOS])
+@fixture(params=TEST_REPOS, ids=[repo.codename() for repo in TEST_REPOS])
 def test_repo(repo_path, paths, request):
     return _make_repo(repo_path, paths, request.param)
 
 
-@fixture(params=[repo for repo in ALL_REPOS if repo.enabled_for_stress_testing()],
-         ids=[repo.codename() for repo in ALL_REPOS if repo.enabled_for_stress_testing()])
+@fixture(params=STRESS_TEST_REPOS, ids=[repo.codename() for repo in STRESS_TEST_REPOS])
 def stress_test_repo(repo_path, paths, request):
     return _make_repo(repo_path, paths, request.param)
 
 
+@fixture
+def flame_graph_repo(stress_test_repo):
+    return stress_test_repo
+
+
 Env = namedtuple('Env', ['server', 'workers', 'client', 'db'])
 
 
 @fixture
-def env(server, workers, profiler, client, sqlite_db):
+def env(server, workers, client, sqlite_db):
     return Env(server, workers, client, sqlite_db)
diff --git a/test/py/lib/test_repo.py b/test/py/lib/test_repo.py
index 2820659..eb2786d 100644
--- a/test/py/lib/test_repo.py
+++ b/test/py/lib/test_repo.py
@@ -57,10 +57,6 @@ class TestRepo(Repo):
     def codename():
         pass
 
-    @staticmethod
-    def enabled_for_stress_testing():
-        return False
-
     @abc.abstractmethod
     def run_exit_code_matches(self, ec):
         pass
@@ -130,10 +126,6 @@ class TestRepoOutputSimple(TestRepoOutput):
     def codename():
         return 'output_simple'
 
-    @staticmethod
-    def enabled_for_stress_testing():
-        return True
-
     def format_output_script(self):
         return OUTPUT_SCRIPT_SIMPLE
 
@@ -173,10 +165,6 @@ class TestRepoOutputLong(TestRepoOutput):
     def codename():
         return 'output_long'
 
-    @staticmethod
-    def enabled_for_stress_testing():
-        return True
-
     def format_output_script(self):
         output_len = TestRepoOutputLong.OUTPUT_LEN_KB
         output_len = shlex.quote(str(output_len))
diff --git a/test/py/test_repo.py b/test/py/test_repo.py
index 9d070ca..544213d 100644
--- a/test/py/test_repo.py
+++ b/test/py/test_repo.py
@@ -72,7 +72,6 @@ def _test_repo_internal(env, repo, numof_processes, runs_per_process):
         assert repo.run_output_matches(output), f"Output doesn't match: {output}"
 
 
-@pytest.mark.valgrind
 @my_parametrize('runs_per_client', [1, 5])
 @my_parametrize('numof_clients', [1, 5])
 def test_repo(env, test_repo, numof_clients, runs_per_client):
@@ -84,7 +83,12 @@ def test_repo(env, test_repo, numof_clients, runs_per_client):
                 [
                     (10, 50),
                     (1, 2000),
-                    pytest.param(4, 500, marks=pytest.mark.flame_graph),
+                    (4, 500),
                 ])
 def test_repo_stress(env, stress_test_repo, numof_clients, runs_per_client):
     _test_repo_internal(env, stress_test_repo, numof_clients, runs_per_client)
+
+
+@pytest.mark.flame_graph
+def test_repo_flame_graph(env, profiler, flame_graph_repo):
+    _test_repo_internal(env, flame_graph_repo, 4, 500)
diff --git a/test/pytest.ini b/test/pytest.ini
index 6b95d54..69a5cc2 100644
--- a/test/pytest.ini
+++ b/test/pytest.ini
@@ -6,5 +6,4 @@ log_cli_level = INFO
 
 markers =
     stress: Long tests; don't run them casually
-    valgrind: These tests are run w/ Valgrind
     flame_graph: Generate the flame graph for these tests