implement flame graph generation

8 files changed, 205 insertions, 8 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5ce967f..731d81c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -55,7 +55,7 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends gcovr libgit2-dev libjson-c-dev libsodium-dev libsqlite3-dev python3-pytest valgrind
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends gcovr libgit2-dev libjson-c-dev libsodium-dev libsqlite3-dev python3-pytest
       - name: Generate report
         run: make coverage
       - name: Upload report
@@ -65,6 +65,31 @@ jobs:
           path: ./build/coverage/
           if-no-files-found: error
 
+  flame_graphs:
+    runs-on: ubuntu-latest
+    name: Flame graphs
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends libgit2-dev libjson-c-dev libsodium-dev libsqlite3-dev python3-pytest
+      - name: Install FlameGraph
+        run: |
+          git clone --depth 1 https://github.com/brendangregg/FlameGraph.git ~/FlameGraph
+          echo ~/FlameGraph >> "$GITHUB_PATH"
+      - name: Build
+        run: make install
+      - name: Generate flame graphs
+        run: sudo --preserve-env=PATH make test/perf
+      - name: Upload graphs
+        uses: actions/upload-artifact@v3
+        with:
+          name: flame_graphs
+          path: ./build/flame_graphs/
+          if-no-files-found: error
+
   publish:
     needs: [lint, build, coverage]
     runs-on: ubuntu-latest
diff --git a/Makefile b/Makefile
index 7b5a8b9..79221d6 100644
--- a/Makefile
+++ b/Makefile
@@ -5,6 +5,7 @@ build_dir := $(src_dir)/build
 cmake_dir := $(build_dir)/cmake
 install_dir := $(build_dir)/install
 coverage_dir := $(build_dir)/coverage
+flame_graphs_dir := $(build_dir)/flame_graphs
 
 COMPILER ?= clang
 CONFIGURATION ?= Debug
@@ -41,6 +42,7 @@ build:
 		-D 'DEFAULT_HOST=$(call escape,$(DEFAULT_HOST))' \
 		-D 'DEFAULT_PORT=$(call escape,$(DEFAULT_PORT))' \
 		-D 'COVERAGE=$(call escape,$(COVERAGE))' \
+		-D 'FLAME_GRAPHS_DIR=$(call escape,$(flame_graphs_dir))' \
 		-S '$(call escape,$(src_dir))' \
 		-B '$(call escape,$(cmake_dir))'
 	cmake --build '$(call escape,$(cmake_dir))' -- -j
@@ -91,6 +93,14 @@ test/stress:
 	ctest --test-dir '$(call escape,$(cmake_dir))' \
 		--verbose --tests-regex python_tests_stress
 
+.PHONY: test/perf
+test/perf:
+	@echo -----------------------------------------------------------------
+	@echo Collecting profiling data
+	@echo -----------------------------------------------------------------
+	ctest --test-dir '$(call escape,$(cmake_dir))' \
+		--verbose --tests-regex python_tests_perf
+
 .PHONY: test/docker
 test/docker: test/sanity
 
diff --git a/scripts/flame_graph.sh b/scripts/flame_graph.sh
new file mode 100755
index 0000000..ae9129c
--- /dev/null
+++ b/scripts/flame_graph.sh
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2023 Egor Tensin <Egor.Tensin@gmail.com>
+# This file is part of the "cimple" project.
+# For details, see https://github.com/egor-tensin/cimple.
+# Distributed under the MIT License.
+
+# This script attaches to a set of processes and makes a flame graph using
+# flamegraph.pl.
+
+set -o errexit -o nounset -o pipefail
+shopt -s inherit_errexit lastpipe
+
+script_name="$( basename -- "${BASH_SOURCE[0]}" )"
+readonly script_name
+
+script_usage() {
+    local msg
+    for msg; do
+        echo "$script_name: $msg"
+    done
+
+    echo "usage: $script_name OUTPUT_PATH PID [PID...]"
+}
+
+join_pids() {
+    local result=''
+    while [ "$#" -gt 0 ]; do
+        if [ -n "$result" ]; then
+            result="$result,"
+        fi
+        result="$result$1"
+        shift
+    done
+    echo "$result"
+}
+
+output_dir=''
+
+cleanup() {
+    if [ -n "$output_dir" ]; then
+        echo "Removing temporary directory: $output_dir"
+        rm -rf -- "$output_dir"
+    fi
+}
+
+make_graph() {
+    wait "$record_pid" || true
+    perf script -i "$output_dir/perf.data" > "$output_dir/perf.out"
+    stackcollapse-perf.pl "$output_dir/perf.out" > "$output_dir/perf.folded"
+    flamegraph.pl --width 2400 "$output_dir/perf.folded" > "$output_path"
+}
+
+record_pid=''
+
+stop_record() {
+    echo "Stopping 'perf record' process $record_pid"
+    kill -SIGINT "$record_pid"
+    make_graph
+}
+
+check_tools() {
+    local tool
+    for tool in perf stackcollapse-perf.pl flamegraph.pl; do
+        if ! command -v "$tool" &> /dev/null; then
+            echo "$script_name: $tool is missing" >&2
+            exit 1
+        fi
+    done
+}
+
+main() {
+    trap cleanup EXIT
+    check_tools
+
+    if [ "$#" -lt 1 ]; then
+        script_usage "output path is required" >&2
+        exit 1
+    fi
+    local output_path="$1"
+    output_path="$( realpath -- "$output_path" )"
+    shift
+
+    if [ "$#" -lt 1 ]; then
+        script_usage "at least one process ID is required" >&2
+        exit 1
+    fi
+    local pids
+    pids="$( join_pids "$@" )"
+    shift
+
+    echo "Output path: $output_path"
+    echo "PIDs: $pids"
+
+    output_dir="$( dirname -- "$output_path" )"
+    output_dir="$( mktemp -d --tmpdir="$output_dir" )"
+    readonly output_dir
+
+    perf record \
+        -o "$output_dir/perf.data" \
+        --freq=99 \
+        --call-graph dwarf,65528 \
+        --pid="$pids" \
+        --no-inherit &
+
+    record_pid="$!"
+    echo "Started 'perf record' process $record_pid"
+    trap stop_record SIGINT SIGTERM
+
+    make_graph
+}
+
+main "$@"
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 7f4ff11..d370c53 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -29,3 +29,12 @@ add_python_tests(python_tests_stress
 add_python_tests(python_tests_valgrind
     Python3::Interpreter -m pytest ${python_test_args} -m "not stress"
         --valgrind-binary "${CMAKE_CURRENT_SOURCE_DIR}/../src/valgrind.sh")
+
+if(NOT DEFINED FLAME_GRAPHS_DIR)
+    set(FLAME_GRAPHS_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+endif()
+
+add_python_tests(python_tests_perf
+    Python3::Interpreter -m pytest ${python_test_args} -m "flame_graph"
+        --flame-graph-binary "${CMAKE_CURRENT_SOURCE_DIR}/../scripts/flame_graph.sh"
+        --flame-graphs-dir "${FLAME_GRAPHS_DIR}")
diff --git a/test/py/conftest.py b/test/py/conftest.py
index ab2cbfb..db3f0b8 100644
--- a/test/py/conftest.py
+++ b/test/py/conftest.py
@@ -51,6 +51,9 @@ BINARY_PARAMS = [
 
 PARAM_VALGRIND = ParamBinary('valgrind', required=False)
 
+PARAM_FLAME_GRAPH = ParamBinary('flame_graph', required=False)
+PARAM_FLAME_GRAPHS_DIR = Param('flame_graphs_dir', 'directory to store flame graphs', required=False)
+
 
 class ParamVersion(Param):
     def __init__(self):
@@ -62,6 +65,8 @@ PARAM_VERSION = ParamVersion()
 PARAMS = list(BINARY_PARAMS)
 PARAMS += [
     PARAM_VALGRIND,
+    PARAM_FLAME_GRAPH,
+    PARAM_FLAME_GRAPHS_DIR,
     PARAM_VERSION,
 ]
 
@@ -94,14 +99,19 @@ def paths(pytestconfig):
     return Paths(pytestconfig)
 
 
+class CmdLineValgrind(CmdLine):
+    def __init__(self, binary):
+        # Signal to Valgrind that ci scripts should obviously be exempt from
+        # memory leak checking:
+        super().__init__(binary, '--trace-children-skip=*/ci', '--')
+
+
 @fixture(scope='session')
 def base_cmd_line(pytestconfig):
     cmd_line = CmdLine.unbuffered()
     valgrind = pytestconfig.getoption(PARAM_VALGRIND.codename)
     if valgrind is not None:
-        # Signal to Valgrind that ci scripts should obviously be exempt from
-        # memory leak checking:
-        cmd_line = CmdLine.wrap(CmdLine(valgrind, '--trace-children-skip=*/ci', '--'), cmd_line)
+        cmd_line = CmdLine.wrap(CmdLineValgrind(valgrind), cmd_line)
     return cmd_line
 
 
@@ -194,6 +204,29 @@ def repo_path(tmp_path):
     return os.path.join(tmp_path, 'repo')
 
 
+@fixture
+def flame_graph_path(pytestconfig, tmp_path):
+    dir = pytestconfig.getoption(PARAM_FLAME_GRAPHS_DIR.codename)
+    if dir is None:
+        return os.path.join(tmp_path, 'flame_graph.svg')
+    os.makedirs(dir, exist_ok=True)
+    return os.path.join(dir, 'flame_graph.svg')
+
+
+@fixture
+def profiler(pytestconfig, server, workers, flame_graph_path):
+    script = pytestconfig.getoption(PARAM_FLAME_GRAPH.codename)
+    if script is None:
+        yield
+        return
+    pids = [server.pid] + [worker.pid for worker in workers]
+    pids = map(str, pids)
+    cmd_line = CmdLine(script, flame_graph_path, *pids)
+    with cmd_line.run_async() as proc:
+        yield
+    assert proc.returncode == 0
+
+
 ALL_REPOS = [
     repo.TestRepoOutputSimple,
     repo.TestRepoOutputEmpty,
@@ -225,5 +258,5 @@ Env = namedtuple('Env', ['server', 'workers', 'client', 'db'])
 
 
 @fixture
-def env(server, workers, client, sqlite_db):
+def env(server, workers, profiler, client, sqlite_db):
     return Env(server, workers, client, sqlite_db)
diff --git a/test/py/lib/tests.py b/test/py/lib/tests.py
index a676021..fe90f96 100644
--- a/test/py/lib/tests.py
+++ b/test/py/lib/tests.py
@@ -14,8 +14,10 @@ def my_parametrize(names, values, ids=None, **kwargs):
         if len(_names) == 1:
             ids = [f'{names}={v}' for v in values]
         else:
+            _values = [combination.values if hasattr(combination, 'values') else combination
+                       for combination in values]
             ids = [
                 '-'.join(f'{k}={v}' for k, v in zip(_names, combination))
-                for combination in values
+                for combination in _values
             ]
     return pytest.mark.parametrize(names, values, ids=ids, **kwargs)
diff --git a/test/py/test_repo.py b/test/py/test_repo.py
index 49d5a32..c973e37 100644
--- a/test/py/test_repo.py
+++ b/test/py/test_repo.py
@@ -79,7 +79,11 @@ def test_repo(env, test_repo, numof_clients, runs_per_client):
 
 
 @pytest.mark.stress
-@my_parametrize(('numof_clients', 'runs_per_client'),
-                [(10, 50), (1, 2000), (4, 500)])
+@my_parametrize('numof_clients,runs_per_client',
+                [
+                    (10, 50),
+                    (1, 2000),
+                    pytest.param(4, 500, marks=pytest.mark.flame_graph),
+                ])
 def test_repo_stress(env, stress_test_repo, numof_clients, runs_per_client):
     _test_repo_internal(env, stress_test_repo, numof_clients, runs_per_client)
diff --git a/test/pytest.ini b/test/pytest.ini
index 766e409..49e206d 100644
--- a/test/pytest.ini
+++ b/test/pytest.ini
@@ -6,3 +6,4 @@ log_cli_level = INFO
 
 markers =
     stress: Big tests; don't run them w/ Valgrind or in QEMU
+    flame_graph: Generate the flame graph for these tests