[LTP] [PATCH 1/2] scripts: Add simple script for calculating timeouts
Andrea Cervesato
andrea.cervesato@suse.de
Wed Jan 22 13:48:26 CET 2025
From: Cyril Hrubis <chrubis@suse.cz>
This script parses JSON results from kirk and LTP metadata in order
calculate timeouts for tests based on the result file. It can also patch
tests automatically.
The script does:
- Take the results and pick all tests that run for longer than 0.5s.
Multiplie the time with a constant (currently 1.2) to get a suggested
timeout.
- Exclude tests that have runtime defined since these are controlled
by the runtime (that filters out all fuzzy sync tests).
There is a special case for timer tests that defines runtime only
dynamically in the timer library code. This should be possibly fixed
with special value for the .runtime in tst_test. E.g.
TST_RUNTIME_DYNAMIC for tests that only set runtime in the setup.
- Normalize the timeout for a single filesystem run if test is running for
more than one filesystem.
- Verify if tests are build on top of old library by checking at
metadata file
- Update test with a with newly calculated timeout.
By default we only increase timeouts but that can be overridden using
the -o option.
Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
Co-developed-by: Andrea Cervesato <andrea.cervesato@suse.com>
Signed-off-by: Andrea Cervesato <andrea.cervesato@suse.com>
---
scripts/calctimeouts.py | 232 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 232 insertions(+)
diff --git a/scripts/calctimeouts.py b/scripts/calctimeouts.py
new file mode 100755
index 0000000000000000000000000000000000000000..d5e8fe2c15faad7c8d1ec8c15541f35a97a8c0c4
--- /dev/null
+++ b/scripts/calctimeouts.py
@@ -0,0 +1,232 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+"""
+This script parses JSON results from kirk and LTP metadata in order to
+calculate timeouts for tests based on the results file.
+It can also patch tests automatically and replace the calculated timeout.
+"""
+
+import re
+import os
+import json
+import argparse
+
+# The test runtime is multiplied by this to get a timeout
+TIMEOUT_MUL = 1.2
+
+
+def _sed(fname, expr, replace):
+ """
+ Pythonic version of sed command.
+ """
+ content = []
+ matcher = re.compile(expr)
+
+ with open(fname, 'r', encoding="utf-8") as data:
+ for line in data:
+ match = matcher.search(line)
+ if not match:
+ content.append(line)
+ else:
+ content.append(replace)
+
+ with open(fname, 'w', encoding="utf-8") as data:
+ data.writelines(content)
+
+
+def _patch(ltp_dir, fname, new_timeout, override):
+ """
+ If `override` is True, it patches a test file, searching for timeout and
+ replacing it with `new_timeout`.
+ """
+ orig_timeout = None
+ file_path = os.path.join(ltp_dir, fname)
+
+ with open(file_path, 'r', encoding="utf-8") as c_source:
+ matcher = re.compile(r'\s*.timeout\s*=\s*(\d+).')
+ for line in c_source:
+ match = matcher.search(line)
+ if not match:
+ continue
+
+ timeout = match.group(1)
+ orig_timeout = int(timeout)
+
+ if orig_timeout:
+ if orig_timeout < new_timeout and override:
+ print(f"CHANGE {fname} timeout {orig_timeout} -> {new_timeout}")
+ _sed(file_path, r".timeout = [0-9]*,\n",
+ f"\t.timeout = {new_timeout},\n")
+ else:
+ print(f"KEEP {fname} timeout {orig_timeout} (new {new_timeout})")
+ else:
+ print(f"ADD {fname} timeout {new_timeout}")
+ _sed(file_path,
+ "static struct tst_test test = {",
+ "static struct tst_test test = {\n"
+ f"\t.timeout = {new_timeout},\n")
+
+
+def _patch_all(ltp_dir, timeouts, override):
+ """
+ Patch all tests.
+ """
+ for timeout in timeouts:
+ if timeout['path']:
+ _patch(ltp_dir, timeout['path'], timeout['timeout'], override)
+
+
+def _print_table(timeouts):
+ """
+ Print the timeouts table.
+ """
+ timeouts.sort(key=lambda x: x['timeout'], reverse=True)
+
+ total = 0
+
+ print("Old library tests\n-----------------\n")
+ for timeout in timeouts:
+ if not timeout['newlib']:
+ print(f"{timeout['name']:30s} {timeout['timeout']}")
+ total += 1
+
+ print(f"\n\t{total} tests in total")
+
+ total = 0
+
+ print("\nNew library tests\n-----------------\n")
+ for timeout in timeouts:
+ if timeout['newlib']:
+ print(f"{timeout['name']:30s} {timeout['timeout']}")
+ total += 1
+
+ print(f"\n\t{total} tests in total")
+
+
+def _parse_data(ltp_dir, results_path):
+ """
+ Parse results data and metadata, then it generates timeouts data.
+ """
+ timeouts = []
+ results = None
+ metadata = None
+
+ with open(results_path, 'r', encoding="utf-8") as file:
+ results = json.load(file)
+
+ metadata_path = os.path.join(ltp_dir, 'metadata', 'ltp.json')
+ with open(metadata_path, 'r', encoding="utf-8") as file:
+ metadata = json.load(file)
+
+ for test in results['results']:
+ name = test['test_fqn']
+ duration = test['test']['duration']
+
+ # if test runs for all_filesystems, normalize runtime to one filesystem
+ filesystems = max(1, test['test']['log'].count('TINFO: Formatting /'))
+
+ # check if test is new library test
+ test_is_newlib = name in metadata['tests']
+
+ # store test file path
+ path = None
+ if test_is_newlib:
+ path = metadata['tests'][name]['fname']
+
+ test_has_runtime = False
+ if test_is_newlib:
+ # filter out tests with runtime
+ test_has_runtime = 'runtime' in metadata['tests'][name]
+
+ # timer tests define runtime dynamically in timer library
+ test_has_runtime = 'sample' in metadata['tests'][name]
+
+ # select tests that does not have runtime and which are executed
+ # for a long time
+ if not test_has_runtime and duration >= 0.5:
+ data = {}
+ data["name"] = name
+ data["timeout"] = int(TIMEOUT_MUL * duration/filesystems + 0.5)
+ data["newlib"] = test_is_newlib
+ data["path"] = path
+
+ timeouts.append(data)
+
+ return timeouts
+
+
+def _file_exists(filepath):
+ """
+ Check if the given file path exists.
+ """
+ if not os.path.isfile(filepath):
+ raise argparse.ArgumentTypeError(
+ f"The file '{filepath}' does not exist.")
+ return filepath
+
+
+def _dir_exists(dirpath):
+ """
+ Check if the given directory path exists.
+ """
+ if not os.path.isdir(dirpath):
+ raise argparse.ArgumentTypeError(
+ f"The directory '{dirpath}' does not exist.")
+ return dirpath
+
+
+def run():
+ """
+ Entry point of the script.
+ """
+ parser = argparse.ArgumentParser(
+ description="Script to calculate LTP tests timeouts")
+
+ parser.add_argument(
+ '-l',
+ '--ltp-dir',
+ type=_dir_exists,
+ help='LTP directory',
+ default='/opt/ltp')
+
+ parser.add_argument(
+ '-r',
+ '--results',
+ type=_file_exists,
+ required=True,
+ help='kirk results.json file location')
+
+ parser.add_argument(
+ '-o',
+ '--override',
+ default=False,
+ action='store_true',
+ help='Override test timeouts')
+
+ parser.add_argument(
+ '-p',
+ '--patch',
+ default=False,
+ action='store_true',
+ help='Patch tests with updated timeout')
+
+ parser.add_argument(
+ '-t',
+ '--print-table',
+ default=True,
+ action='store_true',
+ help='Print table with suggested timeouts')
+
+ args = parser.parse_args()
+
+ timeouts = _parse_data(args.ltp_dir, args.results)
+
+ if args.print_table:
+ _print_table(timeouts)
+
+ if args.patch:
+ _patch_all(args.ltp_dir, timeouts, args.override)
+
+
+if __name__ == "__main__":
+ run()
--
2.43.0
More information about the ltp
mailing list