[LTP] [PATCH 1/2] scripts: Add simple script for calculating timeouts
Cyril Hrubis
chrubis@suse.cz
Wed Jan 22 14:13:16 CET 2025
Hi!
> This script parses JSON results from kirk and LTP metadata in order
> calculate timeouts for tests based on the result file. It can also patch
> tests automatically.
>
> The script does:
>
> - Take the results and pick all tests that run for longer than 0.5s.
> Multiplie the time with a constant (currently 1.2) to get a suggested
> timeout.
>
> - Exclude tests that have runtime defined since these are controlled
> by the runtime (that filters out all fuzzy sync tests).
>
> There is a special case for timer tests that defines runtime only
> dynamically in the timer library code. This should be possibly fixed
> with special value for the .runtime in tst_test. E.g.
> TST_RUNTIME_DYNAMIC for tests that only set runtime in the setup.
>
> - Normalize the timeout for a single filesystem run if test is running for
> more than one filesystem.
>
> - Verify if tests are build on top of old library by checking at
> metadata file
>
> - Update test with a with newly calculated timeout.
> By default we only increase timeouts but that can be overridden using
> the -o option.
>
> Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
> Co-developed-by: Andrea Cervesato <andrea.cervesato@suse.com>
> Signed-off-by: Andrea Cervesato <andrea.cervesato@suse.com>
> ---
> scripts/calctimeouts.py | 232 ++++++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 232 insertions(+)
>
> diff --git a/scripts/calctimeouts.py b/scripts/calctimeouts.py
> new file mode 100755
> index 0000000000000000000000000000000000000000..d5e8fe2c15faad7c8d1ec8c15541f35a97a8c0c4
> --- /dev/null
> +++ b/scripts/calctimeouts.py
> @@ -0,0 +1,232 @@
> +#!/usr/bin/env python3
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +"""
> +This script parses JSON results from kirk and LTP metadata in order to
> +calculate timeouts for tests based on the results file.
> +It can also patch tests automatically and replace the calculated timeout.
> +"""
> +
> +import re
> +import os
> +import json
> +import argparse
> +
> +# The test runtime is multiplied by this to get a timeout
> +TIMEOUT_MUL = 1.2
> +
> +
> +def _sed(fname, expr, replace):
> + """
> + Pythonic version of sed command.
> + """
> + content = []
> + matcher = re.compile(expr)
> +
> + with open(fname, 'r', encoding="utf-8") as data:
> + for line in data:
> + match = matcher.search(line)
> + if not match:
> + content.append(line)
> + else:
> + content.append(replace)
> +
> + with open(fname, 'w', encoding="utf-8") as data:
> + data.writelines(content)
> +
> +
> +def _patch(ltp_dir, fname, new_timeout, override):
> + """
> + If `override` is True, it patches a test file, searching for timeout and
> + replacing it with `new_timeout`.
The override is for the cases where the test already has a timeout and
the timeout in the test is bigger than the one we calculated. By default
the function keeps the bigger timeout.
> + """
> + orig_timeout = None
> + file_path = os.path.join(ltp_dir, fname)
> +
> + with open(file_path, 'r', encoding="utf-8") as c_source:
> + matcher = re.compile(r'\s*.timeout\s*=\s*(\d+).')
> + for line in c_source:
> + match = matcher.search(line)
> + if not match:
> + continue
> +
> + timeout = match.group(1)
> + orig_timeout = int(timeout)
> +
> + if orig_timeout:
> + if orig_timeout < new_timeout and override:
> + print(f"CHANGE {fname} timeout {orig_timeout} -> {new_timeout}")
> + _sed(file_path, r".timeout = [0-9]*,\n",
> + f"\t.timeout = {new_timeout},\n")
> + else:
> + print(f"KEEP {fname} timeout {orig_timeout} (new {new_timeout})")
> + else:
> + print(f"ADD {fname} timeout {new_timeout}")
> + _sed(file_path,
> + "static struct tst_test test = {",
> + "static struct tst_test test = {\n"
> + f"\t.timeout = {new_timeout},\n")
> +
> +
> +def _patch_all(ltp_dir, timeouts, override):
> + """
> + Patch all tests.
> + """
> + for timeout in timeouts:
> + if timeout['path']:
> + _patch(ltp_dir, timeout['path'], timeout['timeout'], override)
> +
> +
> +def _print_table(timeouts):
> + """
> + Print the timeouts table.
> + """
> + timeouts.sort(key=lambda x: x['timeout'], reverse=True)
> +
> + total = 0
> +
> + print("Old library tests\n-----------------\n")
> + for timeout in timeouts:
> + if not timeout['newlib']:
> + print(f"{timeout['name']:30s} {timeout['timeout']}")
> + total += 1
> +
> + print(f"\n\t{total} tests in total")
> +
> + total = 0
> +
> + print("\nNew library tests\n-----------------\n")
> + for timeout in timeouts:
> + if timeout['newlib']:
> + print(f"{timeout['name']:30s} {timeout['timeout']}")
> + total += 1
> +
> + print(f"\n\t{total} tests in total")
> +
> +
> +def _parse_data(ltp_dir, results_path):
> + """
> + Parse results data and metadata, then it generates timeouts data.
> + """
> + timeouts = []
> + results = None
> + metadata = None
> +
> + with open(results_path, 'r', encoding="utf-8") as file:
> + results = json.load(file)
> +
> + metadata_path = os.path.join(ltp_dir, 'metadata', 'ltp.json')
> + with open(metadata_path, 'r', encoding="utf-8") as file:
> + metadata = json.load(file)
> +
> + for test in results['results']:
> + name = test['test_fqn']
> + duration = test['test']['duration']
> +
> + # if test runs for all_filesystems, normalize runtime to one filesystem
> + filesystems = max(1, test['test']['log'].count('TINFO: Formatting /'))
> +
> + # check if test is new library test
> + test_is_newlib = name in metadata['tests']
> +
> + # store test file path
> + path = None
> + if test_is_newlib:
> + path = metadata['tests'][name]['fname']
> +
> + test_has_runtime = False
> + if test_is_newlib:
> + # filter out tests with runtime
> + test_has_runtime = 'runtime' in metadata['tests'][name]
> +
> + # timer tests define runtime dynamically in timer library
> + test_has_runtime = 'sample' in metadata['tests'][name]
> +
> + # select tests that does not have runtime and which are executed
> + # for a long time
> + if not test_has_runtime and duration >= 0.5:
> + data = {}
> + data["name"] = name
> + data["timeout"] = int(TIMEOUT_MUL * duration/filesystems + 0.5)
> + data["newlib"] = test_is_newlib
> + data["path"] = path
> +
> + timeouts.append(data)
> +
> + return timeouts
> +
> +
> +def _file_exists(filepath):
> + """
> + Check if the given file path exists.
> + """
> + if not os.path.isfile(filepath):
> + raise argparse.ArgumentTypeError(
> + f"The file '{filepath}' does not exist.")
> + return filepath
> +
> +
> +def _dir_exists(dirpath):
> + """
> + Check if the given directory path exists.
> + """
> + if not os.path.isdir(dirpath):
> + raise argparse.ArgumentTypeError(
> + f"The directory '{dirpath}' does not exist.")
> + return dirpath
> +
> +
> +def run():
> + """
> + Entry point of the script.
> + """
> + parser = argparse.ArgumentParser(
> + description="Script to calculate LTP tests timeouts")
> +
> + parser.add_argument(
> + '-l',
> + '--ltp-dir',
> + type=_dir_exists,
> + help='LTP directory',
> + default='/opt/ltp')
The script is not supposed to be executed from installed tree, it needs
the C source files. So I would argue that '../' or '.' is better
default.
> + parser.add_argument(
> + '-r',
> + '--results',
> + type=_file_exists,
> + required=True,
> + help='kirk results.json file location')
> +
> + parser.add_argument(
> + '-o',
> + '--override',
> + default=False,
> + action='store_true',
> + help='Override test timeouts')
> +
> + parser.add_argument(
> + '-p',
> + '--patch',
> + default=False,
> + action='store_true',
> + help='Patch tests with updated timeout')
> +
> + parser.add_argument(
> + '-t',
> + '--print-table',
> + default=True,
> + action='store_true',
> + help='Print table with suggested timeouts')
> +
> + args = parser.parse_args()
> +
> + timeouts = _parse_data(args.ltp_dir, args.results)
> +
> + if args.print_table:
> + _print_table(timeouts)
> +
> + if args.patch:
> + _patch_all(args.ltp_dir, timeouts, args.override)
> +
> +
> +if __name__ == "__main__":
> + run()
>
> --
> 2.43.0
>
--
Cyril Hrubis
chrubis@suse.cz
More information about the ltp
mailing list