[LTP] [PATCH] syscalls: munmap: Add munmap04 to check ENOMEM

[Ricardo B. Marlière]

From: Ricardo B. Marlière <rbm@suse.com>

The ERRORS section of the mmap(2) manpage says:

ENOMEM: The process's maximum number of mappings would have been exceeded.
This error can also occur for munmap(), when unmapping a region in the
middle of an existing mapping,  since  this  re‐ sults in two smaller
mappings on either side of the region being unmapped.

Add a new test munmap04 to address this scenario.

Signed-off-by: Ricardo B. Marlière <rbm@suse.com>
---
 runtest/syscalls                            |  1 +
 testcases/kernel/syscalls/munmap/.gitignore |  1 +
 testcases/kernel/syscalls/munmap/munmap04.c | 75 +++++++++++++++++++++++++++++
 3 files changed, 77 insertions(+)

diff --git a/runtest/syscalls b/runtest/syscalls
index 582422ac9ca8ccae598c626a11cf6ee7c30f0e3a..d918963b7ad616ce5931476d0804d2cb566942c0 100644
--- a/runtest/syscalls
+++ b/runtest/syscalls
@@ -951,6 +951,7 @@ munlockall01 munlockall01
 munmap01 munmap01
 munmap02 munmap02
 munmap03 munmap03
+munmap04 munmap04
 
 nanosleep01 nanosleep01
 nanosleep02 nanosleep02
diff --git a/testcases/kernel/syscalls/munmap/.gitignore b/testcases/kernel/syscalls/munmap/.gitignore
index 94d8c10a24cae47c6805546bff41a84a2a2e8173..d418dc4512abe6dffcb279fddbdbde25a4f44230 100644
--- a/testcases/kernel/syscalls/munmap/.gitignore
+++ b/testcases/kernel/syscalls/munmap/.gitignore
@@ -1,3 +1,4 @@
 /munmap01
 /munmap02
 /munmap03
+/munmap04
diff --git a/testcases/kernel/syscalls/munmap/munmap04.c b/testcases/kernel/syscalls/munmap/munmap04.c
new file mode 100644
index 0000000000000000000000000000000000000000..c439a3bd1bb3ec5c24c0ad5979b652ff3789ebd3
--- /dev/null
+++ b/testcases/kernel/syscalls/munmap/munmap04.c
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2025 SUSE LLC Ricardo B. Marlière <rbm@suse.com>
+ */
+
+/*\
+ * Verify that munmap() fails with ENOMEM after partially unmapping an
+ * existing map, while having the maximum amount of maps already allocated.
+ */
+
+#include "tst_test.h"
+
+#define PAD 2 // avoid adjacent mapping merges
+#define PAGES 3
+#define MAX_MAP_COUNT_FILE "/proc/sys/vm/max_map_count"
+
+static uintptr_t base = 0x100000000UL;
+static void *addr;
+static size_t page_sz;
+static unsigned long vma_size;
+static unsigned long max_map_count;
+static int map_count;
+
+static void run(void)
+{
+	TST_EXP_FAIL(munmap(addr + page_sz, page_sz), ENOMEM);
+}
+
+static void set_max_map_count(size_t max)
+{
+	tst_res(TINFO, "setting max_map_count = %zu", max);
+	SAFE_FILE_PRINTF(MAX_MAP_COUNT_FILE, "%zu", max);
+}
+
+static void setup(void)
+{
+	page_sz = SAFE_SYSCONF(_SC_PAGESIZE);
+	vma_size = PAGES * page_sz;
+
+	SAFE_FILE_SCANF(MAX_MAP_COUNT_FILE, "%lu", &max_map_count);
+	tst_res(TINFO, "original max_map_count = %lu", max_map_count);
+	set_max_map_count(500);
+
+	addr = SAFE_MMAP(NULL, vma_size, PROT_READ | PROT_WRITE,
+			 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+
+	while (1) {
+		void *p =
+			mmap((void *)(base + PAD * vma_size * map_count),
+			     vma_size, PROT_NONE,
+			     MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE,
+			     -1, 0);
+		if (p == MAP_FAILED)
+			break;
+		map_count++;
+	}
+
+	tst_res(TINFO, "mapped %d regions", map_count);
+}
+
+static void cleanup(void)
+{
+	set_max_map_count(max_map_count);
+	if (addr)
+		SAFE_MUNMAP(addr, vma_size);
+	for (int i = 0; i < map_count; i++)
+		SAFE_MUNMAP((void *)(base + PAD * vma_size * i), vma_size);
+}
+
+static struct tst_test test = {
+	.test_all = run,
+	.setup = setup,
+	.cleanup = cleanup,
+	.needs_root = 1,
+};

---
base-commit: f6b76fe373d1932ce1d2248e83ff284085145797
change-id: 20250704-new-munmap04-a59ca20ae00c

Best regards,
-- 
Ricardo B. Marlière <rbm@suse.com>