[LTP] [PATCH v4] syscalls: munmap: Add munmap04 to check ENOMEM

[Ricardo B. Marlière]

From: Ricardo B. Marlière <rbm@suse.com>

The ERRORS section of the mmap(2) manpage says:

ENOMEM: The process's maximum number of mappings would have been exceeded.
This error can also occur for munmap(), when unmapping a region in the
middle of an existing mapping, since this results in two smaller mappings
on either side of the region being unmapped.

Add a new test munmap04 to address this scenario.

Signed-off-by: Ricardo B. Marlière <rbm@suse.com>
---
Changes in v4:
- Rebased after munmap conversion to new API
- Link to v3: https://lore.kernel.org/r/20250707-new-munmap04-v3-1-e3efda8e7d2b@suse.com

Changes in v3:
- Added missing include to "lapi/mmap.h"
- Made max_map_count a test option
- Keep the mapped regions in the heap
- Required min. kernel v4.17
- Link to v2: https://lore.kernel.org/r/20250704-new-munmap04-v2-1-436c80df9646@suse.com

Changes in v2:
- Corrected the commit message
- Link to v1: https://lore.kernel.org/r/20250704-new-munmap04-v1-1-6ef96138b092@suse.com
---
 runtest/syscalls                            |  1 +
 testcases/kernel/syscalls/munmap/.gitignore |  1 +
 testcases/kernel/syscalls/munmap/munmap04.c | 94 +++++++++++++++++++++++++++++
 3 files changed, 96 insertions(+)

diff --git a/runtest/syscalls b/runtest/syscalls
index 9c80bccb09114d8b9730fdee05e3e28f5cc44afc..01126879d2d679bd529ee3657f658598728900f1 100644
--- a/runtest/syscalls
+++ b/runtest/syscalls
@@ -950,6 +950,7 @@ munlockall01 munlockall01
 
 munmap01 munmap01
 munmap03 munmap03
+munmap04 munmap04
 
 nanosleep01 nanosleep01
 nanosleep02 nanosleep02
diff --git a/testcases/kernel/syscalls/munmap/.gitignore b/testcases/kernel/syscalls/munmap/.gitignore
index aa6e14a670e85dd17f965b4a465997a021f8b4ac..7ade86390a34cd6fbbf6530385e891e01e2b9137 100644
--- a/testcases/kernel/syscalls/munmap/.gitignore
+++ b/testcases/kernel/syscalls/munmap/.gitignore
@@ -1,2 +1,3 @@
 /munmap01
 /munmap03
+/munmap04
diff --git a/testcases/kernel/syscalls/munmap/munmap04.c b/testcases/kernel/syscalls/munmap/munmap04.c
new file mode 100644
index 0000000000000000000000000000000000000000..e326d24a5fb927b1bb7c7d51438f5a3d769696c2
--- /dev/null
+++ b/testcases/kernel/syscalls/munmap/munmap04.c
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2025 SUSE LLC Ricardo B. Marlière <rbm@suse.com>
+ */
+
+/*\
+ * Verify that munmap() fails with ENOMEM after partially unmapping an
+ * existing map, while having the maximum amount of maps already allocated.
+ */
+
+#include "tst_test.h"
+#include "lapi/mmap.h"
+
+#define PAD 2 /* avoid adjacent mapping merges */
+#define PAGES 3
+#define MAX_MAP_COUNT_FILE "/proc/sys/vm/max_map_count"
+
+static char *str_max_map_count;
+static int max_map_count = 500;
+
+static uintptr_t base = 0x100000000UL;
+static size_t page_sz;
+static unsigned long vma_size;
+static unsigned long max_map_count_orig;
+static int map_count;
+static void **maps;
+
+static void run(void)
+{
+	TST_EXP_FAIL(munmap(maps[2] + page_sz, page_sz), ENOMEM);
+}
+
+static void set_max_map_count(size_t max)
+{
+	tst_res(TINFO, "Setting max_map_count = %zu", max);
+	SAFE_FILE_PRINTF(MAX_MAP_COUNT_FILE, "%zu", max);
+}
+
+static void setup(void)
+{
+	page_sz = SAFE_SYSCONF(_SC_PAGESIZE);
+	vma_size = PAGES * page_sz;
+
+	if (tst_parse_int(str_max_map_count, &max_map_count, 100, INT_MAX))
+		tst_brk(TBROK, "Invalid max. map count '%s'",
+			str_max_map_count);
+
+	SAFE_FILE_SCANF(MAX_MAP_COUNT_FILE, "%lu", &max_map_count_orig);
+	tst_res(TINFO, "Original max_map_count = %lu", max_map_count_orig);
+	set_max_map_count(max_map_count);
+
+	maps = SAFE_MALLOC(max_map_count * sizeof(char *));
+	for (int i = 0; i < max_map_count; i++)
+		maps[i] = NULL;
+
+	while (1) {
+		void *p =
+			mmap((void *)(base + PAD * vma_size * map_count),
+			     vma_size, PROT_NONE,
+			     MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE,
+			     -1, 0);
+		if (p == MAP_FAILED)
+			break;
+		maps[map_count++] = p;
+	}
+
+	if (map_count == max_map_count)
+		tst_brk(TBROK, "Mapped all %d regions, expected less",
+			map_count);
+
+	tst_res(TINFO, "Mapped %d regions", map_count);
+}
+
+static void cleanup(void)
+{
+	set_max_map_count(max_map_count_orig);
+	for (int i = 0; i < map_count; i++) {
+		if (maps[i] == NULL)
+			break;
+		SAFE_MUNMAP((void *)(maps[i]), vma_size);
+	}
+	free(maps);
+}
+
+static struct tst_test test = {
+	.test_all = run,
+	.setup = setup,
+	.cleanup = cleanup,
+	.needs_root = 1,
+	.min_kver = "4.17",
+	.options = (struct tst_option[]){ { "n:", &str_max_map_count,
+					    "Max. map count (default: 500)" },
+					  {} },
+};

---
base-commit: 0c7346cb097440902568856527be7162f5950497
change-id: 20250704-new-munmap04-a59ca20ae00c

Best regards,
-- 
Ricardo B. Marlière <rbm@suse.com>