[LTP] [PATCH v5] syscalls: munmap: Add munmap04 to check ENOMEM

[Ricardo B. Marlière]

From: Ricardo B. Marlière <rbm@suse.com>

The ERRORS section of the mmap(2) manpage says:

ENOMEM: The process's maximum number of mappings would have been exceeded.
This error can also occur for munmap(), when unmapping a region in the
middle of an existing mapping, since this results in two smaller mappings
on either side of the region being unmapped.

Add a new test munmap04 to address this scenario.

Signed-off-by: Ricardo B. Marlière <rbm@suse.com>
---
Changes in v5:
- Dropped argument parsing in favor of tst_test.save_restore
- Link to v4: https://lore.kernel.org/r/20250708-new-munmap04-v4-1-884f774179b1@suse.com

Changes in v4:
- Rebased after munmap conversion to new API
- Link to v3: https://lore.kernel.org/r/20250707-new-munmap04-v3-1-e3efda8e7d2b@suse.com

Changes in v3:
- Added missing include to "lapi/mmap.h"
- Made max_map_count a test option
- Keep the mapped regions in the heap
- Required min. kernel v4.17
- Link to v2: https://lore.kernel.org/r/20250704-new-munmap04-v2-1-436c80df9646@suse.com

Changes in v2:
- Corrected the commit message
- Link to v1: https://lore.kernel.org/r/20250704-new-munmap04-v1-1-6ef96138b092@suse.com
---
 runtest/syscalls                            |  1 +
 testcases/kernel/syscalls/munmap/.gitignore |  1 +
 testcases/kernel/syscalls/munmap/munmap04.c | 74 +++++++++++++++++++++++++++++
 3 files changed, 76 insertions(+)

diff --git a/runtest/syscalls b/runtest/syscalls
index 9c80bccb09114d8b9730fdee05e3e28f5cc44afc..01126879d2d679bd529ee3657f658598728900f1 100644
--- a/runtest/syscalls
+++ b/runtest/syscalls
@@ -950,6 +950,7 @@ munlockall01 munlockall01
 
 munmap01 munmap01
 munmap03 munmap03
+munmap04 munmap04
 
 nanosleep01 nanosleep01
 nanosleep02 nanosleep02
diff --git a/testcases/kernel/syscalls/munmap/.gitignore b/testcases/kernel/syscalls/munmap/.gitignore
index aa6e14a670e85dd17f965b4a465997a021f8b4ac..7ade86390a34cd6fbbf6530385e891e01e2b9137 100644
--- a/testcases/kernel/syscalls/munmap/.gitignore
+++ b/testcases/kernel/syscalls/munmap/.gitignore
@@ -1,2 +1,3 @@
 /munmap01
 /munmap03
+/munmap04
diff --git a/testcases/kernel/syscalls/munmap/munmap04.c b/testcases/kernel/syscalls/munmap/munmap04.c
new file mode 100644
index 0000000000000000000000000000000000000000..044cd0f79fb8f853a1be6c8aec63b54cf3bba02f
--- /dev/null
+++ b/testcases/kernel/syscalls/munmap/munmap04.c
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2025 SUSE LLC Ricardo B. Marlière <rbm@suse.com>
+ */
+
+/*\
+ * Verify that munmap() fails with ENOMEM after partially unmapping an
+ * existing map, while having the maximum amount of maps already allocated.
+ */
+
+#include "tst_test.h"
+#include "lapi/mmap.h"
+
+#define PAD 2 /* avoid adjacent mapping merges */
+#define MEMSIZE 3
+#define MAP_MAX_COUNT 65530
+
+static uintptr_t base = 0x100000000UL;
+static size_t page_sz;
+static unsigned long vma_size;
+static int map_count;
+static void **maps;
+
+static void run(void)
+{
+	TST_EXP_FAIL(munmap(maps[2] + page_sz, page_sz), ENOMEM);
+}
+
+static void setup(void)
+{
+	page_sz = SAFE_SYSCONF(_SC_PAGESIZE);
+	vma_size = MEMSIZE * page_sz;
+
+	maps = SAFE_MALLOC(MAP_MAX_COUNT * sizeof(char *));
+	for (int i = 0; i < MAP_MAX_COUNT; i++)
+		maps[i] = NULL;
+
+	while (1) {
+		void *p = mmap((void *)(base + PAD * vma_size * map_count),
+			     vma_size, PROT_NONE,
+			     MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE,
+			     -1, 0);
+		if (p == MAP_FAILED)
+			break;
+		maps[map_count++] = p;
+	}
+
+	if (map_count == MAP_MAX_COUNT)
+		tst_brk(TBROK, "Mapped all %d regions, expected less",
+			map_count);
+
+	tst_res(TINFO, "Mapped %d regions", map_count);
+}
+
+static void cleanup(void)
+{
+	for (int i = 0; i < map_count; i++) {
+		if (maps[i] == NULL)
+			break;
+		SAFE_MUNMAP((void *)(maps[i]), vma_size);
+	}
+	free(maps);
+}
+
+static struct tst_test test = {
+	.test_all = run,
+	.setup = setup,
+	.cleanup = cleanup,
+	.min_kver = "4.17",
+	.save_restore = (const struct tst_path_val[]){
+		{ "/proc/sys/vm/max_map_count", TST_TO_STR(MAP_MAX_COUNT), TST_SR_SKIP },
+		{},
+	},
+};

---
base-commit: 0c7346cb097440902568856527be7162f5950497
change-id: 20250704-new-munmap04-a59ca20ae00c

Best regards,
-- 
Ricardo B. Marlière <rbm@suse.com>