[LTP] [PATCH] syscalls/mmap21: Add new test for mmap's ENOMEM case

[Avinesh Kumar]

To be more precise in this test and checking for ENOMEM on exact mapping
when limit is crossed, I could only think of counting the existing
mappings from /proc/$pid/maps and counting the remaining possible
mappings. Is there any better approach if we want to test this for exact
case?

Signed-off-by: Avinesh Kumar <akumar@suse.de>
---
 testcases/kernel/syscalls/mmap/mmap21.c | 70 +++++++++++++++++++++++++
 1 file changed, 70 insertions(+)
 create mode 100644 testcases/kernel/syscalls/mmap/mmap21.c

diff --git a/testcases/kernel/syscalls/mmap/mmap21.c b/testcases/kernel/syscalls/mmap/mmap21.c
new file mode 100644
index 000000000..76002edb3
--- /dev/null
+++ b/testcases/kernel/syscalls/mmap/mmap21.c
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2023 SUSE LLC Avinesh Kumar <avinesh.kumar@suse.com>
+ */
+
+/*\
+ * [Description]
+ *
+ * Verify that, mmap(2) fails with errno ENOMEM when process's
+ * maximum number of mappings would have been exceeded.
+ */
+
+#include <stdio.h>
+#include "tst_test.h"
+
+#define TEMPFILE "mmapfile"
+static int fd;
+static size_t page_sz;
+static int max_map;
+
+static void setup(void)
+{
+	page_sz = getpagesize();
+
+	fd = SAFE_OPEN(TEMPFILE, O_RDWR | O_CREAT, 0666);
+
+	SAFE_FILE_SCANF("/proc/sys/vm/max_map_count", "%d   ", &max_map);
+	tst_res(TINFO, "max_map_count: %d", max_map);
+}
+
+static void run(void)
+{
+	int i;
+	char *addr[max_map];
+
+	for (i = 0; i < max_map; i++) {
+		TESTPTR(mmap(0, page_sz, PROT_READ | PROT_WRITE, MAP_FILE | MAP_SHARED, fd, 0));
+		if (TST_RET_PTR != MAP_FAILED) {
+			addr[i] = TST_RET_PTR;
+		} else {
+			if (TST_ERR == ENOMEM) {
+				tst_res(TINFO, "New successful mappings before exhausting the limit: %d", i);
+				tst_res(TPASS, "mmap() failed with ENOMEM");
+			} else {
+				tst_res(TINFO, "New successful mappings before mmap() failed: %d", i);
+				tst_res(TFAIL | TERRNO, "mmap() failed with unexpected errno");
+			}
+			break;
+		}
+	}
+
+	if (i == max_map)
+		tst_res(TFAIL, "mmap() crossed max_map_count limit, no of new mmapings: %d", i);
+
+	while (--i >= 0)
+		SAFE_MUNMAP(addr[i], page_sz);
+}
+
+static void cleanup(void)
+{
+	if (fd > 0)
+		SAFE_CLOSE(fd);
+}
+
+static struct tst_test test = {
+	.setup = setup,
+	.cleanup = cleanup,
+	.test_all = run,
+	.needs_tmpdir = 1
+};
-- 
2.41.0