[LTP] [PATCH v9 2/2] shmctl03: Fix 32-bit compat mode failure

[Wei Gao]

On 64-bit kernels, the default shmmax and shmall values often exceed
the range of a 32-bit unsigned long or are clipped differently by
the kernel's compat syscall layer than they appear in /proc.

Fix this by utilizing the new TST_ASSERT_ULONG() flags to properly
clamp and truncate the expected values under compat mode.

Link: https://lore.kernel.org/ltp/aJm5SBOaRoe1e0PB@yuki.lan/
Suggested-by: Cyril Hrubis <chrubis@suse.cz>
Reviewed-by: Andrea Cervesato <andrea.cervesato@suse.com>
Reported-by: Dan Carpenter <dan.carpenter@linaro.org>
Signed-off-by: Wei Gao <wegao@suse.com>
---
 testcases/kernel/syscalls/shmctl/shmctl03.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/testcases/kernel/syscalls/shmctl/shmctl03.c b/testcases/kernel/syscalls/shmctl/shmctl03.c
index 9e1c2f099..fcb416a1d 100644
--- a/testcases/kernel/syscalls/shmctl/shmctl03.c
+++ b/testcases/kernel/syscalls/shmctl/shmctl03.c
@@ -30,9 +30,16 @@ static void verify_ipcinfo(void)
 	else
 		tst_res(TPASS, "shmmin = 1");
 
-	TST_ASSERT_ULONG(PATH_KERN_SHMMAX, info.shmmax);
-	TST_ASSERT_ULONG(PATH_KERN_SHMMNI, info.shmmni);
-	TST_ASSERT_ULONG(PATH_KERN_SHMALL, info.shmall);
+	int c = tst_is_compat_mode();
+
+	/*
+	 * On 64-bit kernel, shmmax is clamped to INT_MAX for 32-bit
+	 * compat syscall, while shmmni and shmall are truncated
+	 * to 32-bit.
+	 */
+	TST_ASSERT_ULONG(PATH_KERN_SHMMAX, info.shmmax, c ? TST_ASSERT_SATURATED_INT : TST_ASSERT_NONE);
+	TST_ASSERT_ULONG(PATH_KERN_SHMMNI, info.shmmni, c ? TST_ASSERT_TRUNC_32BIT : TST_ASSERT_NONE);
+	TST_ASSERT_ULONG(PATH_KERN_SHMALL, info.shmall, c ? TST_ASSERT_TRUNC_32BIT : TST_ASSERT_NONE);
 }
 
 static struct tst_test test = {
-- 
2.55.0