path: root/tools/testing/selftests/x86/lam.c

// SPDX-License-Identifier: GPL-2.0
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/syscall.h>
#include <time.h>
#include <signal.h>
#include <setjmp.h>
#include <sys/mman.h>
#include <sys/utsname.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <inttypes.h>

#include <sys/uio.h>
#include <linux/io_uring.h>
#include "../kselftest.h"

#ifndef __x86_64__
# error This test is 64-bit only
#endif

/* LAM modes, these definitions were copied from kernel code */
#define LAM_NONE                0
#define LAM_U57_BITS            6

#define LAM_U57_MASK            (0x3fULL << 57)
/* arch prctl for LAM */
#define ARCH_GET_UNTAG_MASK     0x4001
#define ARCH_ENABLE_TAGGED_ADDR 0x4002
#define ARCH_GET_MAX_TAG_BITS   0x4003

/* Specified test function bits */
#define FUNC_MALLOC             0x1
#define FUNC_BITS               0x2
#define FUNC_MMAP               0x4
#define FUNC_SYSCALL            0x8
#define FUNC_URING              0x10

#define TEST_MASK               0x1f

#define LOW_ADDR                (0x1UL << 30)
#define HIGH_ADDR               (0x3UL << 48)

#define MALLOC_LEN              32

#define PAGE_SIZE               (4 << 10)

#define barrier() ({						\
		   __asm__ __volatile__("" : : : "memory");	\
})

#define URING_QUEUE_SZ 1
#define URING_BLOCK_SZ 2048

struct testcases {
	unsigned int later;
	int expected; /* 2: SIGSEGV Error; 1: other errors */
	unsigned long lam;
	uint64_t addr;
	int (*test_func)(struct testcases *test);
	const char *msg;
};

/* Used by CQ of uring, source file handler and file's size */
struct file_io {
	int file_fd;
	off_t file_sz;
	struct iovec iovecs[];
};

struct io_uring_queue {
	unsigned int *head;
	unsigned int *tail;
	unsigned int *ring_mask;
	unsigned int *ring_entries;
	unsigned int *flags;
	unsigned int *array;
	union {
		struct io_uring_cqe *cqes;
		struct io_uring_sqe *sqes;
	} queue;
	size_t ring_sz;
};

struct io_ring {
	int ring_fd;
	struct io_uring_queue sq_ring;
	struct io_uring_queue cq_ring;
};

int tests_cnt;
jmp_buf segv_env;

static void segv_handler(int sig)
{
	ksft_print_msg("Get segmentation fault(%d).", sig);

	siglongjmp(segv_env, 1);
}

static inline int cpu_has_lam(void)
{
	unsigned int cpuinfo[4];

	__cpuid_count(0x7, 1, cpuinfo[0], cpuinfo[1], cpuinfo[2], cpuinfo[3]);

	return (cpuinfo[0] & (1 << 26));
}

/* Check 5-level page table feature in CPUID.(EAX=07H, ECX=00H):ECX.[bit 16] */
static inline int cpu_has_la57(void)
{
	unsigned int cpuinfo[4];

	__cpuid_count(0x7, 0, cpuinfo[0], cpuinfo[1], cpuinfo[2], cpuinfo[3]);

	return (cpuinfo[2] & (1 << 16));
}

/*
 * Set tagged address and read back untag mask.
 * check if the untagged mask is expected.
 *
 * @return:
 * 0: Set LAM mode successfully
 * others: failed to set LAM
 */
static int set_lam(unsigned long lam)
{
	int ret = 0;
	uint64_t ptr = 0;

	if (lam != LAM_U57_BITS && lam != LAM_NONE)
		return -1;

	/* Skip check return */
	syscall(SYS_arch_prctl, ARCH_ENABLE_TAGGED_ADDR, lam);

	/* Get untagged mask */
	syscall(SYS_arch_prctl, ARCH_GET_UNTAG_MASK, &ptr);

	/* Check mask returned is expected */
	if (lam == LAM_U57_BITS)
		ret = (ptr != ~(LAM_U57_MASK));
	else if (lam == LAM_NONE)
		ret = (ptr != -1ULL);

	return ret;
}

static unsigned long get_default_tag_bits(void)
{
	pid_t pid;
	int lam = LAM_NONE;
	int ret = 0;

	pid = fork();
	if (pid < 0) {
		perror("Fork failed.");
	} else if (pid == 0) {
		/* Set LAM mode in child process */
		if (set_lam(LAM_U57_BITS) == 0)
			lam = LAM_U57_BITS;
		else
			lam = LAM_NONE;
		exit(lam);
	} else {
		wait(&ret);
		lam = WEXITSTATUS(ret);
	}

	return lam;
}

/* According to LAM mode, set metadata in high bits */
static uint64_t set_metadata(uint64_t src, unsigned long lam)
{
	uint64_t metadata;

	srand(time(NULL));

	switch (lam) {
	case LAM_U57_BITS: /* Set metadata in bits 62:57 */
		/* Get a random non-zero value as metadata */
		metadata = (rand() % ((1UL << LAM_U57_BITS) - 1) + 1) << 57;
		metadata |= (src & ~(LAM_U57_MASK));
		break;
	default:
		metadata = src;
		break;
	}

	return metadata;
}

/*
 * Set metadata in user pointer, compare new pointer with original pointer.
 * both pointers should point to the same address.
 *
 * @return:
 * 0: value on the pointer with metadate and value on original are same
 * 1: not same.
 */
static int handle_lam_test(void *src, unsigned int lam)
{
	char *ptr;

	strcpy((char *)src, "USER POINTER");

	ptr = (char *)set_metadata((uint64_t)src, lam);
	if (src == ptr)
		return 0;

	/* Copy a string into the pointer with metadata */
	strcpy((char *)ptr, "METADATA POINTER");

	return (!!strcmp((char *)src, (char *)ptr));
}


int handle_max_bits(struct testcases *test)
{
	unsigned long exp_bits = get_default_tag_bits();
	unsigned long bits = 0;

	if (exp_bits != LAM_NONE)
		exp_bits = LAM_U57_BITS;

	/* Get LAM max tag bits */
	if (syscall(SYS_arch_prctl, ARCH_GET_MAX_TAG_BITS, &bits) == -1)
		return 1;

	return (exp_bits != bits);
}

/*
 * Test lam feature through dereference pointer get from malloc.
 * @return 0: Pass test. 1: Get failure during test 2: Get SIGSEGV
 */
static int handle_malloc(struct testcases *test)
{
	char *ptr = NULL;
	int ret = 0;

	if (test->later == 0 && test->lam != 0)
		if (set_lam(test->lam) == -1)
			return 1;

	ptr = (char *)malloc(MALLOC_LEN);
	if (ptr == NULL) {
		perror("malloc() failure\n");
		return 1;
	}

	/* Set signal handler */
	if (sigsetjmp(segv_env, 1) == 0) {
		signal(SIGSEGV, segv_handler);
		ret = handle_lam_test(ptr, test->lam);
	} else {
		ret = 2;
	}

	if (test->later != 0 && test->lam != 0)
		if (set_lam(test->lam) == -1 && ret == 0)
			ret = 1;

	free(ptr);

	return ret;
}

static int handle_mmap(struct testcases *test)
{
	void *ptr;
	unsigned int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED;
	int ret = 0;

	if (test->later == 0 && test->lam != 0)
		if (set_lam(test->lam) != 0)
			return 1;

	ptr = mmap((void *)test->addr, PAGE_SIZE, PROT_READ | PROT_WRITE,
		   flags, -1, 0);
	if (ptr == MAP_FAILED) {
		if (test->addr == HIGH_ADDR)
			if (!cpu_has_la57())
				return 3; /* unsupport LA57 */
		return 1;
	}

	if (test->later != 0 && test->lam != 0)
		if (set_lam(test->lam) != 0)
			ret = 1;

	if (ret == 0) {
		if (sigsetjmp(segv_env, 1) == 0) {
			signal(SIGSEGV, segv_handler);
			ret = handle_lam_test(ptr, test->lam);
		} else {
			ret = 2;
		}
	}

	munmap(ptr, PAGE_SIZE);
	return ret;
}

static int handle_syscall(struct testcases *test)
{
	struct utsname unme, *pu;
	int ret = 0;

	if (test->later == 0 && test->lam != 0)
		if (set_lam(test->lam) != 0)
			return 1;

	if (sigsetjmp(segv_env, 1) == 0) {
		signal(SIGSEGV, segv_handler);
		pu = (struct utsname *)set_metadata((uint64_t)&unme, test->lam);
		ret = uname(pu);
		if (ret < 0)
			ret = 1;
	} else {
		ret = 2;
	}

	if (test->later != 0 && test->lam != 0)
		if (set_lam(test->lam) != -1 && ret == 0)
			ret = 1;

	return ret;
}

int sys_uring_setup(unsigned int entries, struct io_uring_params *p)
{
	return (int)syscall(__NR_io_uring_setup, entries, p);
}

int sys_uring_enter(int fd, unsigned int to, unsigned int min, unsigned int flags)
{
	return (int)syscall(__NR_io_uring_enter, fd, to, min, flags, NULL, 0);
}

/* Init submission queue and completion queue */
int mmap_io_uring(struct io_uring_params p, struct io_ring *s)
{
	struct io_uring_queue *sring = &s->sq_ring;
	struct io_uring_queue *cring = &s->cq_ring;

	sring->ring_sz = p.sq_off.array + p.sq_entries * sizeof(unsigned int);
	cring->ring_sz = p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe);

	if (p.features & IORING_FEAT_SINGLE_MMAP) {
		if (cring->ring_sz > sring->ring_sz)
			sring->ring_sz = cring->ring_sz;

		cring->ring_sz = sring->ring_sz;
	}

	void *sq_ptr = mmap(0, sring->ring_sz, PROT_READ | PROT_WRITE,
			    MAP_SHARED | MAP_POPULATE, s->ring_fd,
			    IORING_OFF_SQ_RING);

	if (sq_ptr == MAP_FAILED) {
		perror("sub-queue!");
		return 1;
	}

	void *cq_ptr = sq_ptr;

	if (!(p.features & IORING_FEAT_SINGLE_MMAP)) {
		cq_ptr = mmap(0, cring->ring_sz, PROT_READ | PROT_WRITE,
			      MAP_SHARED | MAP_POPULATE, s->ring_fd,
			      IORING_OFF_CQ_RING);
		if (cq_ptr == MAP_FAILED) {
			perror("cpl-queue!");
			munmap(sq_ptr, sring->ring_sz);
			return 1;
		}
	}

	sring->head = sq_ptr + p.sq_off.head;
	sring->tail = sq_ptr + p.sq_off.tail;
	sring->ring_mask = sq_ptr + p.sq_off.ring_mask;
	sring->ring_entries = sq_ptr + p.sq_off.ring_entries;
	sring->flags = sq_ptr + p.sq_off.flags;
	sring->array = sq_ptr + p.sq_off.array;

	/* Map a queue as mem map */
	s->sq_ring.queue.sqes = mmap(0, p.sq_entries * sizeof(struct io_uring_sqe),
				     PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
				     s->ring_fd, IORING_OFF_SQES);
	if (s->sq_ring.queue.sqes == MAP_FAILED) {
		munmap(sq_ptr, sring->ring_sz);
		if (sq_ptr != cq_ptr) {
			ksft_print_msg("failed to mmap uring queue!");
			munmap(cq_ptr, cring->ring_sz);
			return 1;
		}
	}

	cring->head = cq_ptr + p.cq_off.head;
	cring->tail = cq_ptr + p.cq_off.tail;
	cring->ring_mask = cq_ptr + p.cq_off.ring_mask;
	cring->ring_entries = cq_ptr + p.cq_off.ring_entries;
	cring->queue.cqes = cq_ptr + p.cq_off.cqes;

	return 0;
}

/* Init io_uring queues */
int setup_io_uring(struct io_ring *s)
{
	struct io_uring_params para;

	memset(&para, 0, sizeof(para));
	s->ring_fd = sys_uring_setup(URING_QUEUE_SZ, &para);
	if (s->ring_fd < 0)
		return 1;

	return mmap_io_uring(para, s);
}

/*
 * Get data from completion queue. the data buffer saved the file data
 * return 0: success; others: error;
 */
int handle_uring_cq(struct io_ring *s)
{
	struct file_io *fi = NULL;
	struct io_uring_queue *cring = &s->cq_ring;
	struct io_uring_cqe *cqe;
	unsigned int head;
	off_t len = 0;

	head = *cring->head;

	do {
		barrier();
		if (head == *cring->tail)
			break;
		/* Get the entry */
		cqe = &cring->queue.cqes[head & *s->cq_ring.ring_mask];
		fi = (struct file_io *)cqe->user_data;
		if (cqe->res < 0)
			break;

		int blocks = (int)(fi->file_sz + URING_BLOCK_SZ - 1) / URING_BLOCK_SZ;

		for (int i = 0; i < blocks; i++)
			len += fi->iovecs[i].iov_len;

		head++;
	} while (1);

	*cring->head = head;
	barrier();

	return (len != fi->file_sz);
}

/*
 * Submit squeue. specify via IORING_OP_READV.
 * the buffer need to be set metadata according to LAM mode
 */
int handle_uring_sq(struct io_ring *ring, struct file_io *fi, unsigned long lam)
{
	int file_fd = fi->file_fd;
	struct io_uring_queue *sring = &ring->sq_ring;
	unsigned int index = 0, cur_block = 0, tail = 0, next_tail = 0;
	struct io_uring_sqe *sqe;

	off_t remain = fi->file_sz;
	int blocks = (int)(remain + URING_BLOCK_SZ - 1) / URING_BLOCK_SZ;

	while (remain) {
		off_t bytes = remain;
		void *buf;

		if (bytes > URING_BLOCK_SZ)
			bytes = URING_BLOCK_SZ;

		fi->iovecs[cur_block].iov_len = bytes;

		if (posix_memalign(&buf, URING_BLOCK_SZ, URING_BLOCK_SZ))
			return 1;

		fi->iovecs[cur_block].iov_base = (void *)set_metadata((uint64_t)buf, lam);
		remain -= bytes;
		cur_block++;
	}

	next_tail = *sring->tail;
	tail = next_tail;
	next_tail++;

	barrier();

	index = tail & *ring->sq_ring.ring_mask;

	sqe = &ring->sq_ring.queue.sqes[index];
	sqe->fd = file_fd;
	sqe->flags = 0;
	sqe->opcode = IORING_OP_READV;
	sqe->addr = (unsigned long)fi->iovecs;
	sqe->len = blocks;
	sqe->off = 0;
	sqe->user_data = (uint64_t)fi;

	sring->array[index] = index;
	tail = next_tail;

	if (*sring->tail != tail) {
		*sring->tail = tail;
		barrier();
	}

	if (sys_uring_enter(ring->ring_fd, 1, 1, IORING_ENTER_GETEVENTS) < 0)
		return 1;

	return 0;
}

/*
 * Test LAM in async I/O and io_uring, read current binery through io_uring
 * Set metadata in pointers to iovecs buffer.
 */
int do_uring(unsigned long lam)
{
	struct io_ring *ring;
	struct file_io *fi;
	struct stat st;
	int ret = 1;
	char path[PATH_MAX];

	/* get current process path */
	if (readlink("/proc/self/exe", path, PATH_MAX) <= 0)
		return 1;

	int file_fd = open(path, O_RDONLY);

	if (file_fd < 0)
		return 1;

	if (fstat(file_fd, &st) < 0)
		return 1;

	off_t file_sz = st.st_size;

	int blocks = (int)(file_sz + URING_BLOCK_SZ - 1) / URING_BLOCK_SZ;

	fi = malloc(sizeof(*fi) + sizeof(struct iovec) * blocks);
	if (!fi)
		return 1;

	fi->file_sz = file_sz;
	fi->file_fd = file_fd;

	ring = malloc(sizeof(*ring));
	if (!ring)
		return 1;

	memset(ring, 0, sizeof(struct io_ring));

	if (setup_io_uring(ring))
		goto out;

	if (handle_uring_sq(ring, fi, lam))
		goto out;

	ret = handle_uring_cq(ring);

out:
	free(ring);

	for (int i = 0; i < blocks; i++) {
		if (fi->iovecs[i].iov_base) {
			uint64_t addr = ((uint64_t)fi->iovecs[i].iov_base);

			switch (lam) {
			case LAM_U57_BITS: /* Clear bits 62:57 */
				addr = (addr & ~(0x3fULL << 57));
				break;
			}
			free((void *)addr);
			fi->iovecs[i].iov_base = NULL;
		}
	}

	free(fi);

	return ret;
}

int handle_uring(struct testcases *test)
{
	int ret = 0;

	if (test->later == 0 && test->lam != 0)
		if (set_lam(test->lam) != 0)
			return 1;

	if (sigsetjmp(segv_env, 1) == 0) {
		signal(SIGSEGV, segv_handler);
		ret = do_uring(test->lam);
	} else {
		ret = 2;
	}

	return ret;
}

static int fork_test(struct testcases *test)
{
	int ret, child_ret;
	pid_t pid;

	pid = fork();
	if (pid < 0) {
		perror("Fork failed.");
		ret = 1;
	} else if (pid == 0) {
		ret = test->test_func(test);
		exit(ret);
	} else {
		wait(&child_ret);
		ret = WEXITSTATUS(child_ret);
	}

	return ret;
}

static void run_test(struct testcases *test, int count)
{
	int i, ret = 0;

	for (i = 0; i < count; i++) {
		struct testcases *t = test + i;

		/* fork a process to run test case */
		tests_cnt++;
		ret = fork_test(t);

		/* return 3 is not support LA57, the case should be skipped */
		if (ret == 3) {
			ksft_test_result_skip(t->msg);
			continue;
		}

		if (ret != 0)
			ret = (t->expected == ret);
		else
			ret = !(t->expected);

		ksft_test_result(ret, t->msg);
	}
}

static struct testcases uring_cases[] = {
	{
		.later = 0,
		.lam = LAM_U57_BITS,
		.test_func = handle_uring,
		.msg = "URING: LAM_U57. Dereferencing pointer with metadata\n",
	},
	{
		.later = 1,
		.expected = 1,
		.lam = LAM_U57_BITS,
		.test_func = handle_uring,
		.msg = "URING:[Negative] Disable LAM. Dereferencing pointer with metadata.\n",
	},
};

static struct testcases malloc_cases[] = {
	{
		.later = 0,
		.lam = LAM_U57_BITS,
		.test_func = handle_malloc,
		.msg = "MALLOC: LAM_U57. Dereferencing pointer with metadata\n",
	},
	{
		.later = 1,
		.expected = 2,
		.lam = LAM_U57_BITS,
		.test_func = handle_malloc,
		.msg = "MALLOC:[Negative] Disable LAM. Dereferencing pointer with metadata.\n",
	},
};

static struct testcases bits_cases[] = {
	{
		.test_func = handle_max_bits,
		.msg = "BITS: Check default tag bits\n",
	},
};

static struct testcases syscall_cases[] = {
	{
		.later = 0,
		.lam = LAM_U57_BITS,
		.test_func = handle_syscall,
		.msg = "SYSCALL: LAM_U57. syscall with metadata\n",
	},
	{
		.later = 1,
		.expected = 1,
		.lam = LAM_U57_BITS,
		.test_func = handle_syscall,
		.msg = "SYSCALL:[Negative] Disable LAM. Dereferencing pointer with metadata.\n",
	},
};

static struct testcases mmap_cases[] = {
	{
		.later = 1,
		.expected = 0,
		.lam = LAM_U57_BITS,
		.addr = HIGH_ADDR,
		.test_func = handle_mmap,
		.msg = "MMAP: First mmap high address, then set LAM_U57.\n",
	},
	{
		.later = 0,
		.expected = 0,
		.lam = LAM_U57_BITS,
		.addr = HIGH_ADDR,
		.test_func = handle_mmap,
		.msg = "MMAP: First LAM_U57, then High address.\n",
	},
	{
		.later = 0,
		.expected = 0,
		.lam = LAM_U57_BITS,
		.addr = LOW_ADDR,
		.test_func = handle_mmap,
		.msg = "MMAP: First LAM_U57, then Low address.\n",
	},
};

static void cmd_help(void)
{
	printf("usage: lam [-h] [-t test list]\n");
	printf("\t-t test list: run tests specified in the test list, default:0x%x\n", TEST_MASK);
	printf("\t\t0x1:malloc; 0x2:max_bits; 0x4:mmap; 0x8:syscall; 0x10:io_uring.\n");
	printf("\t-h: help\n");
}

int main(int argc, char **argv)
{
	int c = 0;
	unsigned int tests = TEST_MASK;

	tests_cnt = 0;

	if (!cpu_has_lam()) {
		ksft_print_msg("Unsupported LAM feature!\n");
		return -1;
	}

	while ((c = getopt(argc, argv, "ht:")) != -1) {
		switch (c) {
		case 't':
			tests = strtoul(optarg, NULL, 16);
			if (!(tests & TEST_MASK)) {
				ksft_print_msg("Invalid argument!\n");
				return -1;
			}
			break;
		case 'h':
			cmd_help();
			return 0;
		default:
			ksft_print_msg("Invalid argument\n");
			return -1;
		}
	}

	if (tests & FUNC_MALLOC)
		run_test(malloc_cases, ARRAY_SIZE(malloc_cases));

	if (tests & FUNC_BITS)
		run_test(bits_cases, ARRAY_SIZE(bits_cases));

	if (tests & FUNC_MMAP)
		run_test(mmap_cases, ARRAY_SIZE(mmap_cases));

	if (tests & FUNC_SYSCALL)
		run_test(syscall_cases, ARRAY_SIZE(syscall_cases));

	if (tests & FUNC_URING)
		run_test(uring_cases, ARRAY_SIZE(uring_cases));

	ksft_set_plan(tests_cnt);

	return ksft_exit_pass();
}