aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/xen
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/xen')
-rw-r--r--drivers/xen/Makefile2
-rw-r--r--drivers/xen/balloon.c341
-rw-r--r--drivers/xen/biomerge.c8
-rw-r--r--drivers/xen/cpu_hotplug.c14
-rw-r--r--drivers/xen/events/events_base.c2
-rw-r--r--drivers/xen/events/events_fifo.c2
-rw-r--r--drivers/xen/grant-table.c56
-rw-r--r--drivers/xen/privcmd.c10
-rw-r--r--drivers/xen/swiotlb-xen.c43
-rw-r--r--drivers/xen/xenbus/xenbus_client.c128
-rw-r--r--drivers/xen/xenbus/xenbus_probe.c3
-rw-r--r--drivers/xen/xlate_mmu.c124
12 files changed, 497 insertions, 236 deletions
diff --git a/drivers/xen/Makefile b/drivers/xen/Makefile
index e293bc507cbc..aa8a7f71f310 100644
--- a/drivers/xen/Makefile
+++ b/drivers/xen/Makefile
@@ -1,6 +1,4 @@
-ifeq ($(filter y, $(CONFIG_ARM) $(CONFIG_ARM64)),)
obj-$(CONFIG_HOTPLUG_CPU) += cpu_hotplug.o
-endif
obj-$(CONFIG_X86) += fallback.o
obj-y += grant-table.o features.o balloon.o manage.o preempt.o
obj-y += events/
diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
index c79329fcfa78..12eab503efd1 100644
--- a/drivers/xen/balloon.c
+++ b/drivers/xen/balloon.c
@@ -54,6 +54,8 @@
#include <linux/memory.h>
#include <linux/memory_hotplug.h>
#include <linux/percpu-defs.h>
+#include <linux/slab.h>
+#include <linux/sysctl.h>
#include <asm/page.h>
#include <asm/pgalloc.h>
@@ -70,16 +72,64 @@
#include <xen/features.h>
#include <xen/page.h>
+static int xen_hotplug_unpopulated;
+
+#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
+
+static int zero;
+static int one = 1;
+
+static struct ctl_table balloon_table[] = {
+ {
+ .procname = "hotplug_unpopulated",
+ .data = &xen_hotplug_unpopulated,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec_minmax,
+ .extra1 = &zero,
+ .extra2 = &one,
+ },
+ { }
+};
+
+static struct ctl_table balloon_root[] = {
+ {
+ .procname = "balloon",
+ .mode = 0555,
+ .child = balloon_table,
+ },
+ { }
+};
+
+static struct ctl_table xen_root[] = {
+ {
+ .procname = "xen",
+ .mode = 0555,
+ .child = balloon_root,
+ },
+ { }
+};
+
+#endif
+
+/*
+ * Use one extent per PAGE_SIZE to avoid to break down the page into
+ * multiple frame.
+ */
+#define EXTENT_ORDER (fls(XEN_PFN_PER_PAGE) - 1)
+
/*
* balloon_process() state:
*
* BP_DONE: done or nothing to do,
+ * BP_WAIT: wait to be rescheduled,
* BP_EAGAIN: error, go to sleep,
* BP_ECANCELED: error, balloon operation canceled.
*/
enum bp_state {
BP_DONE,
+ BP_WAIT,
BP_EAGAIN,
BP_ECANCELED
};
@@ -91,11 +141,12 @@ struct balloon_stats balloon_stats;
EXPORT_SYMBOL_GPL(balloon_stats);
/* We increase/decrease in batches which fit in a page */
-static xen_pfn_t frame_list[PAGE_SIZE / sizeof(unsigned long)];
+static xen_pfn_t frame_list[PAGE_SIZE / sizeof(xen_pfn_t)];
/* List of ballooned pages, threaded through the mem_map array. */
static LIST_HEAD(ballooned_pages);
+static DECLARE_WAIT_QUEUE_HEAD(balloon_wq);
/* Main work function, always executed in process context. */
static void balloon_process(struct work_struct *work);
@@ -124,6 +175,7 @@ static void __balloon_append(struct page *page)
list_add(&page->lru, &ballooned_pages);
balloon_stats.balloon_low++;
}
+ wake_up(&balloon_wq);
}
static void balloon_append(struct page *page)
@@ -133,17 +185,16 @@ static void balloon_append(struct page *page)
}
/* balloon_retrieve: rescue a page from the balloon, if it is not empty. */
-static struct page *balloon_retrieve(bool prefer_highmem)
+static struct page *balloon_retrieve(bool require_lowmem)
{
struct page *page;
if (list_empty(&ballooned_pages))
return NULL;
- if (prefer_highmem)
- page = list_entry(ballooned_pages.prev, struct page, lru);
- else
- page = list_entry(ballooned_pages.next, struct page, lru);
+ page = list_entry(ballooned_pages.next, struct page, lru);
+ if (require_lowmem && PageHighMem(page))
+ return NULL;
list_del(&page->lru);
if (PageHighMem(page))
@@ -166,6 +217,9 @@ static struct page *balloon_next_page(struct page *page)
static enum bp_state update_schedule(enum bp_state state)
{
+ if (state == BP_WAIT)
+ return BP_WAIT;
+
if (state == BP_ECANCELED)
return BP_ECANCELED;
@@ -193,43 +247,75 @@ static enum bp_state update_schedule(enum bp_state state)
}
#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
-static long current_credit(void)
+static struct resource *additional_memory_resource(phys_addr_t size)
{
- return balloon_stats.target_pages - balloon_stats.current_pages -
- balloon_stats.hotplug_pages;
+ struct resource *res;
+ int ret;
+
+ res = kzalloc(sizeof(*res), GFP_KERNEL);
+ if (!res)
+ return NULL;
+
+ res->name = "System RAM";
+ res->flags = IORESOURCE_MEM | IORESOURCE_BUSY;
+
+ ret = allocate_resource(&iomem_resource, res,
+ size, 0, -1,
+ PAGES_PER_SECTION * PAGE_SIZE, NULL, NULL);
+ if (ret < 0) {
+ pr_err("Cannot allocate new System RAM resource\n");
+ kfree(res);
+ return NULL;
+ }
+
+ return res;
}
-static bool balloon_is_inflated(void)
+static void release_memory_resource(struct resource *resource)
{
- if (balloon_stats.balloon_low || balloon_stats.balloon_high ||
- balloon_stats.balloon_hotplug)
- return true;
- else
- return false;
-}
+ if (!resource)
+ return;
-/*
- * reserve_additional_memory() adds memory region of size >= credit above
- * max_pfn. New region is section aligned and size is modified to be multiple
- * of section size. Those features allow optimal use of address space and
- * establish proper alignment when this function is called first time after
- * boot (last section not fully populated at boot time contains unused memory
- * pages with PG_reserved bit not set; online_pages_range() does not allow page
- * onlining in whole range if first onlined page does not have PG_reserved
- * bit set). Real size of added memory is established at page onlining stage.
- */
+ /*
+ * No need to reset region to identity mapped since we now
+ * know that no I/O can be in this region
+ */
+ release_resource(resource);
+ kfree(resource);
+}
-static enum bp_state reserve_additional_memory(long credit)
+static enum bp_state reserve_additional_memory(void)
{
+ long credit;
+ struct resource *resource;
int nid, rc;
- u64 hotplug_start_paddr;
- unsigned long balloon_hotplug = credit;
+ unsigned long balloon_hotplug;
+
+ credit = balloon_stats.target_pages + balloon_stats.target_unpopulated
+ - balloon_stats.total_pages;
+
+ /*
+ * Already hotplugged enough pages? Wait for them to be
+ * onlined.
+ */
+ if (credit <= 0)
+ return BP_WAIT;
- hotplug_start_paddr = PFN_PHYS(SECTION_ALIGN_UP(max_pfn));
- balloon_hotplug = round_up(balloon_hotplug, PAGES_PER_SECTION);
- nid = memory_add_physaddr_to_nid(hotplug_start_paddr);
+ balloon_hotplug = round_up(credit, PAGES_PER_SECTION);
+
+ resource = additional_memory_resource(balloon_hotplug * PAGE_SIZE);
+ if (!resource)
+ goto err;
+
+ nid = memory_add_physaddr_to_nid(resource->start);
#ifdef CONFIG_XEN_HAVE_PVMMU
+ /*
+ * We don't support PV MMU when Linux and Xen is using
+ * different page granularity.
+ */
+ BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE);
+
/*
* add_memory() will build page tables for the new memory so
* the p2m must contain invalid entries so the correct
@@ -242,29 +328,28 @@ static enum bp_state reserve_additional_memory(long credit)
if (!xen_feature(XENFEAT_auto_translated_physmap)) {
unsigned long pfn, i;
- pfn = PFN_DOWN(hotplug_start_paddr);
+ pfn = PFN_DOWN(resource->start);
for (i = 0; i < balloon_hotplug; i++) {
if (!set_phys_to_machine(pfn + i, INVALID_P2M_ENTRY)) {
pr_warn("set_phys_to_machine() failed, no memory added\n");
- return BP_ECANCELED;
+ goto err;
}
}
}
#endif
- rc = add_memory(nid, hotplug_start_paddr, balloon_hotplug << PAGE_SHIFT);
-
+ rc = add_memory_resource(nid, resource);
if (rc) {
pr_warn("Cannot add additional memory (%i)\n", rc);
- return BP_ECANCELED;
+ goto err;
}
- balloon_hotplug -= credit;
+ balloon_stats.total_pages += balloon_hotplug;
- balloon_stats.hotplug_pages += credit;
- balloon_stats.balloon_hotplug = balloon_hotplug;
-
- return BP_DONE;
+ return BP_WAIT;
+ err:
+ release_memory_resource(resource);
+ return BP_ECANCELED;
}
static void xen_online_page(struct page *page)
@@ -275,11 +360,6 @@ static void xen_online_page(struct page *page)
__balloon_append(page);
- if (balloon_stats.hotplug_pages)
- --balloon_stats.hotplug_pages;
- else
- --balloon_stats.balloon_hotplug;
-
mutex_unlock(&balloon_mutex);
}
@@ -296,53 +376,34 @@ static struct notifier_block xen_memory_nb = {
.priority = 0
};
#else
-static long current_credit(void)
+static enum bp_state reserve_additional_memory(void)
{
- unsigned long target = balloon_stats.target_pages;
-
- target = min(target,
- balloon_stats.current_pages +
- balloon_stats.balloon_low +
- balloon_stats.balloon_high);
-
- return target - balloon_stats.current_pages;
+ balloon_stats.target_pages = balloon_stats.current_pages;
+ return BP_ECANCELED;
}
+#endif /* CONFIG_XEN_BALLOON_MEMORY_HOTPLUG */
-static bool balloon_is_inflated(void)
+static long current_credit(void)
{
- if (balloon_stats.balloon_low || balloon_stats.balloon_high)
- return true;
- else
- return false;
+ return balloon_stats.target_pages - balloon_stats.current_pages;
}
-static enum bp_state reserve_additional_memory(long credit)
+static bool balloon_is_inflated(void)
{
- balloon_stats.target_pages = balloon_stats.current_pages;
- return BP_DONE;
+ return balloon_stats.balloon_low || balloon_stats.balloon_high;
}
-#endif /* CONFIG_XEN_BALLOON_MEMORY_HOTPLUG */
static enum bp_state increase_reservation(unsigned long nr_pages)
{
int rc;
- unsigned long pfn, i;
+ unsigned long i;
struct page *page;
struct xen_memory_reservation reservation = {
.address_bits = 0,
- .extent_order = 0,
+ .extent_order = EXTENT_ORDER,
.domid = DOMID_SELF
};
-#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
- if (!balloon_stats.balloon_low && !balloon_stats.balloon_high) {
- nr_pages = min(nr_pages, balloon_stats.balloon_hotplug);
- balloon_stats.hotplug_pages += nr_pages;
- balloon_stats.balloon_hotplug -= nr_pages;
- return BP_DONE;
- }
-#endif
-
if (nr_pages > ARRAY_SIZE(frame_list))
nr_pages = ARRAY_SIZE(frame_list);
@@ -352,7 +413,11 @@ static enum bp_state increase_reservation(unsigned long nr_pages)
nr_pages = i;
break;
}
- frame_list[i] = page_to_pfn(page);
+
+ /* XENMEM_populate_physmap requires a PFN based on Xen
+ * granularity.
+ */
+ frame_list[i] = page_to_xen_pfn(page);
page = balloon_next_page(page);
}
@@ -366,10 +431,16 @@ static enum bp_state increase_reservation(unsigned long nr_pages)
page = balloon_retrieve(false);
BUG_ON(page == NULL);
- pfn = page_to_pfn(page);
-
#ifdef CONFIG_XEN_HAVE_PVMMU
+ /*
+ * We don't support PV MMU when Linux and Xen is using
+ * different page granularity.
+ */
+ BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE);
+
if (!xen_feature(XENFEAT_auto_translated_physmap)) {
+ unsigned long pfn = page_to_pfn(page);
+
set_phys_to_machine(pfn, frame_list[i]);
/* Link back into the page tables if not highmem. */
@@ -396,23 +467,15 @@ static enum bp_state increase_reservation(unsigned long nr_pages)
static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
{
enum bp_state state = BP_DONE;
- unsigned long pfn, i;
- struct page *page;
+ unsigned long i;
+ struct page *page, *tmp;
int ret;
struct xen_memory_reservation reservation = {
.address_bits = 0,
- .extent_order = 0,
+ .extent_order = EXTENT_ORDER,
.domid = DOMID_SELF
};
-
-#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
- if (balloon_stats.hotplug_pages) {
- nr_pages = min(nr_pages, balloon_stats.hotplug_pages);
- balloon_stats.hotplug_pages -= nr_pages;
- balloon_stats.balloon_hotplug += nr_pages;
- return BP_DONE;
- }
-#endif
+ LIST_HEAD(pages);
if (nr_pages > ARRAY_SIZE(frame_list))
nr_pages = ARRAY_SIZE(frame_list);
@@ -425,8 +488,7 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
break;
}
scrub_page(page);
-
- frame_list[i] = page_to_pfn(page);
+ list_add(&page->lru, &pages);
}
/*
@@ -438,14 +500,25 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
*/
kmap_flush_unused();
- /* Update direct mapping, invalidate P2M, and add to balloon. */
- for (i = 0; i < nr_pages; i++) {
- pfn = frame_list[i];
- frame_list[i] = pfn_to_gfn(pfn);
- page = pfn_to_page(pfn);
+ /*
+ * Setup the frame, update direct mapping, invalidate P2M,
+ * and add to balloon.
+ */
+ i = 0;
+ list_for_each_entry_safe(page, tmp, &pages, lru) {
+ /* XENMEM_decrease_reservation requires a GFN */
+ frame_list[i++] = xen_page_to_gfn(page);
#ifdef CONFIG_XEN_HAVE_PVMMU
+ /*
+ * We don't support PV MMU when Linux and Xen is using
+ * different page granularity.
+ */
+ BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE);
+
if (!xen_feature(XENFEAT_auto_translated_physmap)) {
+ unsigned long pfn = page_to_pfn(page);
+
if (!PageHighMem(page)) {
ret = HYPERVISOR_update_va_mapping(
(unsigned long)__va(pfn << PAGE_SHIFT),
@@ -455,6 +528,7 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
__set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
}
#endif
+ list_del(&page->lru);
balloon_append(page);
}
@@ -492,7 +566,7 @@ static void balloon_process(struct work_struct *work)
if (balloon_is_inflated())
state = increase_reservation(credit);
else
- state = reserve_additional_memory(credit);
+ state = reserve_additional_memory();
}
if (credit < 0)
@@ -520,41 +594,71 @@ void balloon_set_new_target(unsigned long target)
}
EXPORT_SYMBOL_GPL(balloon_set_new_target);
+static int add_ballooned_pages(int nr_pages)
+{
+ enum bp_state st;
+
+ if (xen_hotplug_unpopulated) {
+ st = reserve_additional_memory();
+ if (st != BP_ECANCELED) {
+ mutex_unlock(&balloon_mutex);
+ wait_event(balloon_wq,
+ !list_empty(&ballooned_pages));
+ mutex_lock(&balloon_mutex);
+ return 0;
+ }
+ }
+
+ st = decrease_reservation(nr_pages, GFP_USER);
+ if (st != BP_DONE)
+ return -ENOMEM;
+
+ return 0;
+}
+
/**
* alloc_xenballooned_pages - get pages that have been ballooned out
* @nr_pages: Number of pages to get
* @pages: pages returned
- * @highmem: allow highmem pages
* @return 0 on success, error otherwise
*/
-int alloc_xenballooned_pages(int nr_pages, struct page **pages, bool highmem)
+int alloc_xenballooned_pages(int nr_pages, struct page **pages)
{
int pgno = 0;
struct page *page;
+ int ret;
+
mutex_lock(&balloon_mutex);
+
+ balloon_stats.target_unpopulated += nr_pages;
+
while (pgno < nr_pages) {
- page = balloon_retrieve(highmem);
- if (page && (highmem || !PageHighMem(page))) {
+ page = balloon_retrieve(true);
+ if (page) {
pages[pgno++] = page;
+#ifdef CONFIG_XEN_HAVE_PVMMU
+ /*
+ * We don't support PV MMU when Linux and Xen is using
+ * different page granularity.
+ */
+ BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE);
+
+ ret = xen_alloc_p2m_entry(page_to_pfn(page));
+ if (ret < 0)
+ goto out_undo;
+#endif
} else {
- enum bp_state st;
- if (page)
- balloon_append(page);
- st = decrease_reservation(nr_pages - pgno,
- highmem ? GFP_HIGHUSER : GFP_USER);
- if (st != BP_DONE)
+ ret = add_ballooned_pages(nr_pages - pgno);
+ if (ret < 0)
goto out_undo;
}
}
mutex_unlock(&balloon_mutex);
return 0;
out_undo:
- while (pgno)
- balloon_append(pages[--pgno]);
- /* Free the memory back to the kernel soon */
- schedule_delayed_work(&balloon_worker, 0);
mutex_unlock(&balloon_mutex);
- return -ENOMEM;
+ free_xenballooned_pages(pgno, pages);
+ return ret;
}
EXPORT_SYMBOL(alloc_xenballooned_pages);
@@ -574,6 +678,8 @@ void free_xenballooned_pages(int nr_pages, struct page **pages)
balloon_append(pages[i]);
}
+ balloon_stats.target_unpopulated -= nr_pages;
+
/* The balloon may be too large now. Shrink it if needed. */
if (current_credit())
schedule_delayed_work(&balloon_worker, 0);
@@ -602,6 +708,8 @@ static void __init balloon_add_region(unsigned long start_pfn,
don't subtract from it. */
__balloon_append(page);
}
+
+ balloon_stats.total_pages += extra_pfn_end - start_pfn;
}
static int __init balloon_init(void)
@@ -619,6 +727,7 @@ static int __init balloon_init(void)
balloon_stats.target_pages = balloon_stats.current_pages;
balloon_stats.balloon_low = 0;
balloon_stats.balloon_high = 0;
+ balloon_stats.total_pages = balloon_stats.current_pages;
balloon_stats.schedule_delay = 1;
balloon_stats.max_schedule_delay = 32;
@@ -626,11 +735,9 @@ static int __init balloon_init(void)
balloon_stats.max_retry_count = RETRY_UNLIMITED;
#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
- balloon_stats.hotplug_pages = 0;
- balloon_stats.balloon_hotplug = 0;
-
set_online_page_callback(&xen_online_page);
register_memory_notifier(&xen_memory_nb);
+ register_sysctl_table(xen_root);
#endif
/*
diff --git a/drivers/xen/biomerge.c b/drivers/xen/biomerge.c
index 8ae2fc90e1ea..4da69dbf7dca 100644
--- a/drivers/xen/biomerge.c
+++ b/drivers/xen/biomerge.c
@@ -6,10 +6,18 @@
bool xen_biovec_phys_mergeable(const struct bio_vec *vec1,
const struct bio_vec *vec2)
{
+#if XEN_PAGE_SIZE == PAGE_SIZE
unsigned long bfn1 = pfn_to_bfn(page_to_pfn(vec1->bv_page));
unsigned long bfn2 = pfn_to_bfn(page_to_pfn(vec2->bv_page));
return __BIOVEC_PHYS_MERGEABLE(vec1, vec2) &&
((bfn1 == bfn2) || ((bfn1+1) == bfn2));
+#else
+ /*
+ * XXX: Add support for merging bio_vec when using different page
+ * size in Xen and Linux.
+ */
+ return 0;
+#endif
}
EXPORT_SYMBOL(xen_biovec_phys_mergeable);
diff --git a/drivers/xen/cpu_hotplug.c b/drivers/xen/cpu_hotplug.c
index cc6513a176b0..5676aefdf2bc 100644
--- a/drivers/xen/cpu_hotplug.c
+++ b/drivers/xen/cpu_hotplug.c
@@ -11,15 +11,20 @@
static void enable_hotplug_cpu(int cpu)
{
if (!cpu_present(cpu))
- arch_register_cpu(cpu);
+ xen_arch_register_cpu(cpu);
set_cpu_present(cpu, true);
}
static void disable_hotplug_cpu(int cpu)
{
+ if (cpu_online(cpu)) {
+ lock_device_hotplug();
+ device_offline(get_cpu_device(cpu));
+ unlock_device_hotplug();
+ }
if (cpu_present(cpu))
- arch_unregister_cpu(cpu);
+ xen_arch_unregister_cpu(cpu);
set_cpu_present(cpu, false);
}
@@ -55,7 +60,6 @@ static void vcpu_hotplug(unsigned int cpu)
enable_hotplug_cpu(cpu);
break;
case 0:
- (void)cpu_down(cpu);
disable_hotplug_cpu(cpu);
break;
default:
@@ -102,7 +106,11 @@ static int __init setup_vcpu_hotplug_event(void)
static struct notifier_block xsn_cpu = {
.notifier_call = setup_cpu_watcher };
+#ifdef CONFIG_X86
if (!xen_pv_domain())
+#else
+ if (!xen_domain())
+#endif
return -ENODEV;
register_xenstore_notifier(&xsn_cpu);
diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c
index 6cd5e65c4aff..849500e4e14d 100644
--- a/drivers/xen/events/events_base.c
+++ b/drivers/xen/events/events_base.c
@@ -40,11 +40,11 @@
#include <asm/idle.h>
#include <asm/io_apic.h>
#include <asm/xen/pci.h>
-#include <xen/page.h>
#endif
#include <asm/sync_bitops.h>
#include <asm/xen/hypercall.h>
#include <asm/xen/hypervisor.h>
+#include <xen/page.h>
#include <xen/xen.h>
#include <xen/hvm.h>
diff --git a/drivers/xen/events/events_fifo.c b/drivers/xen/events/events_fifo.c
index 1d4baf56c36b..e3e9e3d46d1b 100644
--- a/drivers/xen/events/events_fifo.c
+++ b/drivers/xen/events/events_fifo.c
@@ -54,7 +54,7 @@
#include "events_internal.h"
-#define EVENT_WORDS_PER_PAGE (PAGE_SIZE / sizeof(event_word_t))
+#define EVENT_WORDS_PER_PAGE (XEN_PAGE_SIZE / sizeof(event_word_t))
#define MAX_EVENT_ARRAY_PAGES (EVTCHN_FIFO_NR_CHANNELS / EVENT_WORDS_PER_PAGE)
struct evtchn_fifo_queue {
diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
index 62f591f8763c..c49f79ed58c5 100644
--- a/drivers/xen/grant-table.c
+++ b/drivers/xen/grant-table.c
@@ -642,7 +642,7 @@ int gnttab_setup_auto_xlat_frames(phys_addr_t addr)
if (xen_auto_xlat_grant_frames.count)
return -EINVAL;
- vaddr = xen_remap(addr, PAGE_SIZE * max_nr_gframes);
+ vaddr = xen_remap(addr, XEN_PAGE_SIZE * max_nr_gframes);
if (vaddr == NULL) {
pr_warn("Failed to ioremap gnttab share frames (addr=%pa)!\n",
&addr);
@@ -654,7 +654,7 @@ int gnttab_setup_auto_xlat_frames(phys_addr_t addr)
return -ENOMEM;
}
for (i = 0; i < max_nr_gframes; i++)
- pfn[i] = PFN_DOWN(addr) + i;
+ pfn[i] = XEN_PFN_DOWN(addr) + i;
xen_auto_xlat_grant_frames.vaddr = vaddr;
xen_auto_xlat_grant_frames.pfn = pfn;
@@ -687,7 +687,7 @@ int gnttab_alloc_pages(int nr_pages, struct page **pages)
int i;
int ret;
- ret = alloc_xenballooned_pages(nr_pages, pages, false);
+ ret = alloc_xenballooned_pages(nr_pages, pages);
if (ret < 0)
return ret;
@@ -776,6 +776,54 @@ void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count)
}
EXPORT_SYMBOL_GPL(gnttab_batch_copy);
+void gnttab_foreach_grant_in_range(struct page *page,
+ unsigned int offset,
+ unsigned int len,
+ xen_grant_fn_t fn,
+ void *data)
+{
+ unsigned int goffset;
+ unsigned int glen;
+ unsigned long xen_pfn;
+
+ len = min_t(unsigned int, PAGE_SIZE - offset, len);
+ goffset = xen_offset_in_page(offset);
+
+ xen_pfn = page_to_xen_pfn(page) + XEN_PFN_DOWN(offset);
+
+ while (len) {
+ glen = min_t(unsigned int, XEN_PAGE_SIZE - goffset, len);
+ fn(pfn_to_gfn(xen_pfn), goffset, glen, data);
+
+ goffset = 0;
+ xen_pfn++;
+ len -= glen;
+ }
+}
+EXPORT_SYMBOL_GPL(gnttab_foreach_grant_in_range);
+
+void gnttab_foreach_grant(struct page **pages,
+ unsigned int nr_grefs,
+ xen_grant_fn_t fn,
+ void *data)
+{
+ unsigned int goffset = 0;
+ unsigned long xen_pfn = 0;
+ unsigned int i;
+
+ for (i = 0; i < nr_grefs; i++) {
+ if ((i % XEN_PFN_PER_PAGE) == 0) {
+ xen_pfn = page_to_xen_pfn(pages[i / XEN_PFN_PER_PAGE]);
+ goffset = 0;
+ }
+
+ fn(pfn_to_gfn(xen_pfn), goffset, XEN_PAGE_SIZE, data);
+
+ goffset += XEN_PAGE_SIZE;
+ xen_pfn++;
+ }
+}
+
int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
struct gnttab_map_grant_ref *kmap_ops,
struct page **pages, unsigned int count)
@@ -978,7 +1026,7 @@ static void gnttab_request_version(void)
{
/* Only version 1 is used, which will always be available. */
grant_table_version = 1;
- grefs_per_grant_frame = PAGE_SIZE / sizeof(struct grant_entry_v1);
+ grefs_per_grant_frame = XEN_PAGE_SIZE / sizeof(struct grant_entry_v1);
gnttab_interface = &gnttab_v1_ops;
pr_info("Grant tables using version %d layout\n", grant_table_version);
diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c
index 5e9adac928e6..df2e6f783318 100644
--- a/drivers/xen/privcmd.c
+++ b/drivers/xen/privcmd.c
@@ -401,7 +401,7 @@ static int alloc_empty_pages(struct vm_area_struct *vma, int numpgs)
if (pages == NULL)
return -ENOMEM;
- rc = alloc_xenballooned_pages(numpgs, pages, 0);
+ rc = alloc_xenballooned_pages(numpgs, pages);
if (rc != 0) {
pr_warn("%s Could not alloc %d pfns rc:%d\n", __func__,
numpgs, rc);
@@ -446,7 +446,7 @@ static long privcmd_ioctl_mmap_batch(void __user *udata, int version)
return -EINVAL;
}
- nr_pages = m.num;
+ nr_pages = DIV_ROUND_UP(m.num, XEN_PFN_PER_PAGE);
if ((m.num <= 0) || (nr_pages > (LONG_MAX >> PAGE_SHIFT)))
return -EINVAL;
@@ -494,7 +494,7 @@ static long privcmd_ioctl_mmap_batch(void __user *udata, int version)
goto out_unlock;
}
if (xen_feature(XENFEAT_auto_translated_physmap)) {
- ret = alloc_empty_pages(vma, m.num);
+ ret = alloc_empty_pages(vma, nr_pages);
if (ret < 0)
goto out_unlock;
} else
@@ -518,6 +518,7 @@ static long privcmd_ioctl_mmap_batch(void __user *udata, int version)
state.global_error = 0;
state.version = version;
+ BUILD_BUG_ON(((PAGE_SIZE / sizeof(xen_pfn_t)) % XEN_PFN_PER_PAGE) != 0);
/* mmap_batch_fn guarantees ret == 0 */
BUG_ON(traverse_pages_block(m.num, sizeof(xen_pfn_t),
&pagelist, mmap_batch_fn, &state));
@@ -582,12 +583,13 @@ static void privcmd_close(struct vm_area_struct *vma)
{
struct page **pages = vma->vm_private_data;
int numpgs = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+ int numgfns = (vma->vm_end - vma->vm_start) >> XEN_PAGE_SHIFT;
int rc;
if (!xen_feature(XENFEAT_auto_translated_physmap) || !numpgs || !pages)
return;
- rc = xen_unmap_domain_gfn_range(vma, numpgs, pages);
+ rc = xen_unmap_domain_gfn_range(vma, numgfns, pages);
if (rc == 0)
free_xenballooned_pages(numpgs, pages);
else
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index 79bc4933b13e..7399782c0998 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -76,27 +76,27 @@ static unsigned long xen_io_tlb_nslabs;
static u64 start_dma_addr;
/*
- * Both of these functions should avoid PFN_PHYS because phys_addr_t
+ * Both of these functions should avoid XEN_PFN_PHYS because phys_addr_t
* can be 32bit when dma_addr_t is 64bit leading to a loss in
* information if the shift is done before casting to 64bit.
*/
static inline dma_addr_t xen_phys_to_bus(phys_addr_t paddr)
{
- unsigned long bfn = pfn_to_bfn(PFN_DOWN(paddr));
- dma_addr_t dma = (dma_addr_t)bfn << PAGE_SHIFT;
+ unsigned long bfn = pfn_to_bfn(XEN_PFN_DOWN(paddr));
+ dma_addr_t dma = (dma_addr_t)bfn << XEN_PAGE_SHIFT;
- dma |= paddr & ~PAGE_MASK;
+ dma |= paddr & ~XEN_PAGE_MASK;
return dma;
}
static inline phys_addr_t xen_bus_to_phys(dma_addr_t baddr)
{
- unsigned long pfn = bfn_to_pfn(PFN_DOWN(baddr));
- dma_addr_t dma = (dma_addr_t)pfn << PAGE_SHIFT;
+ unsigned long xen_pfn = bfn_to_pfn(XEN_PFN_DOWN(baddr));
+ dma_addr_t dma = (dma_addr_t)xen_pfn << XEN_PAGE_SHIFT;
phys_addr_t paddr = dma;
- paddr |= baddr & ~PAGE_MASK;
+ paddr |= baddr & ~XEN_PAGE_MASK;
return paddr;
}
@@ -106,7 +106,7 @@ static inline dma_addr_t xen_virt_to_bus(void *address)
return xen_phys_to_bus(virt_to_phys(address));
}
-static int check_pages_physically_contiguous(unsigned long pfn,
+static int check_pages_physically_contiguous(unsigned long xen_pfn,
unsigned int offset,
size_t length)
{
@@ -114,11 +114,11 @@ static int check_pages_physically_contiguous(unsigned long pfn,
int i;
int nr_pages;
- next_bfn = pfn_to_bfn(pfn);
- nr_pages = (offset + length + PAGE_SIZE-1) >> PAGE_SHIFT;
+ next_bfn = pfn_to_bfn(xen_pfn);
+ nr_pages = (offset + length + XEN_PAGE_SIZE-1) >> XEN_PAGE_SHIFT;
for (i = 1; i < nr_pages; i++) {
- if (pfn_to_bfn(++pfn) != ++next_bfn)
+ if (pfn_to_bfn(++xen_pfn) != ++next_bfn)
return 0;
}
return 1;
@@ -126,28 +126,27 @@ static int check_pages_physically_contiguous(unsigned long pfn,
static inline int range_straddles_page_boundary(phys_addr_t p, size_t size)
{
- unsigned long pfn = PFN_DOWN(p);
- unsigned int offset = p & ~PAGE_MASK;
+ unsigned long xen_pfn = XEN_PFN_DOWN(p);
+ unsigned int offset = p & ~XEN_PAGE_MASK;
- if (offset + size <= PAGE_SIZE)
+ if (offset + size <= XEN_PAGE_SIZE)
return 0;
- if (check_pages_physically_contiguous(pfn, offset, size))
+ if (check_pages_physically_contiguous(xen_pfn, offset, size))
return 0;
return 1;
}
static int is_xen_swiotlb_buffer(dma_addr_t dma_addr)
{
- unsigned long bfn = PFN_DOWN(dma_addr);
- unsigned long pfn = bfn_to_local_pfn(bfn);
- phys_addr_t paddr;
+ unsigned long bfn = XEN_PFN_DOWN(dma_addr);
+ unsigned long xen_pfn = bfn_to_local_pfn(bfn);
+ phys_addr_t paddr = XEN_PFN_PHYS(xen_pfn);
/* If the address is outside our domain, it CAN
* have the same virtual address as another address
* in our domain. Therefore _only_ check address within our domain.
*/
- if (pfn_valid(pfn)) {
- paddr = PFN_PHYS(pfn);
+ if (pfn_valid(PFN_DOWN(paddr))) {
return paddr >= virt_to_phys(xen_io_tlb_start) &&
paddr < virt_to_phys(xen_io_tlb_end);
}
@@ -392,7 +391,7 @@ dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page,
*/
if (dma_capable(dev, dev_addr, size) &&
!range_straddles_page_boundary(phys, size) &&
- !xen_arch_need_swiotlb(dev, PFN_DOWN(phys), PFN_DOWN(dev_addr)) &&
+ !xen_arch_need_swiotlb(dev, phys, dev_addr) &&
!swiotlb_force) {
/* we are not interested in the dma_addr returned by
* xen_dma_map_page, only in the potential cache flushes executed
@@ -551,7 +550,7 @@ xen_swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
dma_addr_t dev_addr = xen_phys_to_bus(paddr);
if (swiotlb_force ||
- xen_arch_need_swiotlb(hwdev, PFN_DOWN(paddr), PFN_DOWN(dev_addr)) ||
+ xen_arch_need_swiotlb(hwdev, paddr, dev_addr) ||
!dma_capable(hwdev, dev_addr, sg->length) ||
range_straddles_page_boundary(paddr, sg->length)) {
phys_addr_t map = swiotlb_tbl_map_single(hwdev,
diff --git a/drivers/xen/xenbus/xenbus_client.c b/drivers/xen/xenbus/xenbus_client.c
index 2ba09c1195c8..056da6ee1a35 100644
--- a/drivers/xen/xenbus/xenbus_client.c
+++ b/drivers/xen/xenbus/xenbus_client.c
@@ -49,6 +49,10 @@
#include "xenbus_probe.h"
+#define XENBUS_PAGES(_grants) (DIV_ROUND_UP(_grants, XEN_PFN_PER_PAGE))
+
+#define XENBUS_MAX_RING_PAGES (XENBUS_PAGES(XENBUS_MAX_RING_GRANTS))
+
struct xenbus_map_node {
struct list_head next;
union {
@@ -57,10 +61,11 @@ struct xenbus_map_node {
} pv;
struct {
struct page *pages[XENBUS_MAX_RING_PAGES];
+ unsigned long addrs[XENBUS_MAX_RING_GRANTS];
void *addr;
} hvm;
};
- grant_handle_t handles[XENBUS_MAX_RING_PAGES];
+ grant_handle_t handles[XENBUS_MAX_RING_GRANTS];
unsigned int nr_handles;
};
@@ -388,7 +393,7 @@ int xenbus_grant_ring(struct xenbus_device *dev, void *vaddr,
}
grefs[i] = err;
- vaddr = vaddr + PAGE_SIZE;
+ vaddr = vaddr + XEN_PAGE_SIZE;
}
return 0;
@@ -479,12 +484,12 @@ static int __xenbus_map_ring(struct xenbus_device *dev,
unsigned int flags,
bool *leaked)
{
- struct gnttab_map_grant_ref map[XENBUS_MAX_RING_PAGES];
- struct gnttab_unmap_grant_ref unmap[XENBUS_MAX_RING_PAGES];
+ struct gnttab_map_grant_ref map[XENBUS_MAX_RING_GRANTS];
+ struct gnttab_unmap_grant_ref unmap[XENBUS_MAX_RING_GRANTS];
int i, j;
int err = GNTST_okay;
- if (nr_grefs > XENBUS_MAX_RING_PAGES)
+ if (nr_grefs > XENBUS_MAX_RING_GRANTS)
return -EINVAL;
for (i = 0; i < nr_grefs; i++) {
@@ -540,22 +545,22 @@ static int xenbus_map_ring_valloc_pv(struct xenbus_device *dev,
{
struct xenbus_map_node *node;
struct vm_struct *area;
- pte_t *ptes[XENBUS_MAX_RING_PAGES];
- phys_addr_t phys_addrs[XENBUS_MAX_RING_PAGES];
+ pte_t *ptes[XENBUS_MAX_RING_GRANTS];
+ phys_addr_t phys_addrs[XENBUS_MAX_RING_GRANTS];
int err = GNTST_okay;
int i;
bool leaked;
*vaddr = NULL;
- if (nr_grefs > XENBUS_MAX_RING_PAGES)
+ if (nr_grefs > XENBUS_MAX_RING_GRANTS)
return -EINVAL;
node = kzalloc(sizeof(*node), GFP_KERNEL);
if (!node)
return -ENOMEM;
- area = alloc_vm_area(PAGE_SIZE * nr_grefs, ptes);
+ area = alloc_vm_area(XEN_PAGE_SIZE * nr_grefs, ptes);
if (!area) {
kfree(node);
return -ENOMEM;
@@ -591,21 +596,44 @@ failed:
return err;
}
+struct map_ring_valloc_hvm
+{
+ unsigned int idx;
+
+ /* Why do we need two arrays? See comment of __xenbus_map_ring */
+ phys_addr_t phys_addrs[XENBUS_MAX_RING_GRANTS];
+ unsigned long addrs[XENBUS_MAX_RING_GRANTS];
+};
+
+static void xenbus_map_ring_setup_grant_hvm(unsigned long gfn,
+ unsigned int goffset,
+ unsigned int len,
+ void *data)
+{
+ struct map_ring_valloc_hvm *info = data;
+ unsigned long vaddr = (unsigned long)gfn_to_virt(gfn);
+
+ info->phys_addrs[info->idx] = vaddr;
+ info->addrs[info->idx] = vaddr;
+
+ info->idx++;
+}
+
static int xenbus_map_ring_valloc_hvm(struct xenbus_device *dev,
grant_ref_t *gnt_ref,
unsigned int nr_grefs,
void **vaddr)
{
struct xenbus_map_node *node;
- int i;
int err;
void *addr;
bool leaked = false;
- /* Why do we need two arrays? See comment of __xenbus_map_ring */
- phys_addr_t phys_addrs[XENBUS_MAX_RING_PAGES];
- unsigned long addrs[XENBUS_MAX_RING_PAGES];
+ struct map_ring_valloc_hvm info = {
+ .idx = 0,
+ };
+ unsigned int nr_pages = XENBUS_PAGES(nr_grefs);
- if (nr_grefs > XENBUS_MAX_RING_PAGES)
+ if (nr_grefs > XENBUS_MAX_RING_GRANTS)
return -EINVAL;
*vaddr = NULL;
@@ -614,25 +642,22 @@ static int xenbus_map_ring_valloc_hvm(struct xenbus_device *dev,
if (!node)
return -ENOMEM;
- err = alloc_xenballooned_pages(nr_grefs, node->hvm.pages,
- false /* lowmem */);
+ err = alloc_xenballooned_pages(nr_pages, node->hvm.pages);
if (err)
goto out_err;
- for (i = 0; i < nr_grefs; i++) {
- unsigned long pfn = page_to_pfn(node->hvm.pages[i]);
- phys_addrs[i] = (unsigned long)pfn_to_kaddr(pfn);
- addrs[i] = (unsigned long)pfn_to_kaddr(pfn);
- }
+ gnttab_foreach_grant(node->hvm.pages, nr_grefs,
+ xenbus_map_ring_setup_grant_hvm,
+ &info);
err = __xenbus_map_ring(dev, gnt_ref, nr_grefs, node->handles,
- phys_addrs, GNTMAP_host_map, &leaked);
+ info.phys_addrs, GNTMAP_host_map, &leaked);
node->nr_handles = nr_grefs;
if (err)
goto out_free_ballooned_pages;
- addr = vmap(node->hvm.pages, nr_grefs, VM_MAP | VM_IOREMAP,
+ addr = vmap(node->hvm.pages, nr_pages, VM_MAP | VM_IOREMAP,
PAGE_KERNEL);
if (!addr) {
err = -ENOMEM;
@@ -650,14 +675,13 @@ static int xenbus_map_ring_valloc_hvm(struct xenbus_device *dev,
out_xenbus_unmap_ring:
if (!leaked)
- xenbus_unmap_ring(dev, node->handles, node->nr_handles,
- addrs);
+ xenbus_unmap_ring(dev, node->handles, nr_grefs, info.addrs);
else
pr_alert("leaking %p size %u page(s)",
- addr, nr_grefs);
+ addr, nr_pages);
out_free_ballooned_pages:
if (!leaked)
- free_xenballooned_pages(nr_grefs, node->hvm.pages);
+ free_xenballooned_pages(nr_pages, node->hvm.pages);
out_err:
kfree(node);
return err;
@@ -687,10 +711,10 @@ int xenbus_map_ring(struct xenbus_device *dev, grant_ref_t *gnt_refs,
unsigned int nr_grefs, grant_handle_t *handles,
unsigned long *vaddrs, bool *leaked)
{
- phys_addr_t phys_addrs[XENBUS_MAX_RING_PAGES];
+ phys_addr_t phys_addrs[XENBUS_MAX_RING_GRANTS];
int i;
- if (nr_grefs > XENBUS_MAX_RING_PAGES)
+ if (nr_grefs > XENBUS_MAX_RING_GRANTS)
return -EINVAL;
for (i = 0; i < nr_grefs; i++)
@@ -723,7 +747,7 @@ EXPORT_SYMBOL_GPL(xenbus_unmap_ring_vfree);
static int xenbus_unmap_ring_vfree_pv(struct xenbus_device *dev, void *vaddr)
{
struct xenbus_map_node *node;
- struct gnttab_unmap_grant_ref unmap[XENBUS_MAX_RING_PAGES];
+ struct gnttab_unmap_grant_ref unmap[XENBUS_MAX_RING_GRANTS];
unsigned int level;
int i;
bool leaked = false;
@@ -750,7 +774,7 @@ static int xenbus_unmap_ring_vfree_pv(struct xenbus_device *dev, void *vaddr)
unsigned long addr;
memset(&unmap[i], 0, sizeof(unmap[i]));
- addr = (unsigned long)vaddr + (PAGE_SIZE * i);
+ addr = (unsigned long)vaddr + (XEN_PAGE_SIZE * i);
unmap[i].host_addr = arbitrary_virt_to_machine(
lookup_address(addr, &level)).maddr;
unmap[i].dev_bus_addr = 0;
@@ -783,13 +807,33 @@ static int xenbus_unmap_ring_vfree_pv(struct xenbus_device *dev, void *vaddr)
return err;
}
+struct unmap_ring_vfree_hvm
+{
+ unsigned int idx;
+ unsigned long addrs[XENBUS_MAX_RING_GRANTS];
+};
+
+static void xenbus_unmap_ring_setup_grant_hvm(unsigned long gfn,
+ unsigned int goffset,
+ unsigned int len,
+ void *data)
+{
+ struct unmap_ring_vfree_hvm *info = data;
+
+ info->addrs[info->idx] = (unsigned long)gfn_to_virt(gfn);
+
+ info->idx++;
+}
+
static int xenbus_unmap_ring_vfree_hvm(struct xenbus_device *dev, void *vaddr)
{
int rv;
struct xenbus_map_node *node;
void *addr;
- unsigned long addrs[XENBUS_MAX_RING_PAGES];
- int i;
+ struct unmap_ring_vfree_hvm info = {
+ .idx = 0,
+ };
+ unsigned int nr_pages;
spin_lock(&xenbus_valloc_lock);
list_for_each_entry(node, &xenbus_valloc_pages, next) {
@@ -809,18 +853,20 @@ static int xenbus_unmap_ring_vfree_hvm(struct xenbus_device *dev, void *vaddr)
return GNTST_bad_virt_addr;
}
- for (i = 0; i < node->nr_handles; i++)
- addrs[i] = (unsigned long)pfn_to_kaddr(page_to_pfn(node->hvm.pages[i]));
+ nr_pages = XENBUS_PAGES(node->nr_handles);
+
+ gnttab_foreach_grant(node->hvm.pages, node->nr_handles,
+ xenbus_unmap_ring_setup_grant_hvm,
+ &info);
rv = xenbus_unmap_ring(dev, node->handles, node->nr_handles,
- addrs);
+ info.addrs);
if (!rv) {
vunmap(vaddr);
- free_xenballooned_pages(node->nr_handles, node->hvm.pages);
+ free_xenballooned_pages(nr_pages, node->hvm.pages);
}
else
- WARN(1, "Leaking %p, size %u page(s)\n", vaddr,
- node->nr_handles);
+ WARN(1, "Leaking %p, size %u page(s)\n", vaddr, nr_pages);
kfree(node);
return rv;
@@ -841,11 +887,11 @@ int xenbus_unmap_ring(struct xenbus_device *dev,
grant_handle_t *handles, unsigned int nr_handles,
unsigned long *vaddrs)
{
- struct gnttab_unmap_grant_ref unmap[XENBUS_MAX_RING_PAGES];
+ struct gnttab_unmap_grant_ref unmap[XENBUS_MAX_RING_GRANTS];
int i;
int err;
- if (nr_handles > XENBUS_MAX_RING_PAGES)
+ if (nr_handles > XENBUS_MAX_RING_GRANTS)
return -EINVAL;
for (i = 0; i < nr_handles; i++)
diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c
index 3cbe0556de26..33a31cfef55d 100644
--- a/drivers/xen/xenbus/xenbus_probe.c
+++ b/drivers/xen/xenbus/xenbus_probe.c
@@ -802,7 +802,8 @@ static int __init xenbus_init(void)
goto out_error;
xen_store_gfn = (unsigned long)v;
xen_store_interface =
- xen_remap(xen_store_gfn << PAGE_SHIFT, PAGE_SIZE);
+ xen_remap(xen_store_gfn << XEN_PAGE_SHIFT,
+ XEN_PAGE_SIZE);
break;
default:
pr_warn("Xenstore state unknown\n");
diff --git a/drivers/xen/xlate_mmu.c b/drivers/xen/xlate_mmu.c
index cff23872c5a9..5063c5e796b7 100644
--- a/drivers/xen/xlate_mmu.c
+++ b/drivers/xen/xlate_mmu.c
@@ -38,31 +38,28 @@
#include <xen/interface/xen.h>
#include <xen/interface/memory.h>
-/* map fgfn of domid to lpfn in the current domain */
-static int map_foreign_page(unsigned long lpfn, unsigned long fgfn,
- unsigned int domid)
-{
- int rc;
- struct xen_add_to_physmap_range xatp = {
- .domid = DOMID_SELF,
- .foreign_domid = domid,
- .size = 1,
- .space = XENMAPSPACE_gmfn_foreign,
- };
- xen_ulong_t idx = fgfn;
- xen_pfn_t gpfn = lpfn;
- int err = 0;
+typedef void (*xen_gfn_fn_t)(unsigned long gfn, void *data);
- set_xen_guest_handle(xatp.idxs, &idx);
- set_xen_guest_handle(xatp.gpfns, &gpfn);
- set_xen_guest_handle(xatp.errs, &err);
+/* Break down the pages in 4KB chunk and call fn for each gfn */
+static void xen_for_each_gfn(struct page **pages, unsigned nr_gfn,
+ xen_gfn_fn_t fn, void *data)
+{
+ unsigned long xen_pfn = 0;
+ struct page *page;
+ int i;
- rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap_range, &xatp);
- return rc < 0 ? rc : err;
+ for (i = 0; i < nr_gfn; i++) {
+ if ((i % XEN_PFN_PER_PAGE) == 0) {
+ page = pages[i / XEN_PFN_PER_PAGE];
+ xen_pfn = page_to_xen_pfn(page);
+ }
+ fn(pfn_to_gfn(xen_pfn++), data);
+ }
}
struct remap_data {
xen_pfn_t *fgfn; /* foreign domain's gfn */
+ int nr_fgfn; /* Number of foreign gfn left to map */
pgprot_t prot;
domid_t domid;
struct vm_area_struct *vma;
@@ -71,24 +68,71 @@ struct remap_data {
struct xen_remap_gfn_info *info;
int *err_ptr;
int mapped;
+
+ /* Hypercall parameters */
+ int h_errs[XEN_PFN_PER_PAGE];
+ xen_ulong_t h_idxs[XEN_PFN_PER_PAGE];
+ xen_pfn_t h_gpfns[XEN_PFN_PER_PAGE];
+
+ int h_iter; /* Iterator */
};
+static void setup_hparams(unsigned long gfn, void *data)
+{
+ struct remap_data *info = data;
+
+ info->h_idxs[info->h_iter] = *info->fgfn;
+ info->h_gpfns[info->h_iter] = gfn;
+ info->h_errs[info->h_iter] = 0;
+
+ info->h_iter++;
+ info->fgfn++;
+}
+
static int remap_pte_fn(pte_t *ptep, pgtable_t token, unsigned long addr,
void *data)
{
struct remap_data *info = data;
struct page *page = info->pages[info->index++];
- unsigned long pfn = page_to_pfn(page);
- pte_t pte = pte_mkspecial(pfn_pte(pfn, info->prot));
- int rc;
+ pte_t pte = pte_mkspecial(pfn_pte(page_to_pfn(page), info->prot));
+ int rc, nr_gfn;
+ uint32_t i;
+ struct xen_add_to_physmap_range xatp = {
+ .domid = DOMID_SELF,
+ .foreign_domid = info->domid,
+ .space = XENMAPSPACE_gmfn_foreign,
+ };
- rc = map_foreign_page(pfn, *info->fgfn, info->domid);
- *info->err_ptr++ = rc;
- if (!rc) {
- set_pte_at(info->vma->vm_mm, addr, ptep, pte);
- info->mapped++;
+ nr_gfn = min_t(typeof(info->nr_fgfn), XEN_PFN_PER_PAGE, info->nr_fgfn);
+ info->nr_fgfn -= nr_gfn;
+
+ info->h_iter = 0;
+ xen_for_each_gfn(&page, nr_gfn, setup_hparams, info);
+ BUG_ON(info->h_iter != nr_gfn);
+
+ set_xen_guest_handle(xatp.idxs, info->h_idxs);
+ set_xen_guest_handle(xatp.gpfns, info->h_gpfns);
+ set_xen_guest_handle(xatp.errs, info->h_errs);
+ xatp.size = nr_gfn;
+
+ rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap_range, &xatp);
+
+ /* info->err_ptr expect to have one error status per Xen PFN */
+ for (i = 0; i < nr_gfn; i++) {
+ int err = (rc < 0) ? rc : info->h_errs[i];
+
+ *(info->err_ptr++) = err;
+ if (!err)
+ info->mapped++;
}
- info->fgfn++;
+
+ /*
+ * Note: The hypercall will return 0 in most of the case if even if
+ * all the fgmfn are not mapped. We still have to update the pte
+ * as the userspace may decide to continue.
+ */
+ if (!rc)
+ set_pte_at(info->vma->vm_mm, addr, ptep, pte);
return 0;
}
@@ -102,13 +146,14 @@ int xen_xlate_remap_gfn_array(struct vm_area_struct *vma,
{
int err;
struct remap_data data;
- unsigned long range = nr << PAGE_SHIFT;
+ unsigned long range = DIV_ROUND_UP(nr, XEN_PFN_PER_PAGE) << PAGE_SHIFT;
/* Kept here for the purpose of making sure code doesn't break
x86 PVOPS */
BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO)));
data.fgfn = gfn;
+ data.nr_fgfn = nr;
data.prot = prot;
data.domid = domid;
data.vma = vma;
@@ -123,21 +168,20 @@ int xen_xlate_remap_gfn_array(struct vm_area_struct *vma,
}
EXPORT_SYMBOL_GPL(xen_xlate_remap_gfn_array);
-int xen_xlate_unmap_gfn_range(struct vm_area_struct *vma,
- int nr, struct page **pages)
+static void unmap_gfn(unsigned long gfn, void *data)
{
- int i;
+ struct xen_remove_from_physmap xrp;
- for (i = 0; i < nr; i++) {
- struct xen_remove_from_physmap xrp;
- unsigned long pfn;
+ xrp.domid = DOMID_SELF;
+ xrp.gpfn = gfn;
+ (void)HYPERVISOR_memory_op(XENMEM_remove_from_physmap, &xrp);
+}
- pfn = page_to_pfn(pages[i]);
+int xen_xlate_unmap_gfn_range(struct vm_area_struct *vma,
+ int nr, struct page **pages)
+{
+ xen_for_each_gfn(pages, nr, unmap_gfn, NULL);
- xrp.domid = DOMID_SELF;
- xrp.gpfn = pfn;
- (void)HYPERVISOR_memory_op(XENMEM_remove_from_physmap, &xrp);
- }
return 0;
}
EXPORT_SYMBOL_GPL(xen_xlate_unmap_gfn_range);
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.