gitlab.arm.com will be in the maintainance mode on Wednesday June 29th 01:00 - 10:00 (UTC+1). Repositories is read only during the maintainance.

gitlab.arm.com will be in the maintainance mode on Wednesday June 29th 01:00 - 10:00 (UTC+1). Repositories is read only during the maintainance.

You are on a read-only GitLab instance.
Commit 22447828 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Pull KVM fixes from Paolo Bonzini:
 "ARM fixes:

   - Another state update on exit to userspace fix

   - Prevent the creation of mixed 32/64 VMs

   - Fix regression with irqbypass not restarting the guest on failed
     connect

   - Fix regression with debug register decoding resulting in
     overlapping access

   - Commit exception state on exit to usrspace

   - Fix the MMU notifier return values

   - Add missing 'static' qualifiers in the new host stage-2 code

  x86 fixes:

   - fix guest missed wakeup with assigned devices

   - fix WARN reported by syzkaller

   - do not use BIT() in UAPI headers

   - make the kvm_amd.avic parameter bool

  PPC fixes:

   - make halt polling heuristics consistent with other architectures

  selftests:

   - various fixes

   - new performance selftest memslot_perf_test

   - test UFFD minor faults in demand_paging_test"

* tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (44 commits)
  selftests: kvm: fix overlapping addresses in memslot_perf_test
  KVM: X86: Kill off ctxt->ud
  KVM: X86: Fix warning caused by stale emulation context
  KVM: X86: Use kvm_get_linear_rip() in single-step and #DB/#BP interception
  KVM: x86/mmu: Fix comment mentioning skip_4k
  KVM: VMX: update vcpu posted-interrupt descriptor when assigning device
  KVM: rename KVM_REQ_PENDING_TIMER to KVM_REQ_UNBLOCK
  KVM: x86: add start_assignment hook to kvm_x86_ops
  KVM: LAPIC: Narrow the timer latency between wait_lapic_expire and world switch
  selftests: kvm: do only 1 memslot_perf_test run by default
  KVM: X86: Use _BITUL() macro in UAPI headers
  KVM: selftests: add shared hugetlbfs backing source type
  KVM: selftests: allow using UFFD minor faults for demand paging
  KVM: selftests: create alias mappings when using shared memory
  KVM: selftests: add shmem backing source type
  KVM: selftests: refactor vm_mem_backing_src_type flags
  KVM: selftests: allow different backing source types
  KVM: selftests: compute correct demand paging size
  KVM: selftests: simplify setup_demand_paging error handling
  KVM: selftests: Print a message if /dev/kvm is missing
  ...
parents 866c4b8a 000ac429
......@@ -31,6 +31,34 @@ static void *align(void *x, size_t size)
return (void *) (((size_t) x + mask) & ~mask);
}
/*
* Open KVM_DEV_PATH if available, otherwise exit the entire program.
*
* Input Args:
* flags - The flags to pass when opening KVM_DEV_PATH.
*
* Return:
* The opened file descriptor of /dev/kvm.
*/
static int _open_kvm_dev_path_or_exit(int flags)
{
int fd;
fd = open(KVM_DEV_PATH, flags);
if (fd < 0) {
print_skip("%s not available, is KVM loaded? (errno: %d)",
KVM_DEV_PATH, errno);
exit(KSFT_SKIP);
}
return fd;
}
int open_kvm_dev_path_or_exit(void)
{
return _open_kvm_dev_path_or_exit(O_RDONLY);
}
/*
* Capability
*
......@@ -52,10 +80,7 @@ int kvm_check_cap(long cap)
int ret;
int kvm_fd;
kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
if (kvm_fd < 0)
exit(KSFT_SKIP);
kvm_fd = open_kvm_dev_path_or_exit();
ret = ioctl(kvm_fd, KVM_CHECK_EXTENSION, cap);
TEST_ASSERT(ret != -1, "KVM_CHECK_EXTENSION IOCTL failed,\n"
" rc: %i errno: %i", ret, errno);
......@@ -128,9 +153,7 @@ void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size)
static void vm_open(struct kvm_vm *vm, int perm)
{
vm->kvm_fd = open(KVM_DEV_PATH, perm);
if (vm->kvm_fd < 0)
exit(KSFT_SKIP);
vm->kvm_fd = _open_kvm_dev_path_or_exit(perm);
if (!kvm_check_cap(KVM_CAP_IMMEDIATE_EXIT)) {
print_skip("immediate_exit not available");
......@@ -203,7 +226,9 @@ struct kvm_vm *vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm)
TEST_ASSERT(vm != NULL, "Insufficient Memory");
INIT_LIST_HEAD(&vm->vcpus);
INIT_LIST_HEAD(&vm->userspace_mem_regions);
vm->regions.gpa_tree = RB_ROOT;
vm->regions.hva_tree = RB_ROOT;
hash_init(vm->regions.slot_hash);
vm->mode = mode;
vm->type = 0;
......@@ -295,7 +320,7 @@ struct kvm_vm *vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus,
*/
uint64_t vcpu_pages = (DEFAULT_STACK_PGS + num_percpu_pages) * nr_vcpus;
uint64_t extra_pg_pages = (extra_mem_pages + vcpu_pages) / PTES_PER_MIN_PAGE * 2;
uint64_t pages = DEFAULT_GUEST_PHY_PAGES + vcpu_pages + extra_pg_pages;
uint64_t pages = DEFAULT_GUEST_PHY_PAGES + extra_mem_pages + vcpu_pages + extra_pg_pages;
struct kvm_vm *vm;
int i;
......@@ -355,13 +380,14 @@ struct kvm_vm *vm_create_default(uint32_t vcpuid, uint64_t extra_mem_pages,
*/
void kvm_vm_restart(struct kvm_vm *vmp, int perm)
{
int ctr;
struct userspace_mem_region *region;
vm_open(vmp, perm);
if (vmp->has_irqchip)
vm_create_irqchip(vmp);
list_for_each_entry(region, &vmp->userspace_mem_regions, list) {
hash_for_each(vmp->regions.slot_hash, ctr, region, slot_node) {
int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION, &region->region);
TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
" rc: %i errno: %i\n"
......@@ -424,14 +450,21 @@ uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm)
static struct userspace_mem_region *
userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end)
{
struct userspace_mem_region *region;
struct rb_node *node;
list_for_each_entry(region, &vm->userspace_mem_regions, list) {
for (node = vm->regions.gpa_tree.rb_node; node; ) {
struct userspace_mem_region *region =
container_of(node, struct userspace_mem_region, gpa_node);
uint64_t existing_start = region->region.guest_phys_addr;
uint64_t existing_end = region->region.guest_phys_addr
+ region->region.memory_size - 1;
if (start <= existing_end && end >= existing_start)
return region;
if (start < existing_start)
node = node->rb_left;
else
node = node->rb_right;
}
return NULL;
......@@ -546,11 +579,16 @@ void kvm_vm_release(struct kvm_vm *vmp)
}
static void __vm_mem_region_delete(struct kvm_vm *vm,
struct userspace_mem_region *region)
struct userspace_mem_region *region,
bool unlink)
{
int ret;
list_del(&region->list);
if (unlink) {
rb_erase(&region->gpa_node, &vm->regions.gpa_tree);
rb_erase(&region->hva_node, &vm->regions.hva_tree);
hash_del(&region->slot_node);
}
region->region.memory_size = 0;
ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, &region->region);
......@@ -569,14 +607,16 @@ static void __vm_mem_region_delete(struct kvm_vm *vm,
*/
void kvm_vm_free(struct kvm_vm *vmp)
{
struct userspace_mem_region *region, *tmp;
int ctr;
struct hlist_node *node;
struct userspace_mem_region *region;
if (vmp == NULL)
return;
/* Free userspace_mem_regions. */
list_for_each_entry_safe(region, tmp, &vmp->userspace_mem_regions, list)
__vm_mem_region_delete(vmp, region);
hash_for_each_safe(vmp->regions.slot_hash, ctr, node, region, slot_node)
__vm_mem_region_delete(vmp, region, false);
/* Free sparsebit arrays. */
sparsebit_free(&vmp->vpages_valid);
......@@ -658,13 +698,64 @@ int kvm_memcmp_hva_gva(void *hva, struct kvm_vm *vm, vm_vaddr_t gva, size_t len)
return 0;
}
static void vm_userspace_mem_region_gpa_insert(struct rb_root *gpa_tree,
struct userspace_mem_region *region)
{
struct rb_node **cur, *parent;
for (cur = &gpa_tree->rb_node, parent = NULL; *cur; ) {
struct userspace_mem_region *cregion;
cregion = container_of(*cur, typeof(*cregion), gpa_node);
parent = *cur;
if (region->region.guest_phys_addr <
cregion->region.guest_phys_addr)
cur = &(*cur)->rb_left;
else {
TEST_ASSERT(region->region.guest_phys_addr !=
cregion->region.guest_phys_addr,
"Duplicate GPA in region tree");
cur = &(*cur)->rb_right;
}
}
rb_link_node(&region->gpa_node, parent, cur);
rb_insert_color(&region->gpa_node, gpa_tree);
}
static void vm_userspace_mem_region_hva_insert(struct rb_root *hva_tree,
struct userspace_mem_region *region)
{
struct rb_node **cur, *parent;
for (cur = &hva_tree->rb_node, parent = NULL; *cur; ) {
struct userspace_mem_region *cregion;
cregion = container_of(*cur, typeof(*cregion), hva_node);
parent = *cur;
if (region->host_mem < cregion->host_mem)
cur = &(*cur)->rb_left;
else {
TEST_ASSERT(region->host_mem !=
cregion->host_mem,
"Duplicate HVA in region tree");
cur = &(*cur)->rb_right;
}
}
rb_link_node(&region->hva_node, parent, cur);
rb_insert_color(&region->hva_node, hva_tree);
}
/*
* VM Userspace Memory Region Add
*
* Input Args:
* vm - Virtual Machine
* backing_src - Storage source for this region.
* NULL to use anonymous memory.
* src_type - Storage source for this region.
* NULL to use anonymous memory.
* guest_paddr - Starting guest physical address
* slot - KVM region slot
* npages - Number of physical pages
......@@ -722,7 +813,8 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm,
(uint64_t) region->region.memory_size);
/* Confirm no region with the requested slot already exists. */
list_for_each_entry(region, &vm->userspace_mem_regions, list) {
hash_for_each_possible(vm->regions.slot_hash, region, slot_node,
slot) {
if (region->region.slot != slot)
continue;
......@@ -755,11 +847,30 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm,
if (alignment > 1)
region->mmap_size += alignment;
region->fd = -1;
if (backing_src_is_shared(src_type)) {
int memfd_flags = MFD_CLOEXEC;
if (src_type == VM_MEM_SRC_SHARED_HUGETLB)
memfd_flags |= MFD_HUGETLB;
region->fd = memfd_create("kvm_selftest", memfd_flags);
TEST_ASSERT(region->fd != -1,
"memfd_create failed, errno: %i", errno);
ret = ftruncate(region->fd, region->mmap_size);
TEST_ASSERT(ret == 0, "ftruncate failed, errno: %i", errno);
ret = fallocate(region->fd,
FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 0,
region->mmap_size);
TEST_ASSERT(ret == 0, "fallocate failed, errno: %i", errno);
}
region->mmap_start = mmap(NULL, region->mmap_size,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS
| vm_mem_backing_src_alias(src_type)->flag,
-1, 0);
vm_mem_backing_src_alias(src_type)->flag,
region->fd, 0);
TEST_ASSERT(region->mmap_start != MAP_FAILED,
"test_malloc failed, mmap_start: %p errno: %i",
region->mmap_start, errno);
......@@ -793,8 +904,23 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm,
ret, errno, slot, flags,
guest_paddr, (uint64_t) region->region.memory_size);
/* Add to linked-list of memory regions. */
list_add(&region->list, &vm->userspace_mem_regions);
/* Add to quick lookup data structures */
vm_userspace_mem_region_gpa_insert(&vm->regions.gpa_tree, region);
vm_userspace_mem_region_hva_insert(&vm->regions.hva_tree, region);
hash_add(vm->regions.slot_hash, &region->slot_node, slot);
/* If shared memory, create an alias. */
if (region->fd >= 0) {
region->mmap_alias = mmap(NULL, region->mmap_size,
PROT_READ | PROT_WRITE,
vm_mem_backing_src_alias(src_type)->flag,
region->fd, 0);
TEST_ASSERT(region->mmap_alias != MAP_FAILED,
"mmap of alias failed, errno: %i", errno);
/* Align host alias address */
region->host_alias = align(region->mmap_alias, alignment);
}
}
/*
......@@ -817,10 +943,10 @@ memslot2region(struct kvm_vm *vm, uint32_t memslot)
{
struct userspace_mem_region *region;
list_for_each_entry(region, &vm->userspace_mem_regions, list) {
hash_for_each_possible(vm->regions.slot_hash, region, slot_node,
memslot)
if (region->region.slot == memslot)
return region;
}
fprintf(stderr, "No mem region with the requested slot found,\n"
" requested slot: %u\n", memslot);
......@@ -905,7 +1031,7 @@ void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa)
*/
void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot)
{
__vm_mem_region_delete(vm, memslot2region(vm, slot));
__vm_mem_region_delete(vm, memslot2region(vm, slot), true);
}
/*
......@@ -925,9 +1051,7 @@ static int vcpu_mmap_sz(void)
{
int dev_fd, ret;
dev_fd = open(KVM_DEV_PATH, O_RDONLY);
if (dev_fd < 0)
exit(KSFT_SKIP);
dev_fd = open_kvm_dev_path_or_exit();
ret = ioctl(dev_fd, KVM_GET_VCPU_MMAP_SIZE, NULL);
TEST_ASSERT(ret >= sizeof(struct kvm_run),
......@@ -1099,6 +1223,9 @@ vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);
virt_pgd_alloc(vm, pgd_memslot);
vm_paddr_t paddr = vm_phy_pages_alloc(vm, pages,
KVM_UTIL_MIN_PFN * vm->page_size,
data_memslot);
/*
* Find an unused range of virtual page addresses of at least
......@@ -1108,11 +1235,7 @@ vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
/* Map the virtual pages. */
for (vm_vaddr_t vaddr = vaddr_start; pages > 0;
pages--, vaddr += vm->page_size) {
vm_paddr_t paddr;
paddr = vm_phy_page_alloc(vm,
KVM_UTIL_MIN_PFN * vm->page_size, data_memslot);
pages--, vaddr += vm->page_size, paddr += vm->page_size) {
virt_pg_map(vm, vaddr, paddr, pgd_memslot);
......@@ -1177,16 +1300,14 @@ void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa)
{
struct userspace_mem_region *region;
list_for_each_entry(region, &vm->userspace_mem_regions, list) {
if ((gpa >= region->region.guest_phys_addr)
&& (gpa <= (region->region.guest_phys_addr
+ region->region.memory_size - 1)))
return (void *) ((uintptr_t) region->host_mem
+ (gpa - region->region.guest_phys_addr));
region = userspace_mem_region_find(vm, gpa, gpa);
if (!region) {
TEST_FAIL("No vm physical memory at 0x%lx", gpa);
return NULL;
}
TEST_FAIL("No vm physical memory at 0x%lx", gpa);
return NULL;
return (void *)((uintptr_t)region->host_mem
+ (gpa - region->region.guest_phys_addr));
}
/*
......@@ -1208,21 +1329,64 @@ void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa)
*/
vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva)
{
struct userspace_mem_region *region;
struct rb_node *node;
list_for_each_entry(region, &vm->userspace_mem_regions, list) {
if ((hva >= region->host_mem)
&& (hva <= (region->host_mem
+ region->region.memory_size - 1)))
return (vm_paddr_t) ((uintptr_t)
region->region.guest_phys_addr
+ (hva - (uintptr_t) region->host_mem));
for (node = vm->regions.hva_tree.rb_node; node; ) {
struct userspace_mem_region *region =
container_of(node, struct userspace_mem_region, hva_node);
if (hva >= region->host_mem) {
if (hva <= (region->host_mem
+ region->region.memory_size - 1))
return (vm_paddr_t)((uintptr_t)
region->region.guest_phys_addr
+ (hva - (uintptr_t)region->host_mem));
node = node->rb_right;
} else
node = node->rb_left;
}
TEST_FAIL("No mapping to a guest physical address, hva: %p", hva);
return -1;
}
/*
* Address VM physical to Host Virtual *alias*.
*
* Input Args:
* vm - Virtual Machine
* gpa - VM physical address
*
* Output Args: None
*
* Return:
* Equivalent address within the host virtual *alias* area, or NULL
* (without failing the test) if the guest memory is not shared (so
* no alias exists).
*
* When vm_create() and related functions are called with a shared memory
* src_type, we also create a writable, shared alias mapping of the
* underlying guest memory. This allows the host to manipulate guest memory
* without mapping that memory in the guest's address space. And, for
* userfaultfd-based demand paging, we can do so without triggering userfaults.
*/
void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa)
{
struct userspace_mem_region *region;
uintptr_t offset;
region = userspace_mem_region_find(vm, gpa, gpa);
if (!region)
return NULL;
if (!region->host_alias)
return NULL;
offset = gpa - region->region.guest_phys_addr;
return (void *) ((uintptr_t) region->host_alias + offset);
}
/*
* VM Create IRQ Chip
*
......@@ -1822,6 +1986,7 @@ int kvm_device_access(int dev_fd, uint32_t group, uint64_t attr,
*/
void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
{
int ctr;
struct userspace_mem_region *region;
struct vcpu *vcpu;
......@@ -1829,7 +1994,7 @@ void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
fprintf(stream, "%*sfd: %i\n", indent, "", vm->fd);
fprintf(stream, "%*spage_size: 0x%x\n", indent, "", vm->page_size);
fprintf(stream, "%*sMem Regions:\n", indent, "");
list_for_each_entry(region, &vm->userspace_mem_regions, list) {
hash_for_each(vm->regions.slot_hash, ctr, region, slot_node) {
fprintf(stream, "%*sguest_phys: 0x%lx size: 0x%lx "
"host_virt: %p\n", indent + 2, "",
(uint64_t) region->region.guest_phys_addr,
......@@ -2015,10 +2180,7 @@ bool vm_is_unrestricted_guest(struct kvm_vm *vm)
if (vm == NULL) {
/* Ensure that the KVM vendor-specific module is loaded. */
f = fopen(KVM_DEV_PATH, "r");
TEST_ASSERT(f != NULL, "Error in opening KVM dev file: %d",
errno);
fclose(f);
close(open_kvm_dev_path_or_exit());
}
f = fopen("/sys/module/kvm_intel/parameters/unrestricted_guest", "r");
......@@ -2041,7 +2203,7 @@ unsigned int vm_get_page_shift(struct kvm_vm *vm)
return vm->page_shift;
}
unsigned int vm_get_max_gfn(struct kvm_vm *vm)
uint64_t vm_get_max_gfn(struct kvm_vm *vm)
{
return vm->max_gfn;
}
......
......@@ -8,6 +8,9 @@
#ifndef SELFTEST_KVM_UTIL_INTERNAL_H
#define SELFTEST_KVM_UTIL_INTERNAL_H
#include "linux/hashtable.h"
#include "linux/rbtree.h"
#include "sparsebit.h"
struct userspace_mem_region {
......@@ -16,9 +19,13 @@ struct userspace_mem_region {
int fd;
off_t offset;
void *host_mem;
void *host_alias;
void *mmap_start;
void *mmap_alias;
size_t mmap_size;
struct list_head list;
struct rb_node gpa_node;
struct rb_node hva_node;
struct hlist_node slot_node;
};
struct vcpu {
......@@ -31,6 +38,12 @@ struct vcpu {
uint32_t dirty_gfns_count;
};
struct userspace_mem_regions {
struct rb_root gpa_tree;
struct rb_root hva_tree;
DECLARE_HASHTABLE(slot_hash, 9);
};
struct kvm_vm {
int mode;
unsigned long type;
......@@ -43,7 +56,7 @@ struct kvm_vm {
unsigned int va_bits;
uint64_t max_gfn;
struct list_head vcpus;
struct list_head userspace_mem_regions;
struct userspace_mem_regions regions;
struct sparsebit *vpages_valid;
struct sparsebit *vpages_mapped;
bool has_irqchip;
......
......@@ -2,6 +2,7 @@
/*
* Copyright (C) 2020, Google LLC.
*/
#include <inttypes.h>
#include "kvm_util.h"
#include "perf_test_util.h"
......@@ -80,7 +81,8 @@ struct kvm_vm *perf_test_create_vm(enum vm_guest_mode mode, int vcpus,
*/
TEST_ASSERT(guest_num_pages < vm_get_max_gfn(vm),
"Requested more guest memory than address space allows.\n"
" guest pages: %lx max gfn: %x vcpus: %d wss: %lx]\n",
" guest pages: %" PRIx64 " max gfn: %" PRIx64
" vcpus: %d wss: %" PRIx64 "]\n",
guest_num_pages, vm_get_max_gfn(vm), vcpus,
vcpu_memory_bytes);
......
#include "../../../../lib/rbtree.c"
......@@ -168,70 +168,87 @@ size_t get_def_hugetlb_pagesz(void)
const struct vm_mem_backing_src_alias *vm_mem_backing_src_alias(uint32_t i)
{
static const int anon_flags = MAP_PRIVATE | MAP_ANONYMOUS;
static const int anon_huge_flags = anon_flags | MAP_HUGETLB;
static const struct vm_mem_backing_src_alias aliases[] = {
[VM_MEM_SRC_ANONYMOUS] = {
.name = "anonymous",
.flag = 0,
.flag = anon_flags,
},
[VM_MEM_SRC_ANONYMOUS_THP] = {
.name = "anonymous_thp",
.flag = 0,
.flag = anon_flags,
},
[VM_MEM_SRC_ANONYMOUS_HUGETLB] = {
.name = "anonymous_hugetlb",
.flag = MAP_HUGETLB,
.flag = anon_huge_flags,
},
[VM_MEM_SRC_ANONYMOUS_HUGETLB_16KB] = {
.name = "anonymous_hugetlb_16kb",
.flag = MAP_HUGETLB | MAP_HUGE_16KB,
.flag = anon_huge_flags | MAP_HUGE_16KB,
},
[VM_MEM_SRC_ANONYMOUS_HUGETLB_64KB] = {
.name = "anonymous_hugetlb_64kb",
.flag = MAP_HUGETLB | MAP_HUGE_64KB,
.flag = anon_huge_flags | MAP_HUGE_64KB,
},
[VM_MEM_SRC_ANONYMOUS_HUGETLB_512KB] = {
.name = "anonymous_hugetlb_512kb",
.flag = MAP_HUGETLB | MAP_HUGE_512KB,
.flag = anon_huge_flags | MAP_HUGE_512KB,
},
[VM_MEM_SRC_ANONYMOUS_HUGETLB_1MB] = {
.name = "anonymous_hugetlb_1mb",
.flag = MAP_HUGETLB | MAP_HUGE_1MB,
.flag = anon_huge_flags | MAP_HUGE_1MB,
},
[VM_MEM_SRC_ANONYMOUS_HUGETLB_2MB] = {
.name = "anonymous_hugetlb_2mb",
.flag = MAP_HUGETLB | MAP_HUGE_2MB,
.flag = anon_huge_flags | MAP_HUGE_2MB,
},
[VM_MEM_SRC_ANONYMOUS_HUGETLB_8MB] = {
.name = "anonymous_hugetlb_8mb",
.flag = MAP_HUGETLB | MAP_HUGE_8MB,
.flag = anon_huge_flags | MAP_HUGE_8MB,
},
[VM_MEM_SRC_ANONYMOUS_HUGETLB_16MB] = {
.name = "anonymous_hugetlb_16mb",
.flag = MAP_HUGETLB | MAP_HUGE_16MB,
.flag = anon_huge_flags | MAP_HUGE_16MB,
},
[VM_MEM_SRC_ANONYMOUS_HUGETLB_32MB] = {
.name = "anonymous_hugetlb_32mb",