Commit c3a82e06 authored by Stephen Rothwell's avatar Stephen Rothwell
Browse files

Merge remote-tracking branch 'iomem-mmap-vs-gup/topic/iomem-mmap-vs-gup'

parents da10c99e 74b30195
......@@ -31,9 +31,6 @@
#include <linux/uio.h>
#include <linux/uaccess.h>
#include <linux/security.h>
#include <linux/pseudo_fs.h>
#include <uapi/linux/magic.h>
#include <linux/mount.h>
#ifdef CONFIG_IA64
# include <linux/efi.h>
......@@ -829,42 +826,6 @@ static loff_t memory_lseek(struct file *file, loff_t offset, int orig)
return ret;
}
static struct inode *devmem_inode;
#ifdef CONFIG_IO_STRICT_DEVMEM
void revoke_devmem(struct resource *res)
{
/* pairs with smp_store_release() in devmem_init_inode() */
struct inode *inode = smp_load_acquire(&devmem_inode);
/*
* Check that the initialization has completed. Losing the race
* is ok because it means drivers are claiming resources before
* the fs_initcall level of init and prevent /dev/mem from
* establishing mappings.
*/
if (!inode)
return;
/*
* The expectation is that the driver has successfully marked
* the resource busy by this point, so devmem_is_allowed()
* should start returning false, however for performance this
* does not iterate the entire resource range.
*/
if (devmem_is_allowed(PHYS_PFN(res->start)) &&
devmem_is_allowed(PHYS_PFN(res->end))) {
/*
* *cringe* iomem=relaxed says "go ahead, what's the
* worst that can happen?"
*/
return;
}
unmap_mapping_range(inode->i_mapping, res->start, resource_size(res), 1);
}
#endif
static int open_port(struct inode *inode, struct file *filp)
{
int rc;
......@@ -884,8 +845,7 @@ static int open_port(struct inode *inode, struct file *filp)
* revocations when drivers want to take over a /dev/mem mapped
* range.
*/
inode->i_mapping = devmem_inode->i_mapping;
filp->f_mapping = inode->i_mapping;
filp->f_mapping = iomem_get_mapping();
return 0;
}
......@@ -1017,48 +977,6 @@ static char *mem_devnode(struct device *dev, umode_t *mode)
static struct class *mem_class;
static int devmem_fs_init_fs_context(struct fs_context *fc)
{
return init_pseudo(fc, DEVMEM_MAGIC) ? 0 : -ENOMEM;
}
static struct file_system_type devmem_fs_type = {
.name = "devmem",
.owner = THIS_MODULE,
.init_fs_context = devmem_fs_init_fs_context,
.kill_sb = kill_anon_super,
};
static int devmem_init_inode(void)
{
static struct vfsmount *devmem_vfs_mount;
static int devmem_fs_cnt;
struct inode *inode;
int rc;
rc = simple_pin_fs(&devmem_fs_type, &devmem_vfs_mount, &devmem_fs_cnt);
if (rc < 0) {
pr_err("Cannot mount /dev/mem pseudo filesystem: %d\n", rc);
return rc;
}
inode = alloc_anon_inode(devmem_vfs_mount->mnt_sb);
if (IS_ERR(inode)) {
rc = PTR_ERR(inode);
pr_err("Cannot allocate inode for /dev/mem: %d\n", rc);
simple_release_fs(&devmem_vfs_mount, &devmem_fs_cnt);
return rc;
}
/*
* Publish /dev/mem initialized.
* Pairs with smp_load_acquire() in revoke_devmem().
*/
smp_store_release(&devmem_inode, inode);
return 0;
}
static int __init chr_dev_init(void)
{
int minor;
......@@ -1080,8 +998,6 @@ static int __init chr_dev_init(void)
*/
if ((minor == DEVPORT_MINOR) && !arch_has_dev_port())
continue;
if ((minor == DEVMEM_MINOR) && devmem_init_inode() != 0)
continue;
device_create(mem_class, NULL, MKDEV(MEM_MAJOR, minor),
NULL, devlist[minor].name);
......
......@@ -89,7 +89,6 @@ comment "Sub-drivers"
config DRM_EXYNOS_G2D
bool "G2D"
depends on VIDEO_SAMSUNG_S5P_G2D=n || COMPILE_TEST
select FRAME_VECTOR
help
Choose this option if you want to use Exynos G2D for DRM.
......
......@@ -205,7 +205,8 @@ struct g2d_cmdlist_userptr {
dma_addr_t dma_addr;
unsigned long userptr;
unsigned long size;
struct frame_vector *vec;
struct page **pages;
unsigned int npages;
struct sg_table *sgt;
atomic_t refcount;
bool in_pool;
......@@ -378,7 +379,6 @@ static void g2d_userptr_put_dma_addr(struct g2d_data *g2d,
bool force)
{
struct g2d_cmdlist_userptr *g2d_userptr = obj;
struct page **pages;
if (!obj)
return;
......@@ -398,15 +398,9 @@ static void g2d_userptr_put_dma_addr(struct g2d_data *g2d,
dma_unmap_sgtable(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt,
DMA_BIDIRECTIONAL, 0);
pages = frame_vector_pages(g2d_userptr->vec);
if (!IS_ERR(pages)) {
int i;
for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++)
set_page_dirty_lock(pages[i]);
}
put_vaddr_frames(g2d_userptr->vec);
frame_vector_destroy(g2d_userptr->vec);
unpin_user_pages_dirty_lock(g2d_userptr->pages, g2d_userptr->npages,
true);
kvfree(g2d_userptr->pages);
if (!g2d_userptr->out_of_list)
list_del_init(&g2d_userptr->list);
......@@ -474,35 +468,35 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d,
offset = userptr & ~PAGE_MASK;
end = PAGE_ALIGN(userptr + size);
npages = (end - start) >> PAGE_SHIFT;
g2d_userptr->vec = frame_vector_create(npages);
if (!g2d_userptr->vec) {
g2d_userptr->pages = kvmalloc_array(npages, sizeof(*g2d_userptr->pages),
GFP_KERNEL);
if (!g2d_userptr->pages) {
ret = -ENOMEM;
goto err_free;
}
ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE,
g2d_userptr->vec);
ret = pin_user_pages_fast(start, npages,
FOLL_FORCE | FOLL_WRITE | FOLL_LONGTERM,
g2d_userptr->pages);
if (ret != npages) {
DRM_DEV_ERROR(g2d->dev,
"failed to get user pages from userptr.\n");
if (ret < 0)
goto err_destroy_framevec;
ret = -EFAULT;
goto err_put_framevec;
}
if (frame_vector_to_pages(g2d_userptr->vec) < 0) {
goto err_destroy_pages;
npages = ret;
ret = -EFAULT;
goto err_put_framevec;
goto err_unpin_pages;
}
g2d_userptr->npages = npages;
sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
if (!sgt) {
ret = -ENOMEM;
goto err_put_framevec;
goto err_unpin_pages;
}
ret = sg_alloc_table_from_pages(sgt,
frame_vector_pages(g2d_userptr->vec),
g2d_userptr->pages,
npages, offset, size, GFP_KERNEL);
if (ret < 0) {
DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n");
......@@ -538,11 +532,11 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d,
err_free_sgt:
kfree(sgt);
err_put_framevec:
put_vaddr_frames(g2d_userptr->vec);
err_unpin_pages:
unpin_user_pages(g2d_userptr->pages, npages);
err_destroy_framevec:
frame_vector_destroy(g2d_userptr->vec);
err_destroy_pages:
kvfree(g2d_userptr->pages);
err_free:
kfree(g2d_userptr);
......
......@@ -9,7 +9,6 @@ config VIDEOBUF2_V4L2
config VIDEOBUF2_MEMOPS
tristate
select FRAME_VECTOR
config VIDEOBUF2_DMA_CONTIG
tristate
......
# SPDX-License-Identifier: GPL-2.0
videobuf2-common-objs := videobuf2-core.o
videobuf2-common-objs += frame_vector.o
ifeq ($(CONFIG_TRACEPOINTS),y)
videobuf2-common-objs += vb2-trace.o
......
......@@ -8,6 +8,8 @@
#include <linux/pagemap.h>
#include <linux/sched.h>
#include <media/frame_vector.h>
/**
* get_vaddr_frames() - map virtual addresses to pfns
* @start: starting user address
......@@ -32,13 +34,12 @@
* This function takes care of grabbing mmap_lock as necessary.
*/
int get_vaddr_frames(unsigned long start, unsigned int nr_frames,
unsigned int gup_flags, struct frame_vector *vec)
struct frame_vector *vec)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
int ret = 0;
int err;
int locked;
if (nr_frames == 0)
return 0;
......@@ -48,40 +49,26 @@ int get_vaddr_frames(unsigned long start, unsigned int nr_frames,
start = untagged_addr(start);
mmap_read_lock(mm);
locked = 1;
vma = find_vma_intersection(mm, start, start + 1);
if (!vma) {
ret = -EFAULT;
goto out;
}
/*
* While get_vaddr_frames() could be used for transient (kernel
* controlled lifetime) pinning of memory pages all current
* users establish long term (userspace controlled lifetime)
* page pinning. Treat get_vaddr_frames() like
* get_user_pages_longterm() and disallow it for filesystem-dax
* mappings.
*/
if (vma_is_fsdax(vma)) {
ret = -EOPNOTSUPP;
goto out;
}
if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) {
ret = pin_user_pages_fast(start, nr_frames,
FOLL_FORCE | FOLL_WRITE | FOLL_LONGTERM,
(struct page **)(vec->ptrs));
if (ret > 0) {
vec->got_ref = true;
vec->is_pfns = false;
ret = pin_user_pages_locked(start, nr_frames,
gup_flags, (struct page **)(vec->ptrs), &locked);
goto out;
goto out_unlocked;
}
mmap_read_lock(mm);
vec->got_ref = false;
vec->is_pfns = true;
ret = 0;
do {
unsigned long *nums = frame_vector_pfns(vec);
vma = find_vma_intersection(mm, start, start + 1);
if (!vma)
break;
while (ret < nr_frames && start + PAGE_SIZE <= vma->vm_end) {
err = follow_pfn(vma, start, &nums[ret]);
if (err) {
......@@ -92,17 +79,13 @@ int get_vaddr_frames(unsigned long start, unsigned int nr_frames,
start += PAGE_SIZE;
ret++;
}
/*
* We stop if we have enough pages or if VMA doesn't completely
* cover the tail page.
*/
if (ret >= nr_frames || start < vma->vm_end)
/* Bail out if VMA doesn't completely cover the tail page. */
if (start < vma->vm_end)
break;
vma = find_vma_intersection(mm, start, start + 1);
} while (vma && vma->vm_flags & (VM_IO | VM_PFNMAP));
} while (ret < nr_frames);
out:
if (locked)
mmap_read_unlock(mm);
mmap_read_unlock(mm);
out_unlocked:
if (!ret)
ret = -EFAULT;
if (ret > 0)
......
......@@ -40,7 +40,6 @@ struct frame_vector *vb2_create_framevec(unsigned long start,
unsigned long first, last;
unsigned long nr;
struct frame_vector *vec;
unsigned int flags = FOLL_FORCE | FOLL_WRITE;
first = start >> PAGE_SHIFT;
last = (start + length - 1) >> PAGE_SHIFT;
......@@ -48,7 +47,7 @@ struct frame_vector *vb2_create_framevec(unsigned long start,
vec = frame_vector_create(nr);
if (!vec)
return ERR_PTR(-ENOMEM);
ret = get_vaddr_frames(start & PAGE_MASK, nr, flags, vec);
ret = get_vaddr_frames(start & PAGE_MASK, nr, vec);
if (ret < 0)
goto out_destroy;
/* We accept only complete set of PFNs */
......
......@@ -12,6 +12,5 @@ config VIDEO_OMAP2_VOUT
depends on VIDEO_V4L2
select VIDEOBUF2_DMA_CONTIG
select OMAP2_VRFB if ARCH_OMAP2 || ARCH_OMAP3
select FRAME_VECTOR
help
V4L2 Display driver support for OMAP2/3 based boards.
......@@ -6,7 +6,6 @@
config HABANA_AI
tristate "HabanaAI accelerators (habanalabs)"
depends on PCI && HAS_IOMEM
select FRAME_VECTOR
select GENERIC_ALLOCATOR
select HWMON
help
......
......@@ -1145,7 +1145,8 @@ struct hl_ctx_mgr {
* struct hl_userptr - memory mapping chunk information
* @vm_type: type of the VM.
* @job_node: linked-list node for hanging the object on the Job's list.
* @vec: pointer to the frame vector.
* @pages: pointer to struct page array
* @npages: size of @pages array
* @sgt: pointer to the scatter-gather table that holds the pages.
* @dir: for DMA unmapping, the direction must be supplied, so save it.
* @debugfs_list: node in debugfs list of command submissions.
......@@ -1156,7 +1157,8 @@ struct hl_ctx_mgr {
struct hl_userptr {
enum vm_type_t vm_type; /* must be first */
struct list_head job_node;
struct frame_vector *vec;
struct page **pages;
unsigned int npages;
struct sg_table *sgt;
enum dma_data_direction dir;
struct list_head debugfs_list;
......
......@@ -1552,45 +1552,40 @@ static int get_user_memory(struct hl_device *hdev, u64 addr, u64 size,
return -EFAULT;
}
userptr->vec = frame_vector_create(npages);
if (!userptr->vec) {
dev_err(hdev->dev, "Failed to create frame vector\n");
userptr->pages = kvmalloc_array(npages, sizeof(*userptr->pages),
GFP_KERNEL);
if (!userptr->pages)
return -ENOMEM;
}
rc = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE,
userptr->vec);
rc = pin_user_pages_fast(start, npages,
FOLL_FORCE | FOLL_WRITE | FOLL_LONGTERM,
userptr->pages);
if (rc != npages) {
dev_err(hdev->dev,
"Failed to map host memory, user ptr probably wrong\n");
if (rc < 0)
goto destroy_framevec;
rc = -EFAULT;
goto put_framevec;
}
if (frame_vector_to_pages(userptr->vec) < 0) {
dev_err(hdev->dev,
"Failed to translate frame vector to pages\n");
goto destroy_pages;
npages = rc;
rc = -EFAULT;
goto put_framevec;
goto put_pages;
}
userptr->npages = npages;
rc = sg_alloc_table_from_pages(userptr->sgt,
frame_vector_pages(userptr->vec),
npages, offset, size, GFP_ATOMIC);
userptr->pages,
npages, offset, size, GFP_ATOMIC);
if (rc < 0) {
dev_err(hdev->dev, "failed to create SG table from pages\n");
goto put_framevec;
goto put_pages;
}
return 0;
put_framevec:
put_vaddr_frames(userptr->vec);
destroy_framevec:
frame_vector_destroy(userptr->vec);
put_pages:
unpin_user_pages(userptr->pages, npages);
destroy_pages:
kvfree(userptr->pages);
return rc;
}
......@@ -1676,8 +1671,6 @@ int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size,
*/
void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr)
{
struct page **pages;
hl_debugfs_remove_userptr(hdev, userptr);
if (userptr->dma_mapped)
......@@ -1685,15 +1678,8 @@ void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr)
userptr->sgt->nents,
userptr->dir);
pages = frame_vector_pages(userptr->vec);
if (!IS_ERR(pages)) {
int i;
for (i = 0; i < frame_vector_count(userptr->vec); i++)
set_page_dirty_lock(pages[i]);
}
put_vaddr_frames(userptr->vec);
frame_vector_destroy(userptr->vec);
unpin_user_pages_dirty_lock(userptr->pages, userptr->npages, true);
kvfree(userptr->pages);
list_del(&userptr->job_node);
......
......@@ -274,6 +274,11 @@ static int proc_bus_pci_mmap(struct file *file, struct vm_area_struct *vma)
else
return -EINVAL;
}
if (dev->resource[i].flags & IORESOURCE_MEM &&
iomem_is_exclusive(dev->resource[i].start))
return -EINVAL;
ret = pci_mmap_page_range(dev, i, vma,
fpriv->mmap_state, write_combine);
if (ret < 0)
......
......@@ -170,6 +170,16 @@ static int sysfs_kf_bin_mmap(struct kernfs_open_file *of,
return battr->mmap(of->file, kobj, battr, vma);
}
static int sysfs_kf_bin_open(struct kernfs_open_file *of)
{
struct bin_attribute *battr = of->kn->priv;
if (battr->mapping)
of->file->f_mapping = battr->mapping;
return 0;
}
void sysfs_notify(struct kobject *kobj, const char *dir, const char *attr)
{
struct kernfs_node *kn = kobj->sd, *tmp;
......@@ -241,6 +251,7 @@ static const struct kernfs_ops sysfs_bin_kfops_mmap = {
.read = sysfs_kf_bin_read,
.write = sysfs_kf_bin_write,
.mmap = sysfs_kf_bin_mmap,
.open = sysfs_kf_bin_open,
};
int sysfs_add_file_mode_ns(struct kernfs_node *parent,
......
......@@ -334,11 +334,7 @@ static inline void irqresource_disabled(struct resource *res, u32 irq)
res->flags = IORESOURCE_IRQ | IORESOURCE_DISABLED | IORESOURCE_UNSET;
}
#ifdef CONFIG_IO_STRICT_DEVMEM
void revoke_devmem(struct resource *res);
#else
static inline void revoke_devmem(struct resource *res) { };
#endif
extern struct address_space *iomem_get_mapping(void);
#endif /* __ASSEMBLY__ */
#endif /* _LINUX_IOPORT_H */
......@@ -593,7 +593,8 @@ struct vm_operations_struct {
vm_fault_t (*pfn_mkwrite)(struct vm_fault *vmf);
/* called by access_process_vm when get_user_pages() fails, typically
* for use by special VMAs that can switch between memory and hardware
* for use by special VMAs. See also generic_access_phys() for a generic
* implementation useful for any iomem mapping.
*/
int (*access)(struct vm_area_struct *vma, unsigned long addr,
void *buf, int len, int write);
......@@ -1761,48 +1762,6 @@ int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc);
int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
struct task_struct *task, bool bypass_rlim);
/* Container for pinned pfns / pages */
struct frame_vector {
unsigned int nr_allocated; /* Number of frames we have space for */
unsigned int nr_frames; /* Number of frames stored in ptrs array */
bool got_ref; /* Did we pin pages by getting page ref? */
bool is_pfns; /* Does array contain pages or pfns? */
void *ptrs[]; /* Array of pinned pfns / pages. Use
* pfns_vector_pages() or pfns_vector_pfns()
* for access */
};
struct frame_vector *frame_vector_create(unsigned int nr_frames);
void frame_vector_destroy(struct frame_vector *vec);
int get_vaddr_frames(unsigned long start, unsigned int nr_pfns,
unsigned int gup_flags, struct frame_vector *vec);
void put_vaddr_frames(struct frame_vector *vec);
int frame_vector_to_pages(struct frame_vector *vec);
void frame_vector_to_pfns(struct frame_vector *vec);
static inline unsigned int frame_vector_count(struct frame_vector *vec)
{
return vec->nr_frames;
}
static inline struct page **frame_vector_pages(struct frame_vector *vec)
{
if (vec->is_pfns) {
int err = frame_vector_to_pages(vec);
if (err)
return ERR_PTR(err);
}
return (struct page **)(vec->ptrs);
}
static inline unsigned long *frame_vector_pfns(struct frame_vector *vec)
{
if (!vec->is_pfns)
frame_vector_to_pfns(vec);
return (unsigned long *)(vec->ptrs);
}
struct kvec;
int get_kernel_pages(const struct kvec *iov, int nr_pages, int write,
struct page **pages);
......
......@@ -164,11 +164,13 @@ __ATTRIBUTE_GROUPS(_name)
struct file;
struct vm_area_struct;
struct address_space;
struct bin_attribute {
struct attribute attr;
size_t size;
void *private;
struct address_space *mapping;
ssize_t (*read)(struct file *, struct kobject *, struct bin_attribute *,