NED
/
tegrakernel
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
tegrakernel/kernel/kernel-4.9/arch/arm64/mm/dma-mapping.c

2937 lines
77 KiB
C
Raw Blame History

/*
* SWIOTLB-based DMA API implementation
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Copyright (C) 2012 ARM Ltd.
* Author: Catalin Marinas <catalin.marinas@arm.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define pr_fmt(fmt) "%s():%d: " fmt, __func__, __LINE__
#include <linux/gfp.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/genalloc.h>
#include <linux/dma-mapping.h>
#include <linux/dma-contiguous.h>
#include <linux/vmalloc.h>
#include <linux/swiotlb.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/errno.h>
#include <linux/list.h>
#include <linux/init.h>
#include <linux/device.h>
#include <linux/highmem.h>
#include <linux/memblock.h>
#include <linux/iommu.h>
#include <linux/io.h>
#include <linux/seq_file.h>
#include <linux/debugfs.h>
#include <linux/cma.h>
#include <linux/dma-attrs.h>
#include <asm/cacheflush.h>
#include <asm/memory.h>
#include <asm/tlbflush.h>
#include <asm/dma-iommu.h>
#include <linux/dma-iommu.h>
#include <asm/dma-contiguous.h>
#include "mm.h"
#define CREATE_TRACE_POINTS
#include <trace/events/dmadebug.h>
#define ENABLE_IOMMU_DMA_OPS 0
#define ENABLE_IOMMU_DMA_OPS_NOTIFIER 0
#define ENABLE_IOMMU_SETUP_DMA_OPS 0
#ifndef CONFIG_DMA_API_DEBUG
char *__weak debug_dma_platformdata(struct device *dev)
{
/* empty string by default */
static char buf[1];
return buf;
}
#endif
static int dma_get_ioprot(unsigned long attrs,
enum dma_data_direction dir, bool coherent);
struct iommu_dma_cookie {
struct iova_domain iovad;
struct list_head msi_page_list;
spinlock_t msi_lock;
};
inline struct iova_domain *cookie_iovad(struct iommu_domain *domain)
{
return &((struct iommu_dma_cookie *)domain->iova_cookie)->iovad;
}
dma_addr_t __iommu_dma_alloc_iova(struct iommu_domain *domain, size_t size,
dma_addr_t dma_limit, bool size_aligned);
void __iommu_dma_free_iova(struct iova_domain *iovad,
dma_addr_t iova, size_t size);
static void arm__free_iova(struct device *dev,
dma_addr_t addr, size_t size)
{
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
struct iommu_domain *domain = mapping->domain;
struct iova_domain *iovad = cookie_iovad(domain);
__iommu_dma_free_iova(iovad, addr, size);
}
dma_addr_t arm__alloc_iova_at(struct device *dev, dma_addr_t dma_handle,
size_t size)
{
dma_addr_t dma_addr;
struct iommu_domain *domain;
struct iova_domain *iovad;
size_t len = PAGE_ALIGN(size);
dma_addr_t limit_addr;
domain = iommu_get_domain_for_dev(dev);
if (!domain) {
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
domain = mapping->domain;
if (!domain)
return DMA_ERROR_CODE;
}
iovad = domain->iova_cookie;
if (!iovad)
return DMA_ERROR_CODE;
/* limit addr is inclusive. */
limit_addr = dma_handle + iova_align(iovad, size) -
iovad->granule;
dma_addr = __iommu_dma_alloc_iova(domain, len, limit_addr, false);
if (!dma_addr)
return DMA_ERROR_CODE;
if (dma_addr != dma_handle) {
pr_err("iova alloc don't match, dh=%pad, da=%pad\n",
&dma_handle, &dma_addr);
__iommu_dma_free_iova(iovad, dma_addr, len);
return DMA_ERROR_CODE;
}
return dma_addr;
}
struct dma_map_ops arm_dma_ops;
enum dma_operation {
ALLOC_OR_FREE = 1,
ATOMIC_ALLOC_OR_FREE,
MAP_OR_UNMAP,
CPU_MAP_OR_UNMAP
};
#ifdef CONFIG_DMA_API_DEBUG
#define NULL_DEV "null_dev"
struct iommu_usage {
struct device *dev;
struct list_head recordlist;
};
struct null_device {
char const *dev_name;
atomic64_t map_size;
atomic64_t atomic_alloc_size;
atomic64_t alloc_size;
atomic64_t cpu_map_size;
};
static struct null_device *dev_is_null;
static LIST_HEAD(iommu_rlist_head);
static size_t dmastats_alloc_or_map(struct device *dev, size_t size,
const int type);
static size_t dmastats_free_or_unmap(struct device *dev, size_t size,
const int type);
static void add_value(struct dma_iommu_mapping *iu, size_t size,
const int type);
static void sub_value(struct dma_iommu_mapping *device_ref, size_t size,
const int type);
#else
#define dmastats_alloc_or_map(dev, size, type)
#define dmastats_free_or_unmap(dev, size, type)
#endif
static struct page **__alloc_buffer_pages(struct device *dev, size_t size,
gfp_t gfp, unsigned long attrs);
static int __free_buffer_pages(struct device *dev, struct page **pages,
size_t size, unsigned long attrs);
static struct page **__get_pages(void *cpu_addr, unsigned long attrs);
struct dma_map_ops *dma_ops;
EXPORT_SYMBOL(dma_ops);
static pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot,
bool coherent)
{
if (!coherent || dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs))
return pgprot_writecombine(prot);
return prot;
}
static struct gen_pool *atomic_pool;
static struct page **atomic_pool_pages;
static size_t atomic_pool_size = SZ_1M;
static int __init early_coherent_pool(char *p)
{
atomic_pool_size = memparse(p, &p);
return 0;
}
early_param("coherent_pool", early_coherent_pool);
static void *__alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags)
{
unsigned long val;
void *ptr = NULL;
if (!atomic_pool) {
WARN(1, "coherent pool not initialised!\n");
return NULL;
}
val = gen_pool_alloc(atomic_pool, size);
if (val) {
phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
*ret_page = phys_to_page(phys);
ptr = (void *)val;
memset(ptr, 0, size);
}
return ptr;
}
static bool __in_atomic_pool(void *start, size_t size)
{
if (!atomic_pool)
return false;
return addr_in_gen_pool(atomic_pool, (unsigned long)start, size);
}
static int __free_from_pool(void *start, size_t size)
{
if (!__in_atomic_pool(start, size))
return 0;
gen_pool_free(atomic_pool, (unsigned long)start, size);
return 1;
}
#ifdef CONFIG_DMA_API_DEBUG
static void *___alloc_from_pool(struct device *dev, size_t size,
struct page **ret_page, gfp_t flags)
{
struct dma_iommu_mapping *mapping;
void *ptr = __alloc_from_pool(size, ret_page, flags);
mapping = to_dma_iommu_mapping(dev);
if (ptr && mapping)
dmastats_alloc_or_map(dev, size, ATOMIC_ALLOC_OR_FREE);
return ptr;
}
static int ___free_from_pool(struct device *dev, void *start, size_t size)
{
int ret = __free_from_pool(start, size);
if (ret)
dmastats_free_or_unmap(dev, size, ATOMIC_ALLOC_OR_FREE);
return ret;
}
#define __free_from_pool(start, size) \
___free_from_pool(dev, start, size)
#define __alloc_from_pool(size, ret_page, flags) \
___alloc_from_pool(dev, size, ret_page, flags)
#endif
#ifdef CONFIG_SWIOTLB
static void *__dma_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
unsigned long attrs)
{
if (dev == NULL) {
WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
return NULL;
}
if (IS_ENABLED(CONFIG_ZONE_DMA) &&
dev->coherent_dma_mask <= DMA_BIT_MASK(32))
flags |= GFP_DMA;
if (IS_ENABLED(CONFIG_DMA_CMA) && (gfpflags_allow_blocking(flags))) {
struct page *page;
void *addr;
size = PAGE_ALIGN(size);
page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
get_order(size));
if (!page)
return NULL;
*dma_handle = phys_to_dma(dev, page_to_phys(page));
addr = page_address(page);
memset(addr, 0, size);
return addr;
}
return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
}
static void __dma_free_coherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle,
unsigned long attrs)
{
bool freed;
phys_addr_t paddr = dma_to_phys(dev, dma_handle);
if (dev == NULL) {
WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
return;
}
freed = dma_release_from_contiguous(dev,
phys_to_page(paddr),
size >> PAGE_SHIFT);
if (!freed)
swiotlb_free_coherent(dev, size, vaddr, dma_handle);
}
static void *__dma_alloc_noncoherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
unsigned long attrs)
{
struct page *page;
void *ptr, *coherent_ptr;
size = PAGE_ALIGN(size);
if (!gfpflags_allow_blocking(flags)) {
struct page *page = NULL;
void *addr = __alloc_from_pool(size, &page, flags);
if (addr)
*dma_handle = phys_to_dma(dev, page_to_phys(page));
return addr;
}
ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs);
if (!ptr)
goto no_mem;
/* remove any dirty cache lines on the kernel alias */
__dma_flush_area(ptr, size);
/* create a coherent mapping */
page = virt_to_page(ptr);
coherent_ptr = dma_common_contiguous_remap(page, size, VM_USERMAP,
__get_dma_pgprot(attrs,
__pgprot(PROT_NORMAL_NC), false),
NULL);
if (!coherent_ptr)
goto no_map;
return coherent_ptr;
no_map:
__dma_free_coherent(dev, size, ptr, *dma_handle, attrs);
no_mem:
*dma_handle = DMA_ERROR_CODE;
return NULL;
}
static void __dma_free_noncoherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle,
unsigned long attrs)
{
void *swiotlb_addr = phys_to_virt(dma_to_phys(dev, dma_handle));
if (__free_from_pool(vaddr, size))
return;
vunmap(vaddr);
__dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs);
}
static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size,
enum dma_data_direction dir,
unsigned long attrs)
{
dma_addr_t dev_addr;
dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs);
__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
return dev_addr;
}
static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr,
size_t size, enum dma_data_direction dir,
unsigned long attrs)
{
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
swiotlb_unmap_page(dev, dev_addr, size, dir, attrs);
}
static int __swiotlb_map_sg_attrs(struct device *dev, struct scatterlist *sgl,
int nelems, enum dma_data_direction dir,
unsigned long attrs)
{
struct scatterlist *sg;
int i, ret;
ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs);
for_each_sg(sgl, sg, ret, i)
__dma_map_area_no_dsb(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
dsb(sy);
return ret;
}
static void __swiotlb_unmap_sg_attrs(struct device *dev,
struct scatterlist *sgl, int nelems,
enum dma_data_direction dir,
unsigned long attrs)
{
struct scatterlist *sg;
int i;
for_each_sg(sgl, sg, nelems, i)
__dma_unmap_area_no_dsb(
phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
dsb(sy);
swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs);
}
static void __swiotlb_sync_single_for_cpu(struct device *dev,
dma_addr_t dev_addr, size_t size,
enum dma_data_direction dir)
{
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
swiotlb_sync_single_for_cpu(dev, dev_addr, size, dir);
}
static void __swiotlb_sync_single_for_device(struct device *dev,
dma_addr_t dev_addr, size_t size,
enum dma_data_direction dir)
{
swiotlb_sync_single_for_device(dev, dev_addr, size, dir);
__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
}
static void __swiotlb_sync_sg_for_cpu(struct device *dev,
struct scatterlist *sgl, int nelems,
enum dma_data_direction dir)
{
struct scatterlist *sg;
int i;
for_each_sg(sgl, sg, nelems, i)
__dma_unmap_area_no_dsb(
phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
dsb(sy);
swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir);
}
static void __swiotlb_sync_sg_for_device(struct device *dev,
struct scatterlist *sgl, int nelems,
enum dma_data_direction dir)
{
struct scatterlist *sg;
int i;
swiotlb_sync_sg_for_device(dev, sgl, nelems, dir);
for_each_sg(sgl, sg, nelems, i)
__dma_map_area_no_dsb(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
dsb(sy);
}
/* vma->vm_page_prot must be set appropriately before calling this function */
static int __dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t dma_addr, size_t size)
{
int ret = -ENXIO;
unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >>
PAGE_SHIFT;
unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
unsigned long pfn = dma_to_phys(dev, dma_addr) >> PAGE_SHIFT;
unsigned long off = vma->vm_pgoff;
if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
return ret;
if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
ret = remap_pfn_range(vma, vma->vm_start,
pfn + off,
vma->vm_end - vma->vm_start,
vma->vm_page_prot);
}
return ret;
}
static int __swiotlb_mmap_noncoherent(struct device *dev,
struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t dma_addr, size_t size,
unsigned long attrs)
{
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, false);
return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
}
static int __swiotlb_mmap_coherent(struct device *dev,
struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t dma_addr, size_t size,
unsigned long attrs)
{
/* Just use whatever page_prot attributes were specified */
return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
}
struct dma_map_ops noncoherent_swiotlb_dma_ops = {
.alloc = __dma_alloc_noncoherent,
.free = __dma_free_noncoherent,
.mmap = __swiotlb_mmap_noncoherent,
.map_page = __swiotlb_map_page,
.unmap_page = __swiotlb_unmap_page,
.map_sg = __swiotlb_map_sg_attrs,
.unmap_sg = __swiotlb_unmap_sg_attrs,
.sync_single_for_cpu = __swiotlb_sync_single_for_cpu,
.sync_single_for_device = __swiotlb_sync_single_for_device,
.sync_sg_for_cpu = __swiotlb_sync_sg_for_cpu,
.sync_sg_for_device = __swiotlb_sync_sg_for_device,
.dma_supported = swiotlb_dma_supported,
.mapping_error = swiotlb_dma_mapping_error,
};
EXPORT_SYMBOL(noncoherent_swiotlb_dma_ops);
struct dma_map_ops coherent_swiotlb_dma_ops = {
.alloc = __dma_alloc_coherent,
.free = __dma_free_coherent,
.mmap = __swiotlb_mmap_coherent,
.map_page = swiotlb_map_page,
.unmap_page = swiotlb_unmap_page,
.map_sg = swiotlb_map_sg_attrs,
.unmap_sg = swiotlb_unmap_sg_attrs,
.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
.sync_single_for_device = swiotlb_sync_single_for_device,
.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
.sync_sg_for_device = swiotlb_sync_sg_for_device,
.dma_supported = swiotlb_dma_supported,
.mapping_error = swiotlb_dma_mapping_error,
};
EXPORT_SYMBOL(coherent_swiotlb_dma_ops);
extern int swiotlb_late_init_with_default_size(size_t default_size);
#endif /* CONFIG_SWIOTLB */
static int __init atomic_pool_init(void)
{
pgprot_t prot = __pgprot(PROT_NORMAL_NC);
unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT;
struct page *page;
struct page **pages = NULL;
void *addr;
unsigned int pool_size_order = get_order(atomic_pool_size);
if (dev_get_cma_area(NULL))
page = dma_alloc_from_contiguous(NULL, nr_pages,
pool_size_order);
else
page = alloc_pages(GFP_DMA, pool_size_order);
if (page) {
int ret, i = 0;
void *page_addr = page_address(page);
memset(page_addr, 0, atomic_pool_size);
__dma_flush_area(page_addr, atomic_pool_size);
atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
if (!atomic_pool)
goto free_page;
pages = kmalloc(sizeof(struct page *) << pool_size_order,
GFP_KERNEL);
if (!pages)
goto free_page;
for (; i < nr_pages; i++)
pages[i] = page + i;
addr = dma_common_pages_remap(pages, atomic_pool_size,
VM_USERMAP,
prot, atomic_pool_init);
if (!addr)
goto destroy_genpool;
ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr,
page_to_phys(page),
atomic_pool_size, -1);
if (ret)
goto remove_mapping;
atomic_pool_pages = pages;
gen_pool_set_algo(atomic_pool,
gen_pool_first_fit_order_align,
(void *)PAGE_SHIFT);
pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",
atomic_pool_size / 1024);
return 0;
}
goto out;
remove_mapping:
dma_common_free_remap(addr, atomic_pool_size,
VM_USERMAP);
destroy_genpool:
gen_pool_destroy(atomic_pool);
atomic_pool = NULL;
kfree(pages);
free_page:
if (!dma_release_from_contiguous(NULL, page, nr_pages))
__free_pages(page, pool_size_order);
out:
pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
atomic_pool_size / 1024);
return -ENOMEM;
}
/********************************************
* The following APIs are for dummy DMA ops *
********************************************/
static void *__dummy_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
unsigned long attrs)
{
return NULL;
}
static void __dummy_free(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle,
unsigned long attrs)
{
}
static int __dummy_mmap(struct device *dev,
struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t dma_addr, size_t size,
unsigned long attrs)
{
return -ENXIO;
}
static dma_addr_t __dummy_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size,
enum dma_data_direction dir,
unsigned long attrs)
{
return DMA_ERROR_CODE;
}
static void __dummy_unmap_page(struct device *dev, dma_addr_t dev_addr,
size_t size, enum dma_data_direction dir,
unsigned long attrs)
{
}
static int __dummy_map_sg(struct device *dev, struct scatterlist *sgl,
int nelems, enum dma_data_direction dir,
unsigned long attrs)
{
return 0;
}
static void __dummy_unmap_sg(struct device *dev,
struct scatterlist *sgl, int nelems,
enum dma_data_direction dir,
unsigned long attrs)
{
}
static void __dummy_sync_single(struct device *dev,
dma_addr_t dev_addr, size_t size,
enum dma_data_direction dir)
{
}
static void __dummy_sync_sg(struct device *dev,
struct scatterlist *sgl, int nelems,
enum dma_data_direction dir)
{
}
static int __dummy_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
{
return 1;
}
static int __dummy_dma_supported(struct device *hwdev, u64 mask)
{
return 0;
}
struct dma_map_ops dummy_dma_ops = {
.alloc = __dummy_alloc,
.free = __dummy_free,
.mmap = __dummy_mmap,
.map_page = __dummy_map_page,
.unmap_page = __dummy_unmap_page,
.map_sg = __dummy_map_sg,
.unmap_sg = __dummy_unmap_sg,
.sync_single_for_cpu = __dummy_sync_single,
.sync_single_for_device = __dummy_sync_single,
.sync_sg_for_cpu = __dummy_sync_sg,
.sync_sg_for_device = __dummy_sync_sg,
.mapping_error = __dummy_mapping_error,
.dma_supported = __dummy_dma_supported,
};
EXPORT_SYMBOL(dummy_dma_ops);
#ifdef CONFIG_SWIOTLB
static int __init swiotlb_late_init(void)
{
size_t swiotlb_size = min(SZ_64M, MAX_ORDER_NR_PAGES << PAGE_SHIFT);
dma_ops = &noncoherent_swiotlb_dma_ops;
return swiotlb_late_init_with_default_size(swiotlb_size);
}
#endif /* CONFIG_SWIOTLB */
static int __init arm64_dma_init(void)
{
int ret = 0;
dma_ops = &arm_dma_ops;
#ifdef CONFIG_SWIOTLB
ret |= swiotlb_late_init();
#endif /* CONFIG_SWIOTLB */
ret |= atomic_pool_init();
return ret;
}
arch_initcall(arm64_dma_init);
#ifdef CONFIG_IOMMU_DMA
#include <linux/dma-iommu.h>
#include <linux/platform_device.h>
#include <linux/amba/bus.h>
#define UL_ATR(a) (a)
static void flush_sg(struct device *dev, struct sg_table *sgt)
{
struct scatterlist *s;
int i = 0;
int nents = sgt->orig_nents;
for_each_sg(sgt->sgl, s, nents, i) {
__dma_flush_area(sg_virt(s), s->length);
}
}
static void *__iommu_alloc_attrs(struct device *dev, size_t size,
dma_addr_t *handle, gfp_t gfp,
unsigned long attrs)
{
bool coherent = is_device_dma_coherent(dev);
int ioprot = dma_get_ioprot(attrs, DMA_BIDIRECTIONAL, coherent);
size_t iosize = size;
void *addr;
/* Following is a work-around (a.k.a. hack) to prevent pages
* with __GFP_COMP being passed to split_page() which cannot
* handle them. The real problem is that this flag probably
* should be 0 on ARM as it is not supported on this
* platform--see CONFIG_HUGETLB_PAGE.
*/
gfp &= ~(__GFP_COMP);
if (WARN(!dev, "cannot create IOMMU mapping for unknown device\n"))
return NULL;
size = PAGE_ALIGN(size);
if (gfpflags_allow_blocking(gfp)) {
struct page **pages;
pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, coherent);
pages = iommu_dma_alloc(dev, iosize, gfp, UL_ATR(attrs),
ioprot, handle, flush_sg);
if (!pages)
return NULL;
trace_dmadebug_alloc_attrs(dev, *handle, size, pages[0]);
if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
return pages;
addr = dma_common_pages_remap(pages, size, VM_USERMAP, prot,
__builtin_return_address(0));
if (!addr)
iommu_dma_free(dev, pages, iosize, handle, attrs);
} else {
struct page *page;
/*
* In atomic context we can't remap anything, so we'll only
* get the virtually contiguous buffer we need by way of a
* physically contiguous allocation.
*/
if (coherent) {
page = alloc_pages(gfp, get_order(size));
addr = page ? page_address(page) : NULL;
} else {
addr = __alloc_from_pool(size, &page, gfp);
}
if (!addr)
return NULL;
*handle = iommu_dma_map_page(dev, page, 0, iosize, ioprot);
if (iommu_dma_mapping_error(dev, *handle)) {
if (coherent)
__free_pages(page, get_order(size));
else
__free_from_pool(addr, size);
addr = NULL;
} else
trace_dmadebug_alloc_attrs(dev, *handle, size, page);
}
return addr;
}
static void __iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
dma_addr_t handle, unsigned long attrs)
{
size_t iosize = size;
size = PAGE_ALIGN(size);
trace_dmadebug_free_attrs(dev, handle, size, NULL);
/*
* @cpu_addr will be one of 3 things depending on how it was allocated:
* - A remapped array of pages from iommu_dma_alloc(), for all
* non-atomic allocations.
* - A non-cacheable alias from the atomic pool, for atomic
* allocations by non-coherent devices.
* - A normal lowmem address, for atomic allocations by
* coherent devices.
* Hence how dodgy the below logic looks...
*/
if (__in_atomic_pool(cpu_addr, size)) {
iommu_dma_unmap_page(dev, handle, iosize, 0, 0);
__free_from_pool(cpu_addr, size);
} else if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) {
iommu_dma_free(dev, (struct page **)cpu_addr, iosize,
&handle, attrs);
} else if (is_vmalloc_addr(cpu_addr)){
struct vm_struct *area = find_vm_area(cpu_addr);
if (WARN_ON(!area || !area->pages))
return;
iommu_dma_free(dev, area->pages, iosize, &handle, attrs);
dma_common_free_remap(cpu_addr, size, VM_USERMAP);
} else {
iommu_dma_unmap_page(dev, handle, iosize, 0, 0);
__free_pages(virt_to_page(cpu_addr), get_order(size));
}
}
static int __iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t dma_addr, size_t size,
unsigned long attrs)
{
struct vm_struct *area;
int ret;
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
is_device_dma_coherent(dev));
if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
return ret;
area = find_vm_area(cpu_addr);
if (WARN_ON(!area || !area->pages))
return -ENXIO;
return iommu_dma_mmap(area->pages, size, vma);
}
static int __iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
void *cpu_addr, dma_addr_t dma_addr,
size_t size, unsigned long attrs)
{
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
struct vm_struct *area = find_vm_area(cpu_addr);
if (WARN_ON(!area || !area->pages))
return -ENXIO;
return sg_alloc_table_from_pages(sgt, area->pages, count, 0, size,
GFP_KERNEL);
}
static void __iommu_sync_single_for_cpu(struct device *dev,
dma_addr_t dev_addr, size_t size,
enum dma_data_direction dir)
{
phys_addr_t phys;
if (is_device_dma_coherent(dev))
return;
phys = iommu_iova_to_phys(iommu_get_domain_for_dev(dev), dev_addr);
__dma_unmap_area(phys_to_virt(phys), size, dir);
}
static void __iommu_sync_single_for_device(struct device *dev,
dma_addr_t dev_addr, size_t size,
enum dma_data_direction dir)
{
phys_addr_t phys;
if (is_device_dma_coherent(dev))
return;
phys = iommu_iova_to_phys(iommu_get_domain_for_dev(dev), dev_addr);
__dma_map_area(phys_to_virt(phys), size, dir);
}
static dma_addr_t __iommu_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size,
enum dma_data_direction dir,
unsigned long attrs)
{
bool coherent = is_device_dma_coherent(dev);
int prot = dma_get_ioprot(attrs, dir, coherent);
dma_addr_t dev_addr = iommu_dma_map_page(dev, page, offset, size, prot);
if (!iommu_dma_mapping_error(dev, dev_addr) &&
(UL_ATR(attrs) & DMA_ATTR_SKIP_CPU_SYNC) == 0)
__iommu_sync_single_for_device(dev, dev_addr, size, dir);
trace_dmadebug_map_page(dev, dev_addr + offset, size, page);
return dev_addr;
}
static dma_addr_t arm_iommu_map_at(struct device *dev, dma_addr_t dma_addr,
phys_addr_t phys, size_t size,
enum dma_data_direction dir,
unsigned long attrs);
#if ENABLE_IOMMU_DMA_OPS
static dma_addr_t __iommu_map_at(struct device *dev, dma_addr_t dma_addr,
phys_addr_t phys, size_t size,
enum dma_data_direction dir,
unsigned long attrs)
{
bool coherent = is_device_dma_coherent(dev);
int prot = dma_get_ioprot(attrs, dir, coherent);
return iommu_dma_map_at(dev, dma_addr, phys, size, prot);
}
#endif
static void __iommu_unmap_page(struct device *dev, dma_addr_t dev_addr,
size_t size, enum dma_data_direction dir,
unsigned long attrs)
{
if ((UL_ATR(attrs) & DMA_ATTR_SKIP_CPU_SYNC) == 0)
__iommu_sync_single_for_cpu(dev, dev_addr, size, dir);
trace_dmadebug_unmap_page(dev, dev_addr, size,
phys_to_page(iommu_iova_to_phys(iommu_get_domain_for_dev(dev),
dev_addr)));
iommu_dma_unmap_page(dev, dev_addr, size, dir, UL_ATR(attrs));
}
static void __iommu_sync_sg_for_cpu(struct device *dev,
struct scatterlist *sgl, int nelems,
enum dma_data_direction dir)
{
struct scatterlist *sg;
int i;
if (is_device_dma_coherent(dev))
return;
for_each_sg(sgl, sg, nelems, i)
__dma_unmap_area_no_dsb(sg_virt(sg), sg->length, dir);
dsb(sy);
}
static void __iommu_sync_sg_for_device(struct device *dev,
struct scatterlist *sgl, int nelems,
enum dma_data_direction dir)
{
struct scatterlist *sg;
int i;
if (is_device_dma_coherent(dev))
return;
for_each_sg(sgl, sg, nelems, i)
__dma_map_area_no_dsb(sg_virt(sg), sg->length, dir);
dsb(sy);
}
static int __iommu_map_sg_attrs(struct device *dev, struct scatterlist *sgl,
int nelems, enum dma_data_direction dir,
unsigned long attrs)
{
bool coherent = is_device_dma_coherent(dev);
if ((UL_ATR(attrs) & DMA_ATTR_SKIP_CPU_SYNC) == 0)
__iommu_sync_sg_for_device(dev, sgl, nelems, dir);
return iommu_dma_map_sg(dev, sgl, nelems,
dma_get_ioprot(attrs, dir, coherent));
}
static void __iommu_unmap_sg_attrs(struct device *dev,
struct scatterlist *sgl, int nelems,
enum dma_data_direction dir,
unsigned long attrs)
{
if ((UL_ATR(attrs) & DMA_ATTR_SKIP_CPU_SYNC) == 0)
__iommu_sync_sg_for_cpu(dev, sgl, nelems, dir);
iommu_dma_unmap_sg(dev, sgl, nelems, dir, UL_ATR(attrs));
}
static struct dma_map_ops iommu_dma_ops = {
.alloc = __iommu_alloc_attrs,
.free = __iommu_free_attrs,
.mmap = __iommu_mmap_attrs,
.get_sgtable = __iommu_get_sgtable,
.map_page = __iommu_map_page,
.unmap_page = __iommu_unmap_page,
.map_sg = __iommu_map_sg_attrs,
.unmap_sg = __iommu_unmap_sg_attrs,
.sync_single_for_cpu = __iommu_sync_single_for_cpu,
.sync_single_for_device = __iommu_sync_single_for_device,
.sync_sg_for_cpu = __iommu_sync_sg_for_cpu,
.sync_sg_for_device = __iommu_sync_sg_for_device,
.dma_supported = iommu_dma_supported,
.mapping_error = iommu_dma_mapping_error,
.map_at = arm_iommu_map_at,
};
#if ENABLE_IOMMU_DMA_OPS_NOTIFIER
/*
* TODO: Right now __iommu_setup_dma_ops() gets called too early to do
* everything it needs to - the device is only partially created and the
* IOMMU driver hasn't seen it yet, so it can't have a group. Thus we
* need this delayed attachment dance. Once IOMMU probe ordering is sorted
* to move the arch_setup_dma_ops() call later, all the notifier bits below
* become unnecessary, and will go away.
*/
struct iommu_dma_notifier_data {
struct list_head list;
struct device *dev;
const struct iommu_ops *ops;
u64 dma_base;
u64 size;
};
static LIST_HEAD(iommu_dma_masters);
static DEFINE_MUTEX(iommu_dma_notifier_lock);
/*
* Temporarily "borrow" a domain feature flag to to tell if we had to resort
* to creating our own domain here, in case we need to clean it up again.
*/
#define __IOMMU_DOMAIN_FAKE_DEFAULT (1U << 31)
static bool do_iommu_attach(struct device *dev, const struct iommu_ops *ops,
u64 dma_base, u64 size)
{
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
/*
* Best case: The device is either part of a group which was
* already attached to a domain in a previous call, or it's
* been put in a default DMA domain by the IOMMU core.
*/
if (!domain) {
/*
* Urgh. The IOMMU core isn't going to do default domains
* for non-PCI devices anyway, until it has some means of
* abstracting the entirely implementation-specific
* sideband data/SoC topology/unicorn dust that may or
* may not differentiate upstream masters.
* So until then, HORRIBLE HACKS!
*/
domain = ops->domain_alloc(IOMMU_DOMAIN_DMA);
if (!domain)
goto out_no_domain;
domain->ops = ops;
domain->type = IOMMU_DOMAIN_DMA | __IOMMU_DOMAIN_FAKE_DEFAULT;
if (iommu_attach_device(domain, dev))
goto out_put_domain;
}
if (iommu_dma_init_domain(domain, dma_base, size, dev))
goto out_detach;
dev->archdata.dma_ops = &iommu_dma_ops;
return true;
out_detach:
iommu_detach_device(domain, dev);
out_put_domain:
if (domain->type & __IOMMU_DOMAIN_FAKE_DEFAULT)
iommu_domain_free(domain);
out_no_domain:
pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n",
dev_name(dev));
return false;
}
static void queue_iommu_attach(struct device *dev, const struct iommu_ops *ops,
u64 dma_base, u64 size)
{
struct iommu_dma_notifier_data *iommudata;
iommudata = kzalloc(sizeof(*iommudata), GFP_KERNEL);
if (!iommudata)
return;
iommudata->dev = dev;
iommudata->ops = ops;
iommudata->dma_base = dma_base;
iommudata->size = size;
mutex_lock(&iommu_dma_notifier_lock);
list_add(&iommudata->list, &iommu_dma_masters);
mutex_unlock(&iommu_dma_notifier_lock);
}
static int __iommu_attach_notifier(struct notifier_block *nb,
unsigned long action, void *data)
{
struct iommu_dma_notifier_data *master, *tmp;
if (action != BUS_NOTIFY_ADD_DEVICE)
return 0;
mutex_lock(&iommu_dma_notifier_lock);
list_for_each_entry_safe(master, tmp, &iommu_dma_masters, list) {
if (do_iommu_attach(master->dev, master->ops,
master->dma_base, master->size)) {
list_del(&master->list);
kfree(master);
}
}
mutex_unlock(&iommu_dma_notifier_lock);
return 0;
}
int register_iommu_dma_ops_notifier(struct bus_type *bus)
{
struct notifier_block *nb = kzalloc(sizeof(*nb), GFP_KERNEL);
int ret;
if (!nb)
return -ENOMEM;
/*
* The device must be attached to a domain before the driver probe
* routine gets a chance to start allocating DMA buffers. However,
* the IOMMU driver also needs a chance to configure the iommu_group
* via its add_device callback first, so we need to make the attach
* happen between those two points. Since the IOMMU core uses a bus
* notifier with default priority for add_device, do the same but
* with a lower priority to ensure the appropriate ordering.
*/
nb->notifier_call = __iommu_attach_notifier;
nb->priority = -100;
ret = bus_register_notifier(bus, nb);
if (ret) {
pr_warn("Failed to register DMA domain notifier; IOMMU DMA ops unavailable on bus '%s'\n",
bus->name);
kfree(nb);
}
return ret;
}
#endif /* ENABLE_IOMMU_DMA_OPS_NOTIFIER */
static int __init __iommu_dma_init(void)
{
int ret = 0;
ret = iommu_dma_init();
#if ENABLE_IOMMU_DMA_OPS_NOTIFIER
if (!ret)
ret = register_iommu_dma_ops_notifier(&platform_bus_type);
if (!ret)
ret = register_iommu_dma_ops_notifier(&amba_bustype);
#endif
return ret;
}
arch_initcall(__iommu_dma_init);
void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
const struct iommu_ops *ops)
{
#if ENABLE_IOMMU_DMA_OPS_NOTIFIER
struct iommu_group *group;
if (!ops)
return;
/*
* TODO: As a concession to the future, we're ready to handle being
* called both early and late (i.e. after bus_add_device). Once all
* the platform bus code is reworked to call us late and the notifier
* junk above goes away, move the body of do_iommu_attach here.
*/
group = iommu_group_get(dev);
if (group) {
do_iommu_attach(dev, ops, dma_base, size);
iommu_group_put(group);
} else {
queue_iommu_attach(dev, ops, dma_base, size);
}
#endif
}
#else
void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
struct iommu_ops *iommu)
{ }
#endif /* CONFIG_IOMMU_DMA */
struct dma_map_ops arm_coherent_dma_ops;
void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
const struct iommu_ops *iommu, bool coherent)
{
if (!dev->archdata.dma_ops) {
if (!coherent)
dev->archdata.dma_ops = &arm_dma_ops;
else
dev->archdata.dma_ops = &arm_coherent_dma_ops;
}
dev->archdata.dma_coherent = coherent;
dev->archdata.dma_noncontig =
of_property_read_bool(dev->of_node, "dma-noncontig");
#if ENABLE_IOMMU_SETUP_DMA_OPS
if (!dev->archdata.dma_ops)
dev->archdata.dma_ops = &swiotlb_dma_ops;
__iommu_setup_dma_ops(dev, dma_base, size, iommu);
#endif
}
#ifdef CONFIG_IOMMU_DMA
void arch_teardown_dma_ops(struct device *dev)
{
#if ENABLE_IOMMU_SETUP_DMA_OPS
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
if (domain) {
iommu_detach_device(domain, dev);
if (domain->type & __IOMMU_DOMAIN_FAKE_DEFAULT)
iommu_domain_free(domain);
}
dev->archdata.dma_ops = NULL;
#endif
}
#endif
/*
*FIXME: from arm
*/
/*
* The DMA API is built upon the notion of "buffer ownership". A buffer
* is either exclusively owned by the CPU (and therefore may be accessed
* by it) or exclusively owned by the DMA device. These helper functions
* represent the transitions between these two ownership states.
*
* Note, however, that on later ARMs, this notion does not work due to
* speculative prefetches. We model our approach on the assumption that
* the CPU does do speculative prefetches, which means we clean caches
* before transfers and delay cache invalidation until transfer completion.
*
*/
static void __dma_page_cpu_to_dev(struct page *, unsigned long,
size_t, enum dma_data_direction);
static void __dma_page_dev_to_cpu(struct page *, unsigned long,
size_t, enum dma_data_direction);
__weak void dma_qualify_ioprot(enum dma_data_direction dir,
unsigned long *ioprot) {}
__weak void dma_marshal_handle(enum dma_data_direction dir, dma_addr_t *handle) {}
__weak void dma_unmarshal_handle(enum dma_data_direction dir, dma_addr_t *handle) {}
/**
* arm_dma_map_page - map a portion of a page for streaming DMA
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
* @page: page that buffer resides in
* @offset: offset into page for start of buffer
* @size: size of buffer to map
* @dir: DMA transfer direction
*
* Ensure that any data held in the cache is appropriately discarded
* or written back.
*
* The device owns this memory once this call has completed. The CPU
* can regain ownership by calling dma_unmap_page().
*/
static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size, enum dma_data_direction dir,
unsigned long attrs)
{
dma_addr_t handle;
if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
__dma_page_cpu_to_dev(page, offset, size, dir & DMA_NONE);
handle = __pfn_to_phys(page_to_pfn(page)) + offset;
dma_marshal_handle(dir, &handle);
return handle;
}
static dma_addr_t arm_coherent_dma_map_page(struct device *dev,
struct page *page, unsigned long offset, size_t size,
enum dma_data_direction dir, unsigned long attrs)
{
dma_addr_t handle = __pfn_to_phys(page_to_pfn(page)) + offset;
dma_marshal_handle(dir, &handle);
return handle;
}
/**
* arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
* @handle: DMA address of buffer
* @size: size of buffer (same as passed to dma_map_page)
* @dir: DMA transfer direction (same as passed to dma_map_page)
*
* Unmap a page streaming mode DMA translation. The handle and size
* must match what was provided in the previous dma_map_page() call.
* All other usages are undefined.
*
* After this call, reads by the CPU to the buffer are guaranteed to see
* whatever the device wrote there.
*/
static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
size_t size, enum dma_data_direction dir,
unsigned long attrs)
{
dma_unmarshal_handle(dir, &handle);
if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
__dma_page_dev_to_cpu(pfn_to_page(__phys_to_pfn(handle)),
handle & ~PAGE_MASK, size, dir & DMA_NONE);
}
static void arm_dma_sync_single_for_cpu(struct device *dev,
dma_addr_t handle, size_t size, enum dma_data_direction dir)
{
unsigned int offset = handle & (PAGE_SIZE - 1);
struct page *page = pfn_to_page(__phys_to_pfn(handle-offset));
__dma_page_dev_to_cpu(page, offset, size, dir);
}
static void arm_dma_sync_single_for_device(struct device *dev,
dma_addr_t handle, size_t size, enum dma_data_direction dir)
{
unsigned int offset = handle & (PAGE_SIZE - 1);
struct page *page = pfn_to_page(__phys_to_pfn(handle-offset));
__dma_page_cpu_to_dev(page, offset, size, dir);
}
struct dma_map_ops arm_dma_ops = {
.alloc = arm_dma_alloc,
.free = arm_dma_free,
.mmap = arm_dma_mmap,
.get_sgtable = arm_dma_get_sgtable,
.map_page = arm_dma_map_page,
.unmap_page = arm_dma_unmap_page,
.map_sg = arm_dma_map_sg,
.unmap_sg = arm_dma_unmap_sg,
.sync_single_for_cpu = arm_dma_sync_single_for_cpu,
.sync_single_for_device = arm_dma_sync_single_for_device,
.sync_sg_for_cpu = arm_dma_sync_sg_for_cpu,
.sync_sg_for_device = arm_dma_sync_sg_for_device,
.set_dma_mask = arm_dma_set_mask,
.mapping_error = arm_dma_mapping_error,
.dma_supported = arm_dma_supported,
};
EXPORT_SYMBOL(arm_dma_ops);
static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
dma_addr_t *handle, gfp_t gfp, unsigned long attrs);
static void arm_coherent_dma_free(struct device *dev, size_t size,
void *cpu_addr, dma_addr_t handle, unsigned long attrs);
struct dma_map_ops arm_coherent_dma_ops = {
.alloc = arm_coherent_dma_alloc,
.free = arm_coherent_dma_free,
.mmap = arm_dma_mmap,
.get_sgtable = arm_dma_get_sgtable,
.map_page = arm_coherent_dma_map_page,
.map_sg = arm_dma_map_sg,
.set_dma_mask = arm_dma_set_mask,
.mapping_error = arm_dma_mapping_error,
.dma_supported = arm_dma_supported,
};
EXPORT_SYMBOL(arm_coherent_dma_ops);
static u64 get_coherent_dma_mask(struct device *dev)
{
u64 mask = (u64)arm_dma_limit;
if (dev) {
mask = dev->coherent_dma_mask;
/*
* Sanity check the DMA mask - it must be non-zero, and
* must be able to be satisfied by a DMA allocation.
*/
if (mask == 0) {
dev_dbg(dev, "coherent DMA mask is unset\n");
return 0;
}
if ((~mask) & (u64)arm_dma_limit) {
dev_warn(dev, "coherent DMA mask %#llx is smaller "
"than system GFP_DMA mask %#llx\n",
mask, (u64)arm_dma_limit);
return 0;
}
}
return mask;
}
static void __dma_clear_buffer(struct page *page, size_t size, bool coherent)
{
void *ptr;
/*
* Ensure that the allocated pages are zeroed, and that any data
* lurking in the kernel direct-mapped region is invalidated.
*/
ptr = page_address(page);
if (ptr) {
memset(ptr, 0, size);
if (!coherent)
__dma_flush_area(ptr, size);
}
}
/*
* Allocate a DMA buffer for 'dev' of size 'size' using the
* specified gfp mask. Note that 'size' must be page aligned.
*/
static struct page *__dma_alloc_buffer(struct device *dev,
size_t size, gfp_t gfp)
{
unsigned long order = get_order(size);
struct page *page, *p, *e;
bool coherent = is_device_dma_coherent(dev);
page = alloc_pages(gfp, order);
if (!page)
return NULL;
/*
* Now split the huge page and free the excess pages
*/
split_page(page, order);
for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order);
p < e; p++)
__free_page(p);
__dma_clear_buffer(page, size, coherent);
return page;
}
/*
* Free a DMA buffer. 'size' must be page aligned.
*/
static void __dma_free_buffer(struct page *page, size_t size)
{
struct page *e = page + (size >> PAGE_SHIFT);
while (page < e) {
__free_page(page);
page++;
}
}
#ifdef CONFIG_MMU
static void *__alloc_from_contiguous(struct device *dev, size_t size,
pgprot_t prot, struct page **ret_page);
static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
pgprot_t prot, struct page **ret_page,
const void *caller);
static void *
__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
const void *caller)
{
struct vm_struct *area;
unsigned long addr;
/*
* DMA allocation can be mapped to user space, so lets
* set VM_USERMAP flags too.
*/
area = get_vm_area_caller(size, VM_USERMAP,
caller);
if (!area)
return NULL;
addr = (unsigned long)area->addr;
area->phys_addr = __pfn_to_phys(page_to_pfn(page));
if (ioremap_page_range(addr, addr + size, area->phys_addr, prot)) {
vunmap((void *)addr);
return NULL;
}
return (void *)addr;
}
static void __dma_free_remap(void *cpu_addr, size_t size)
{
unsigned int flags = VM_USERMAP;
struct vm_struct *area = find_vm_area(cpu_addr);
if (!area || (area->flags & flags) != flags) {
WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
return;
}
unmap_kernel_range((unsigned long)cpu_addr, size);
vunmap(cpu_addr);
}
struct dma_contig_early_reserve {
phys_addr_t base;
unsigned long size;
};
static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;
static int dma_mmu_remap_num __initdata;
void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
{
dma_mmu_remap[dma_mmu_remap_num].base = base;
dma_mmu_remap[dma_mmu_remap_num].size = size;
dma_mmu_remap_num++;
}
void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
phys_addr_t size, pgprot_t prot);
void __init dma_contiguous_remap(void)
{
int i;
for (i = 0; i < dma_mmu_remap_num; i++) {
unsigned long addr;
phys_addr_t start = dma_mmu_remap[i].base;
phys_addr_t end = start + dma_mmu_remap[i].size;
if (start >= end)
continue;
for (addr = start; addr < end; addr += PAGE_SIZE)
create_mapping_noalloc(addr, __phys_to_virt(addr),
PAGE_SIZE, PAGE_KERNEL);
}
}
static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
pgprot_t prot, struct page **ret_page,
const void *caller)
{
struct page *page;
void *ptr;
page = __dma_alloc_buffer(dev, size, gfp);
if (!page)
return NULL;
ptr = __dma_alloc_remap(page, size, gfp, prot, caller);
if (!ptr) {
__dma_free_buffer(page, size);
return NULL;
}
*ret_page = page;
return ptr;
}
static void *__alloc_from_contiguous(struct device *dev, size_t size,
pgprot_t prot, struct page **ret_page)
{
unsigned long order = get_order(size);
size_t count = size >> PAGE_SHIFT;
struct page *page;
page = dma_alloc_from_contiguous(dev, count, order);
if (!page)
return NULL;
*ret_page = page;
return page_address(page);
}
static void __free_from_contiguous(struct device *dev, struct page *page,
size_t size)
{
dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
}
#define nommu() 0
#else /* !CONFIG_MMU */
#define nommu() 1
#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL
#define __alloc_from_pool(size, ret_page) NULL
#define __alloc_from_contiguous(dev, size, prot, ret) NULL
#define __free_from_pool(cpu_addr, size) 0
#define __free_from_contiguous(dev, page, size) do { } while (0)
#define __dma_free_remap(cpu_addr, size) do { } while (0)
#endif /* CONFIG_MMU */
static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
struct page **ret_page)
{
struct page *page;
page = __dma_alloc_buffer(dev, size, gfp);
if (!page)
return NULL;
*ret_page = page;
return page_address(page);
}
static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
gfp_t gfp, pgprot_t prot, bool is_coherent,
unsigned long attrs, const void *caller)
{
u64 mask = get_coherent_dma_mask(dev);
struct page *page = NULL;
struct page **pages = NULL;
void *addr = NULL;
bool noncontig = is_device_dma_noncontig(dev) &&
!dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs) &&
gfpflags_allow_blocking(gfp);
#ifdef CONFIG_DMA_API_DEBUG
u64 limit = (mask + 1) & ~mask;
if (limit && size >= limit) {
dev_warn(dev, "coherent allocation too big (requested %#zx mask %#llx)\n",
size, mask);
return NULL;
}
#endif
if (!mask)
return NULL;
#ifdef CONFIG_ZONE_DMA
if (mask == (u64)arm_dma_limit) {
if ((gfp & GFP_ZONEMASK) != GFP_DMA &&
(gfp & GFP_ZONEMASK) != 0) {
dev_warn(dev, "Invalid GFP flags(%x) passed. "
"GFP_DMA should only be set.",
gfp & GFP_ZONEMASK);
return NULL;
}
gfp |= GFP_DMA;
}
#endif
/*
* Following is a work-around (a.k.a. hack) to prevent pages
* with __GFP_COMP being passed to split_page() which cannot
* handle them. The real problem is that this flag probably
* should be 0 on ARM as it is not supported on this
* platform; see CONFIG_HUGETLBFS.
*/
gfp &= ~(__GFP_COMP);
size = PAGE_ALIGN(size);
if (noncontig)
pages = __alloc_buffer_pages(dev, size, gfp, attrs);
else if (nommu())
addr = __alloc_simple_buffer(dev, size, gfp, &page);
else if (!gfpflags_allow_blocking(gfp))
addr = __alloc_from_pool(size, &page, gfp);
else if (size == PAGE_SIZE || !IS_ENABLED(CONFIG_CMA))
addr = __alloc_remap_buffer(dev, size, gfp, prot,
&page, caller);
else
addr = __alloc_from_contiguous(dev, size, prot, &page);
if (noncontig && !pages)
return NULL;
if (!noncontig && !addr)
return NULL;
*handle = __pfn_to_phys(page_to_pfn(page));
if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) {
if (noncontig) {
return pages;
} else {
int i;
int count = (size >> PAGE_SHIFT);
int array_size = count * sizeof(struct page *);
struct page **pages;
if (array_size <= PAGE_SIZE)
pages = kzalloc(array_size, GFP_KERNEL);
else
pages = vzalloc(array_size);
for (i = 0; i < count; i++)
pages[i] = page + i;
return pages;
}
} else {
if (noncontig) {
addr = dma_common_pages_remap(pages, size, VM_USERMAP,
prot, __builtin_return_address(0));
if (!addr)
__free_buffer_pages(dev, pages, size, attrs);
}
return addr;
}
}
/*
* Allocate DMA-coherent memory space and return both the kernel remapped
* virtual and bus address for that space.
*/
void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
gfp_t gfp, unsigned long attrs)
{
pgprot_t prot = __get_dma_pgprot(attrs, PG_PROT_KERNEL, false);
void *memory;
if (dma_alloc_from_coherent_attr(dev, size, handle, &memory, attrs))
return memory;
return __dma_alloc(dev, size, handle, gfp, prot, false,
attrs, __builtin_return_address(0));
}
static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
{
pgprot_t prot = __get_dma_pgprot(attrs, PG_PROT_KERNEL, true);
void *memory;
if (dma_alloc_from_coherent_attr(dev, size, handle, &memory, attrs))
return memory;
return __dma_alloc(dev, size, handle, gfp, prot, true,
attrs, __builtin_return_address(0));
}
/*
* Create userspace mapping for the DMA-coherent memory.
*/
int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t dma_addr, size_t size,
unsigned long attrs)
{
int ret = -ENXIO;
#ifdef CONFIG_MMU
unsigned long nr_vma_pages =
(vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
unsigned long pfn = __phys_to_pfn(dma_addr);
unsigned long off = vma->vm_pgoff;
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, false);
if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
return ret;
if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
ret = remap_pfn_range(vma, vma->vm_start,
pfn + off,
vma->vm_end - vma->vm_start,
vma->vm_page_prot);
}
#endif /* CONFIG_MMU */
return ret;
}
/*
* Free a buffer as defined by the above mapping.
*/
static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
dma_addr_t handle, unsigned long attrs,
bool is_coherent)
{
struct page *page = pfn_to_page(__phys_to_pfn(handle));
bool noncontig = is_device_dma_noncontig(dev) &&
!dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs);
if (dma_release_from_coherent_attr(dev, size, cpu_addr,
attrs, handle))
return;
if (noncontig) {
struct page **pages = __get_pages(cpu_addr, attrs);
size = PAGE_ALIGN(size);
if (__free_from_pool(cpu_addr, size))
return;
if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) {
unmap_kernel_range((unsigned long)cpu_addr, size);
vunmap(cpu_addr);
}
__free_buffer_pages(dev, pages, size, attrs);
return;
}
if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) {
int count = size >> PAGE_SHIFT;
int array_size = count * sizeof(struct page *);
struct page **pages = (struct page **)cpu_addr;
cpu_addr = (void *)page_address(pages[0]);
if (array_size <= PAGE_SIZE)
kfree(pages);
else
vfree(pages);
}
size = PAGE_ALIGN(size);
if (nommu()) {
__dma_free_buffer(page, size);
} else if (__free_from_pool(cpu_addr, size)) {
return;
} else if (size == PAGE_SIZE || !IS_ENABLED(CONFIG_CMA)) {
__dma_free_remap(cpu_addr, size);
__dma_free_buffer(page, size);
} else {
/*
* Non-atomic allocations cannot be freed with IRQs disabled
*/
WARN_ON(irqs_disabled());
__free_from_contiguous(dev, page, size);
}
}
void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
dma_addr_t handle, unsigned long attrs)
{
__arm_dma_free(dev, size, cpu_addr, handle, attrs, false);
}
static void arm_coherent_dma_free(struct device *dev, size_t size,
void *cpu_addr, dma_addr_t handle,
unsigned long attrs)
{
__arm_dma_free(dev, size, cpu_addr, handle, attrs, true);
}
int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
void *cpu_addr, dma_addr_t handle, size_t size,
unsigned long attrs)
{
bool noncontig = is_device_dma_noncontig(dev) &&
!dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs);
if (noncontig) {
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
struct page **pages = __get_pages(cpu_addr, attrs);
if (!pages) {
dump_stack();
pr_warn("No pages\n");
return -ENXIO;
}
return sg_alloc_table_from_pages(sgt, pages, count, 0, size,
GFP_KERNEL);
} else {
struct page *page = pfn_to_page(__phys_to_pfn(handle));
int ret;
ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
if (unlikely(ret)) {
pr_warn("sg_alloc_table failed\n");
return ret;
}
sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
return 0;
}
}
static void dma_cache_maint_page(struct page *page, unsigned long offset,
size_t size, enum dma_data_direction dir,
void (*op)(const void *, size_t, int))
{
op(page_address(page) + offset, size, dir);
}
/*
* Make an area consistent for devices.
* Note: Drivers should NOT use this function directly, as it will break
* platforms with CONFIG_DMABOUNCE.
* Use the driver DMA support - see dma-mapping.h (dma_sync_*)
*/
static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
size_t size, enum dma_data_direction dir)
{
dma_cache_maint_page(page, off, size, dir, __dma_map_area);
}
static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
size_t size, enum dma_data_direction dir)
{
dma_cache_maint_page(page, off, size, dir, __dma_unmap_area);
/*
* Mark the D-cache clean for this page to avoid extra flushing.
*/
if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE)
set_bit(PG_dcache_clean, &page->flags);
}
/**
* arm_dma_map_sg - map a set of SG buffers for streaming mode DMA
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
* @sg: list of buffers
* @nents: number of buffers to map
* @dir: DMA transfer direction
*
* Map a set of buffers described by scatterlist in streaming mode for DMA.
* This is the scatter-gather version of the dma_map_single interface.
* Here the scatter gather list elements are each tagged with the
* appropriate dma address and length. They are obtained via
* sg_dma_{address,length}.
*
* Device ownership issues as mentioned for dma_map_single are the same
* here.
*/
int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
enum dma_data_direction dir, unsigned long attrs)
{
struct dma_map_ops *ops = get_dma_ops(dev);
struct scatterlist *s;
int i, j;
for_each_sg(sg, s, nents, i) {
#ifdef CONFIG_NEED_SG_DMA_LENGTH
s->dma_length = s->length;
#endif
s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
s->length, dir, attrs);
if (dma_mapping_error(dev, s->dma_address))
goto bad_mapping;
}
return nents;
bad_mapping:
for_each_sg(sg, s, i, j)
ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s),
dir, attrs);
return 0;
}
/**
* arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
* @sg: list of buffers
* @nents: number of buffers to unmap (same as was passed to dma_map_sg)
* @dir: DMA transfer direction (same as was passed to dma_map_sg)
*
* Unmap a set of streaming mode DMA translations. Again, CPU access
* rules concerning calls here are the same as for dma_unmap_single().
*/
void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
enum dma_data_direction dir, unsigned long attrs)
{
struct dma_map_ops *ops = get_dma_ops(dev);
struct scatterlist *s;
int i;
for_each_sg(sg, s, nents, i)
ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s),
dir, attrs);
}
/**
* arm_dma_sync_sg_for_cpu
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
* @sg: list of buffers
* @nents: number of buffers to map (returned from dma_map_sg)
* @dir: DMA transfer direction (same as was passed to dma_map_sg)
*/
void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
int nents, enum dma_data_direction dir)
{
struct dma_map_ops *ops = get_dma_ops(dev);
struct scatterlist *s;
int i;
for_each_sg(sg, s, nents, i)
ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
dir);
}
/**
* arm_dma_sync_sg_for_device
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
* @sg: list of buffers
* @nents: number of buffers to map (returned from dma_map_sg)
* @dir: DMA transfer direction (same as was passed to dma_map_sg)
*/
void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
int nents, enum dma_data_direction dir)
{
struct dma_map_ops *ops = get_dma_ops(dev);
struct scatterlist *s;
int i;
for_each_sg(sg, s, nents, i)
ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
dir);
}
int arm_dma_supported(struct device *dev, u64 mask)
{
if (mask < (u64)arm_dma_limit)
return 0;
return 1;
}
int arm_dma_set_mask(struct device *dev, u64 dma_mask)
{
if (!dev->dma_mask || !dma_supported(dev, dma_mask))
return -EIO;
*dev->dma_mask = dma_mask;
return 0;
}
int arm_dma_mapping_error(struct device *dev, dma_addr_t dev_addr)
{
return dev_addr == DMA_ERROR_CODE;
}
#if defined(CONFIG_ARM_DMA_USE_IOMMU)
static LIST_HEAD(iommu_mapping_list);
static DEFINE_SPINLOCK(iommu_mapping_list_lock);
#if defined(CONFIG_DMA_API_DEBUG)
#if defined(CONFIG_DEBUG_FS)
static dma_addr_t bit_to_addr(size_t pos, dma_addr_t base)
{
return base + pos * (1 << PAGE_SHIFT);
}
static void seq_print_dma_areas(struct seq_file *s, void *bitmap,
dma_addr_t base, size_t bits)
{
/* one bit = one (page + order) sized block */
size_t pos = find_first_bit(bitmap, bits), end;
for (; pos < bits; pos = find_next_bit(bitmap, bits, end + 1)) {
dma_addr_t start_addr, end_addr;
end = find_next_zero_bit(bitmap, bits, pos);
start_addr = bit_to_addr(pos, base);
end_addr = bit_to_addr(end, base) - 1;
seq_printf(s, " %pa-%pa pages=%zu\n",
&start_addr, &end_addr, end - pos);
}
}
static void seq_print_mapping(struct seq_file *s,
struct dma_iommu_mapping *mapping)
{
int i;
size_t mapping_size = mapping->bits << PAGE_SHIFT;
seq_printf(s, " memory map: base=%pa size=%zu domain=%p\n",
&mapping->base, (size_t)(mapping->end - mapping->base),
mapping->domain);
for (i = 0; i < mapping->nr_bitmaps; i++)
seq_print_dma_areas(s, mapping->bitmaps[i],
mapping->base + mapping_size * i, mapping->bits);
}
static void debug_dma_seq_print_mappings(struct seq_file *s)
{
struct dma_iommu_mapping *mapping;
int i = 0;
list_for_each_entry(mapping, &iommu_mapping_list, list) {
seq_printf(s, "Map %d (%p):\n", i, mapping);
seq_print_mapping(s, mapping);
i++;
}
}
static int dump_iommu_mappings(struct seq_file *s, void *data)
{
debug_dma_seq_print_mappings(s);
return 0;
}
static int dump_iommu_mappings_open(struct inode *inode, struct file *file)
{
return single_open(file, dump_iommu_mappings, NULL);
}
static const struct file_operations dump_iommu_mappings_fops = {
.open = dump_iommu_mappings_open,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
#endif /* CONFIG_DEBUG_FS */
#endif /* CONFIG_DMA_API_DEBUG */
void dma_debugfs_platform_info(struct dentry *dent)
{
#ifdef CONFIG_DEBUG_FS
#ifdef CONFIG_DMA_API_DEBUG
debugfs_create_file("dump_mappings", S_IRUGO, dent, NULL,
&dump_iommu_mappings_fops);
#endif
#endif
}
#else /* !CONFIG_ARM_DMA_USE_IOMMU */
static inline void dma_debugfs_platform_info(struct dentry *dent)
{
}
#endif /* !CONFIG_ARM_DMA_USE_IOMMU */
#if defined(CONFIG_DMA_API_DEBUG)
static inline void dma_debug_platform(void)
{
struct dentry *dent;
dent = debugfs_create_dir("dma-api", NULL);
if (dent)
dma_debugfs_platform_info(dent);
}
#else /* !CONFIG_DMA_API_DEBUG */
static void dma_debug_platform(void)
{
}
#endif /* !CONFIG_DMA_API_DEBUG */
#define PREALLOC_DMA_DEBUG_ENTRIES 4096
static int __init dma_debug_do_init(void)
{
dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
dma_debug_platform();
return 0;
}
fs_initcall(dma_debug_do_init);
#ifdef CONFIG_ARM_DMA_USE_IOMMU
/* IOMMU */
static unsigned int prefetch_page_count = 1;
static unsigned int gap_page_count = 1;
#define PF_PAGES_SIZE (prefetch_page_count << PAGE_SHIFT)
#define PG_PAGES (prefetch_page_count + gap_page_count)
static struct page *iova_gap_pages;
static phys_addr_t iova_gap_phys;
static int __init iova_gap_pages_init(void)
{
unsigned long order = get_order(PF_PAGES_SIZE);
iova_gap_pages = alloc_pages(GFP_KERNEL, order);
if (WARN_ON(!iova_gap_pages)) {
prefetch_page_count = 0;
return 0;
}
__dma_clear_buffer(iova_gap_pages, PAGE_SIZE << order, false);
iova_gap_phys = page_to_phys(iova_gap_pages);
return 0;
}
core_initcall(iova_gap_pages_init);
static void iommu_mapping_list_add(struct dma_iommu_mapping *mapping)
{
unsigned long flags;
spin_lock_irqsave(&iommu_mapping_list_lock, flags);
list_add_tail(&mapping->list, &iommu_mapping_list);
spin_unlock_irqrestore(&iommu_mapping_list_lock, flags);
}
static void iommu_mapping_list_del(struct dma_iommu_mapping *mapping)
{
unsigned long flags;
spin_lock_irqsave(&iommu_mapping_list_lock, flags);
list_del(&mapping->list);
spin_unlock_irqrestore(&iommu_mapping_list_lock, flags);
}
static inline int iommu_get_num_pf_pages(struct dma_iommu_mapping *mapping,
unsigned long attrs)
{
/* XXX: give priority to DMA_ATTR_SKIP_IOVA_GAP */
if (dma_get_attr(DMA_ATTR_SKIP_IOVA_GAP, attrs))
return 0;
/* XXX: currently we support only 1 prefetch page */
WARN_ON(mapping->num_pf_page > prefetch_page_count);
return mapping->num_pf_page;
}
static inline size_t iommu_get_size_pf_pages(struct dma_iommu_mapping *mapping,
unsigned long attrs)
{
return iommu_get_num_pf_pages(mapping, attrs) << PAGE_SHIFT;
}
static inline int iommu_gap_pg_count(struct dma_iommu_mapping *mapping,
unsigned long attrs)
{
if (dma_get_attr(DMA_ATTR_SKIP_IOVA_GAP, attrs))
return 0;
return mapping->gap_page ? gap_page_count : 0;
}
#ifdef CONFIG_DMA_API_DEBUG
static size_t _iommu_unmap(struct dma_iommu_mapping *mapping,
unsigned long iova, size_t bytes)
{
unsigned long start, offs, end;
u32 bitmap_index, bitmap_last_index;
dma_addr_t bitmap_base;
size_t mapping_size = mapping->bits << PAGE_SHIFT;
int i;
bitmap_index = (u32)((iova - mapping->base) / mapping_size);
bitmap_last_index = DIV_ROUND_UP(iova + bytes - mapping->base,
mapping_size);
bitmap_base = mapping->base + mapping_size * bitmap_index;
if ((iova < mapping->base) || bitmap_index > mapping->extensions ||
bitmap_last_index > mapping->extensions) {
WARN(1, "trying to unmap invalid iova\n");
return -EINVAL;
}
offs = (iova - bitmap_base) >> PAGE_SHIFT;
end = offs + (PAGE_ALIGN(bytes) >> PAGE_SHIFT);
/*
* [offs, end) is the portion of the requested region for unmap
* that falls into current bitmap.
* NOTE: This logic can't guarantee detection at the first faulty unmap
* unless the page 'end' is free. IOW this will work always when you
* enable gap pages.
*/
for (i = bitmap_index; i < bitmap_last_index;
i++, offs = 0, end -= mapping->bits) {
start = find_next_zero_bit(mapping->bitmaps[i],
min_t(u32, mapping->bits, end),
offs);
if ((start >= offs) && (start < end)) {
WARN(1, "trying to unmap already unmapped area\n");
return -EINVAL;
}
}
return iommu_unmap(mapping->domain, iova, bytes);
}
#else
#define _iommu_unmap(mapping, iova, bytes) \
iommu_unmap(mapping->domain, iova, bytes)
#endif
static int dma_get_ioprot(unsigned long attrs,
enum dma_data_direction dir, bool coherent)
{
unsigned long ioprot;
#if ENABLE_IOMMU_DMA_OPS
ioprot = dma_direction_to_prot(dir, coherent);
#else
ioprot = IOMMU_READ | IOMMU_WRITE;
if (dma_get_attr(DMA_ATTR_READ_ONLY, (unsigned long)attrs))
ioprot &= ~IOMMU_WRITE;
else if (dma_get_attr(DMA_ATTR_WRITE_ONLY, (unsigned long)attrs))
ioprot &= ~IOMMU_READ;
if (coherent)
ioprot |= IOMMU_CACHE;
#endif
dma_qualify_ioprot(dir, &ioprot);
return ioprot;
}
static int pg_iommu_map(struct dma_iommu_mapping *mapping, unsigned long iova,
phys_addr_t phys, size_t len, unsigned long prot,
enum dma_data_direction dir, bool coherent)
{
int err;
unsigned long attrs = (unsigned long)prot;
struct iommu_domain *domain = mapping->domain;
bool need_prefetch_page = !!iommu_get_num_pf_pages(mapping, attrs);
unsigned long ioprot = dma_get_ioprot(attrs, dir, coherent);
if (need_prefetch_page) {
err = iommu_map(domain, iova + len, iova_gap_phys,
PF_PAGES_SIZE, ioprot);
if (err)
return err;
}
err = iommu_map(domain, iova, phys, len, ioprot);
if (err && need_prefetch_page)
_iommu_unmap(mapping, iova + len, PF_PAGES_SIZE);
return err;
}
/*
* __alloc_buffer_pages - Allocates a buffer of pages to be mapped into
* IOVA space. Used by IOVA alloc for all allocations or DMA alloc
* when non-contiguous memory is needed.
*/
static struct page **__alloc_buffer_pages(struct device *dev, size_t size,
gfp_t gfp, unsigned long attrs)
{
struct page **pages;
bool coherent = is_device_dma_coherent(dev);
int count = size >> PAGE_SHIFT;
int array_size = count * sizeof(struct page *);
int i = 0;
if (array_size <= PAGE_SIZE)
pages = kzalloc(array_size, GFP_KERNEL);
else
pages = vzalloc(array_size);
if (!pages)
return NULL;
if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs))
{
unsigned long order = get_order(size);
struct page *page;
page = dma_alloc_from_contiguous(dev, count, order);
if (!page)
goto error;
for (i = 0; i < count; i++)
pages[i] = page + i;
dmastats_alloc_or_map(dev, size, ALLOC_OR_FREE);
return pages;
}
/*
* IOMMU can map any pages, so himem can also be used here
*/
if (!(gfp & GFP_DMA) && !(gfp & GFP_DMA32))
gfp |= __GFP_HIGHMEM;
gfp |= __GFP_NOWARN;
while (count) {
int j, order = __fls(count);
pages[i] = alloc_pages(gfp, order);
while (!pages[i] && order)
pages[i] = alloc_pages(gfp, --order);
if (!pages[i])
goto error;
if (order) {
split_page(pages[i], order);
j = 1 << order;
while (--j)
pages[i + j] = pages[i] + j;
}
__dma_clear_buffer(pages[i], PAGE_SIZE << order, coherent);
i += 1 << order;
count -= 1 << order;
}
dmastats_alloc_or_map(dev, size, ALLOC_OR_FREE);
return pages;
error:
while (i--)
if (pages[i])
__free_pages(pages[i], 0);
if (array_size <= PAGE_SIZE)
kfree(pages);
else
vfree(pages);
return NULL;
}
/*
* __free_buffer_pages: frees the buffer previously allocated by
* __alloc_buffer_pages
*/
static int __free_buffer_pages(struct device *dev, struct page **pages,
size_t size, unsigned long attrs)
{
int count = size >> PAGE_SHIFT;
int array_size = count * sizeof(struct page *);
int i;
if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) {
dma_release_from_contiguous(dev, pages[0], count);
} else {
for (i = 0; i < count; i++)
if (pages[i])
__free_pages(pages[i], 0);
}
if (array_size <= PAGE_SIZE)
kfree(pages);
else
vfree(pages);
dmastats_free_or_unmap(dev, size, ALLOC_OR_FREE);
return 0;
}
#ifdef CONFIG_DMA_API_DEBUG
#define __iommu_alloc_remap(pages, size, gfp, prot, caller) \
___iommu_alloc_remap(dev, pages, size, gfp, prot, caller)
static void *
___iommu_alloc_remap(struct device *dev, struct page **pages, size_t size,
gfp_t gfp, pgprot_t prot, const void *caller)
{
void *ptr = __iommu_alloc_remap(pages, size, gfp, prot, caller);
if (ptr)
dmastats_alloc_or_map(dev, size, CPU_MAP_OR_UNMAP);
return ptr;
}
#endif
/*
* __get_pages: returns the list of pages associated with a given address
* that had previously been allocated by __alloc_buffer_pages
*/
static struct page **__get_pages(void *cpu_addr, unsigned long attrs)
{
struct vm_struct *area;
int offset = 0;
if (__in_atomic_pool(cpu_addr, PAGE_SIZE)) {
phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool,
(unsigned long)cpu_addr);
struct gen_pool_chunk *chunk;
rcu_read_lock();
/* NOTE: this works as only a single chunk is present
* for atomic pool
*/
chunk = list_first_or_null_rcu(&atomic_pool->chunks,
struct gen_pool_chunk,
next_chunk);
phys -= chunk->phys_addr;
rcu_read_unlock();
offset = phys >> PAGE_SHIFT;
return atomic_pool_pages + offset;
}
if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
return cpu_addr;
area = find_vm_area(cpu_addr);
if (area)
return &area->pages[offset];
return NULL;
}
static dma_addr_t arm_iommu_map_page_at(struct device *dev, struct page *page,
dma_addr_t dma_addr, unsigned long offset, size_t size,
enum dma_data_direction dir, unsigned long attrs)
{
struct dma_iommu_mapping *mapping = dev->archdata.mapping;
int ret, len = PAGE_ALIGN(size + offset);
bool coherent = is_device_dma_coherent(dev);
if (!coherent && !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
__dma_page_cpu_to_dev(page, offset, size, dir);
ret = pg_iommu_map(mapping, dma_addr,
page_to_phys(page), len, (ulong)attrs,
dir, coherent);
if (ret < 0)
return DMA_ERROR_CODE;
trace_dmadebug_map_page(dev, dma_addr, size, page);
return dma_addr + offset;
}
static dma_addr_t arm_iommu_map_at(struct device *dev, dma_addr_t dma_addr,
phys_addr_t phys, size_t size,
enum dma_data_direction dir,
unsigned long attrs)
{
dma_addr_t iova, ret;
iova = dma_addr;
ret = arm__alloc_iova_at(dev, iova, size);
if (iova == -ENXIO) {
/* Linear map request is out side iova window.
* It usually happens when an SMMU client need to
* access physcial memory and IOVA range is not overlapping
* with it. The client would need mapping to be able to access
* desired phys memory. Map request should be carried out even
* without successful IOVA allocation.
*/
iova = dma_addr;
} else if (ret == DMA_ERROR_CODE) {
return ret;
} else if (iova != dma_addr) {
arm__free_iova(dev, ret, size);
return DMA_ERROR_CODE;
}
return arm_iommu_map_page_at(dev, phys_to_page(phys), iova,
0, size, dir, attrs);
}
bool device_is_iommuable(struct device *dev)
{
return (dev->archdata.dma_ops == &iommu_dma_ops);
}
EXPORT_SYMBOL(device_is_iommuable);
/* __alloc_iova_at() is broken with extensions enabled.
* Disable the extensions till it is fixed.
*/
#define DISABLE_EXTENSIONS 1
/**
* arm_iommu_create_mapping
* @bus: pointer to the bus holding the client device (for IOMMU calls)
* @base: start address of the valid IO address space
* @size: maximum size of the valid IO address space
*
* Creates a mapping structure which holds information about used/unused
* IO address ranges, which is required to perform memory allocation and
* mapping with IOMMU aware functions.
*
* The client device need to be attached to the mapping with
* arm_iommu_attach_device function.
*/
struct dma_iommu_mapping *
arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size)
{
struct dma_iommu_mapping *mapping;
struct iommu_domain *domain;
int err = -ENOMEM;
mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
if (!mapping)
goto err;
mapping->base = base;
mapping->end = base + size;
spin_lock_init(&mapping->lock);
mapping->domain = iommu_domain_alloc(bus);
if (!mapping->domain)
goto err4;
domain = mapping->domain;
domain->geometry.aperture_start = base;
domain->geometry.aperture_end = base + size;
domain->geometry.force_aperture = 1;
kref_init(&mapping->kref);
iommu_mapping_list_add(mapping);
return mapping;
err4:
kfree(mapping);
err:
return ERR_PTR(err);
}
static void release_iommu_mapping(struct kref *kref)
{
struct dma_iommu_mapping *mapping =
container_of(kref, struct dma_iommu_mapping, kref);
iommu_mapping_list_del(mapping);
iommu_domain_free(mapping->domain);
kfree(mapping);
}
void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
{
if (mapping)
kref_put(&mapping->kref, release_iommu_mapping);
}
/**
* arm_iommu_attach_device
* @dev: valid struct device pointer
* @mapping: io address space mapping structure (returned from
* arm_iommu_create_mapping)
*
* Attaches specified io address space mapping to the provided device,
* this replaces the dma operations (dma_map_ops pointer) with the
* IOMMU aware version. More than one client might be attached to
* the same io address space mapping.
*/
int arm_iommu_attach_device(struct device *dev,
struct dma_iommu_mapping *mapping)
{
int err;
struct dma_map_ops *org_ops;
struct dma_iommu_mapping *org_map;
struct iommu_domain *domain;
#ifdef CONFIG_DMA_API_DEBUG
struct iommu_usage *device_ref;
device_ref = kzalloc(sizeof(*device_ref), GFP_KERNEL);
if (!device_ref)
return -ENOMEM;
device_ref->dev = dev;
list_add(&device_ref->recordlist, &iommu_rlist_head);
#endif
org_ops = get_dma_ops(dev);
set_dma_ops(dev, &iommu_dma_ops);
org_map = dev->archdata.mapping;
dev->archdata.mapping = mapping;
err = iommu_attach_device(mapping->domain, dev);
if (err) {
set_dma_ops(dev, org_ops);
dev->archdata.mapping = org_map;
#ifdef CONFIG_DMA_API_DEBUG
list_del(&device_ref->recordlist);
kfree(device_ref);
#endif
return err;
}
domain = mapping->domain;
if (!iommu_get_dma_cookie(domain)) {
if (iommu_dma_init_domain(domain,
domain->geometry.aperture_start,
domain->geometry.aperture_end -
domain->geometry.aperture_start, dev))
pr_err("iommu_dma_init_domain failed, %s\n",
dev_name(dev));
}
kref_get(&mapping->kref);
pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
return 0;
}
/**
* arm_iommu_detach_device
* @dev: valid struct device pointer
*
* Detaches the provided device from a previously attached map.
* This voids the dma operations (dma_map_ops pointer)
*/
void arm_iommu_detach_device(struct device *dev)
{
struct dma_iommu_mapping *mapping;
#ifdef CONFIG_DMA_API_DEBUG
struct iommu_usage *device_ref, *tmp;
#endif
mapping = to_dma_iommu_mapping(dev);
if (!mapping) {
dev_warn(dev, "Not attached\n");
return;
}
#ifdef CONFIG_DMA_API_DEBUG
list_for_each_entry_safe(device_ref, tmp, &iommu_rlist_head,
recordlist) {
if (dev == device_ref->dev) {
list_del(&device_ref->recordlist);
kfree(device_ref);
break;
}
}
#endif
iommu_detach_device(mapping->domain, dev);
kref_put(&mapping->kref, release_iommu_mapping);
mapping = NULL;
set_dma_ops(dev, NULL);
pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
}
#endif
#ifdef CONFIG_DMA_API_DEBUG
static void
add_value(struct dma_iommu_mapping *device_ref, size_t size, const int type)
{
switch (type) {
case ALLOC_OR_FREE:
atomic64_add(size, &device_ref->alloc_size);
break;
case ATOMIC_ALLOC_OR_FREE:
atomic64_add(size, &device_ref->atomic_alloc_size);
break;
case MAP_OR_UNMAP:
atomic64_add(size, &device_ref->map_size);
break;
case CPU_MAP_OR_UNMAP:
atomic64_add(size, &device_ref->cpu_map_size);
break;
default:
pr_info("Invalid argument in function %s\n", __func__);
}
}
static void
sub_value(struct dma_iommu_mapping *device_ref, size_t size, const int type)
{
switch (type) {
case ALLOC_OR_FREE:
atomic64_sub(size, &device_ref->alloc_size);
break;
case ATOMIC_ALLOC_OR_FREE:
atomic64_sub(size, &device_ref->atomic_alloc_size);
break;
case MAP_OR_UNMAP:
atomic64_sub(size, &device_ref->map_size);
break;
case CPU_MAP_OR_UNMAP:
atomic64_sub(size, &device_ref->cpu_map_size);
break;
default:
pr_info("Invalid argument in function %s\n", __func__);
}
}
static size_t
dmastats_alloc_or_map(struct device *dev, size_t size, const int type)
{
struct dma_iommu_mapping *mapping;
mapping = to_dma_iommu_mapping(dev);
if (mapping)
add_value(mapping, size, type);
else
atomic64_add(size, &dev_is_null->alloc_size);
return 0;
}
static size_t dmastats_free_or_unmap(struct device *dev, size_t size,
const int type)
{
struct dma_iommu_mapping *mapping;
mapping = to_dma_iommu_mapping(dev);
if (mapping)
sub_value(mapping, size, type);
else
atomic64_sub(size, &dev_is_null->alloc_size);
return 0;
}
static int dmastats_debug_show(struct seq_file *s, void *data)
{
struct iommu_usage *device_ref = NULL, *tmp;
size_t alloc_size = 0, map_size = 0;
size_t atomic_alloc_size = 0, cpu_map_size = 0;
struct dma_iommu_mapping *mapping;
seq_printf(s, "%-24s %18s %18s %18s %18s\n", "DEV_NAME", "ALLOCATED",
"ATOMIC_ALLOCATED", "MAPPED", "CPU_MAPPED");
list_for_each_entry_safe(device_ref, tmp, &iommu_rlist_head,
recordlist) {
mapping = to_dma_iommu_mapping(device_ref->dev);
alloc_size += atomic64_read(&mapping->alloc_size);
map_size += atomic64_read(&mapping->map_size);
atomic_alloc_size += atomic64_read(&mapping->atomic_alloc_size);
cpu_map_size += atomic64_read(&mapping->cpu_map_size);
seq_printf(s, "%-24s %18ld %18ld %18ld %18ld\n",
dev_name(device_ref->dev),
atomic64_read(&mapping->alloc_size),
atomic64_read(&mapping->atomic_alloc_size),
atomic64_read(&mapping->map_size),
atomic64_read(&mapping->cpu_map_size));
}
alloc_size += atomic64_read(&dev_is_null->alloc_size);
map_size += atomic64_read(&dev_is_null->map_size);
atomic_alloc_size += atomic64_read(&dev_is_null->atomic_alloc_size);
cpu_map_size += atomic64_read(&dev_is_null->cpu_map_size);
seq_printf(s, "%-24s %18ld %18ld %18ld %18ld\n",
dev_is_null->dev_name,
atomic64_read(&dev_is_null->alloc_size),
atomic64_read(&dev_is_null->atomic_alloc_size),
atomic64_read(&dev_is_null->map_size),
atomic64_read(&dev_is_null->cpu_map_size));
seq_printf(s, "\n%-24s %18zu %18zu %18zu %18zu\n", "Total",
alloc_size, atomic_alloc_size, map_size, cpu_map_size);
return 0;
}
static int dmastats_debug_open(struct inode *inode, struct file *file)
{
return single_open(file, dmastats_debug_show, NULL);
}
static const struct file_operations debug_dmastats_fops = {
.open = dmastats_debug_open,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
static int __init dmastats_debug_init(void)
{
struct dentry *rootdir, *ret;
rootdir = debugfs_create_dir("dma", NULL);
if (!rootdir) {
pr_err("Failed to create dma directory!\n");
return -ENOMEM;
}
ret = debugfs_create_file("usage", S_IRUGO, rootdir, NULL,
&debug_dmastats_fops);
if (!ret) {
pr_err("Failed to create usage debug file!\n");
return -ENOMEM;
}
dev_is_null = kzalloc(sizeof(*dev_is_null), GFP_KERNEL);
if (!dev_is_null)
return -ENOMEM;
dev_is_null->dev_name = NULL_DEV;
return 0;
}
static void __exit dmastats_debug_exit(void)
{
struct iommu_usage *device_ref = NULL, *tmp;
list_for_each_entry_safe(device_ref, tmp, &iommu_rlist_head,
recordlist) {
list_del(&device_ref->recordlist);
kfree(device_ref);
}
kfree(dev_is_null);
}
late_initcall(dmastats_debug_init);
module_exit(dmastats_debug_exit);
#endif
Reference in New Issue View Git Blame Copy Permalink