/* * Coherent per-device memory handling. * Borrowed from i386 */ #include #include #include #include #include #include #include #include #include #ifdef pr_fmt #undef pr_fmt #endif #define pr_fmt(fmt) "%s:%d: " fmt, __func__, __LINE__ #define RESIZE_MAGIC 0xC11A900d struct heap_info { int magic; char *name; /* number of chunks memory to manage in */ unsigned int num_chunks; /* dev to manage cma/coherent memory allocs, if resize allowed */ struct device dev; /* device to allocate memory from cma */ struct device *cma_dev; /* lock to synchronise heap resizing */ struct mutex resize_lock; /* CMA chunk size if resize supported */ size_t cma_chunk_size; /* heap current base */ phys_addr_t curr_base; /* heap current allocated memory in bytes */ size_t curr_used; /* heap current length */ size_t curr_len; /* heap lowest base */ phys_addr_t cma_base; /* heap max length */ size_t cma_len; size_t rem_chunk_size; struct dentry *dma_debug_root; int (*update_resize_cfg)(phys_addr_t , size_t); /* The timer used to wakeup the shrink thread */ struct timer_list shrink_timer; /* Pointer to the current shrink thread for this resizable heap */ struct task_struct *task; unsigned long shrink_interval; size_t floor_size; }; #ifdef CONFIG_ARM_DMA_IOMMU_ALIGNMENT #define DMA_BUF_ALIGNMENT CONFIG_ARM_DMA_IOMMU_ALIGNMENT #else #define DMA_BUF_ALIGNMENT 8 #endif struct dma_coherent_mem { void *virt_base; dma_addr_t device_base; unsigned long pfn_base; size_t size; int flags; unsigned long alloc_shift; unsigned long *bitmap; spinlock_t spinlock; }; static int shrink_thread(void *arg); static void shrink_resizable_heap(struct heap_info *h); static int heap_resize_locked(struct heap_info *h, bool skip_vpr_config); static void release_from_contiguous_heap(struct heap_info *h, phys_addr_t base, size_t len); static int update_vpr_config(struct heap_info *h); #define RESIZE_DEFAULT_SHRINK_AGE 3 bool dma_is_coherent_dev(struct device *dev) { struct heap_info *h; if (!dev) return false; h = dev_get_drvdata(dev); if (!h) return false; if (h->magic != RESIZE_MAGIC) return false; return true; } EXPORT_SYMBOL(dma_is_coherent_dev); static void dma_debugfs_init(struct device *dev, struct heap_info *heap) { if (!heap->dma_debug_root) { heap->dma_debug_root = debugfs_create_dir(dev_name(dev), NULL); if (IS_ERR_OR_NULL(heap->dma_debug_root)) { dev_err(dev, "couldn't create debug files\n"); return; } } if (sizeof(phys_addr_t) == sizeof(u64)) { debugfs_create_x64("curr_base", S_IRUGO, heap->dma_debug_root, (u64 *)&heap->curr_base); debugfs_create_x64("curr_size", S_IRUGO, heap->dma_debug_root, (u64 *)&heap->curr_len); debugfs_create_x64("cma_base", S_IRUGO, heap->dma_debug_root, (u64 *)&heap->cma_base); debugfs_create_x64("cma_size", S_IRUGO, heap->dma_debug_root, (u64 *)&heap->cma_len); debugfs_create_x64("cma_chunk_size", S_IRUGO, heap->dma_debug_root, (u64 *)&heap->cma_chunk_size); debugfs_create_x64("floor_size", S_IRUGO, heap->dma_debug_root, (u64 *)&heap->floor_size); } else { debugfs_create_x32("curr_base", S_IRUGO, heap->dma_debug_root, (u32 *)&heap->curr_base); debugfs_create_x32("curr_size", S_IRUGO, heap->dma_debug_root, (u32 *)&heap->curr_len); debugfs_create_x32("cma_base", S_IRUGO, heap->dma_debug_root, (u32 *)&heap->cma_base); debugfs_create_x32("cma_size", S_IRUGO, heap->dma_debug_root, (u32 *)&heap->cma_len); debugfs_create_x32("cma_chunk_size", S_IRUGO, heap->dma_debug_root, (u32 *)&heap->cma_chunk_size); debugfs_create_x32("floor_size", S_IRUGO, heap->dma_debug_root, (u32 *)&heap->floor_size); } debugfs_create_x32("num_cma_chunks", S_IRUGO, heap->dma_debug_root, (u32 *)&heap->num_chunks); } int dma_set_resizable_heap_floor_size(struct device *dev, size_t floor_size) { int ret = 0; struct heap_info *h = NULL; phys_addr_t orig_base, prev_base, left_chunks_base, right_chunks_base; size_t orig_len, prev_len, left_chunks_len, right_chunks_len; if (!dma_is_coherent_dev(dev)) return -ENODEV; h = dev_get_drvdata(dev); if (!h) return -ENOENT; mutex_lock(&h->resize_lock); orig_base = h->curr_base; orig_len = h->curr_len; right_chunks_base = h->curr_base + h->curr_len; left_chunks_len = right_chunks_len = 0; h->floor_size = floor_size > h->cma_len ? h->cma_len : floor_size; while (h->curr_len < h->floor_size) { prev_base = h->curr_base; prev_len = h->curr_len; ret = heap_resize_locked(h, true); if (ret) goto fail_set_floor; if (h->curr_base < prev_base) { left_chunks_base = h->curr_base; left_chunks_len += (h->curr_len - prev_len); } else { right_chunks_len += (h->curr_len - prev_len); } } ret = update_vpr_config(h); if (!ret) { dev_dbg(&h->dev, "grow heap base from=%pa to=%pa," " len from=0x%zx to=0x%zx\n", &orig_base, &h->curr_base, orig_len, h->curr_len); goto success_set_floor; } fail_set_floor: if (left_chunks_len != 0) release_from_contiguous_heap(h, left_chunks_base, left_chunks_len); if (right_chunks_len != 0) release_from_contiguous_heap(h, right_chunks_base, right_chunks_len); h->curr_base = orig_base; h->curr_len = orig_len; success_set_floor: if (h->task) mod_timer(&h->shrink_timer, jiffies + h->shrink_interval); mutex_unlock(&h->resize_lock); if (!h->task) shrink_resizable_heap(h); return ret; } EXPORT_SYMBOL(dma_set_resizable_heap_floor_size); static bool dma_init_coherent_memory( phys_addr_t phys_addr, dma_addr_t device_addr, size_t size, int flags, unsigned long alloc_shift, struct dma_coherent_mem **mem) { struct dma_coherent_mem *dma_mem = NULL; void __iomem *mem_base = NULL; int bits = size >> alloc_shift; int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long); if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO | DMA_MEMORY_NOMAP)) == 0) goto out; if (!size) goto out; if (flags & DMA_MEMORY_NOMAP) goto skip_mapping; if (flags & DMA_MEMORY_MAP) mem_base = memremap(phys_addr, size, MEMREMAP_WC); else mem_base = ioremap(phys_addr, size); if (!mem_base) goto out; skip_mapping: dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL); if (!dma_mem) goto out; dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL); if (!dma_mem->bitmap) goto out; dma_mem->virt_base = mem_base; dma_mem->device_base = device_addr; dma_mem->pfn_base = PFN_DOWN(phys_addr); dma_mem->size = bits; dma_mem->flags = flags; dma_mem->alloc_shift = alloc_shift; spin_lock_init(&dma_mem->spinlock); *mem = dma_mem; return true; out: kfree(dma_mem); if (mem_base) { if (flags & DMA_MEMORY_MAP) memunmap(mem_base); else iounmap(mem_base); } return false; } static void dma_release_coherent_memory(struct dma_coherent_mem *mem) { if (!mem) return; if (!(mem->flags & DMA_MEMORY_NOMAP)) goto skip_unmapping; if (mem->flags & DMA_MEMORY_MAP) memunmap(mem->virt_base); else iounmap(mem->virt_base); skip_unmapping: kfree(mem->bitmap); kfree(mem); } static int declare_coherent_heap(struct device *dev, phys_addr_t base, size_t size, int map) { int err; int flags = map ? DMA_MEMORY_MAP : DMA_MEMORY_NOMAP; BUG_ON(dev->dma_mem); dma_set_coherent_mask(dev, DMA_BIT_MASK(64)); err = dma_declare_coherent_memory(dev, 0, base, size, flags); if (err & flags) { dev_dbg(dev, "dma coherent mem base (%pa) size (0x%zx) %x\n", &base, size, flags); return 0; } dev_err(dev, "declare dma coherent_mem fail %pa 0x%zx %x\n", &base, size, flags); return -ENOMEM; } int dma_declare_coherent_resizable_cma_memory(struct device *dev, struct dma_declare_info *dma_info) { #ifdef CONFIG_DMA_CMA int err = 0; struct heap_info *heap_info = NULL; struct dma_contiguous_stats stats; if (!dev || !dma_info || !dma_info->name || !dma_info->cma_dev) return -EINVAL; heap_info = kzalloc(sizeof(*heap_info), GFP_KERNEL); if (!heap_info) return -ENOMEM; heap_info->magic = RESIZE_MAGIC; heap_info->name = kmalloc(strlen(dma_info->name) + 1, GFP_KERNEL); if (!heap_info->name) { kfree(heap_info); return -ENOMEM; } dma_get_contiguous_stats(dma_info->cma_dev, &stats); pr_info("resizable heap=%s, base=%pa, size=0x%zx\n", dma_info->name, &stats.base, stats.size); strcpy(heap_info->name, dma_info->name); dev_set_name(dev, "dma-%s", heap_info->name); heap_info->cma_dev = dma_info->cma_dev; heap_info->cma_chunk_size = dma_info->size ? : stats.size; heap_info->cma_base = stats.base; heap_info->cma_len = stats.size; heap_info->curr_base = stats.base; dev_set_name(heap_info->cma_dev, "cma-%s-heap", heap_info->name); mutex_init(&heap_info->resize_lock); if (heap_info->cma_len < heap_info->cma_chunk_size) { dev_err(dev, "error cma_len(0x%zx) < cma_chunk_size(0x%zx)\n", heap_info->cma_len, heap_info->cma_chunk_size); err = -EINVAL; goto fail; } heap_info->num_chunks = div64_u64_rem(heap_info->cma_len, (u64)heap_info->cma_chunk_size, (u64 *)&heap_info->rem_chunk_size); if (heap_info->rem_chunk_size) { heap_info->num_chunks++; dev_info(dev, "heap size is not multiple of cma_chunk_size " "heap_info->num_chunks (%d) rem_chunk_size(0x%zx)\n", heap_info->num_chunks, heap_info->rem_chunk_size); } else heap_info->rem_chunk_size = heap_info->cma_chunk_size; dev_set_name(&heap_info->dev, "%s-heap", heap_info->name); if (dma_info->notifier.ops) heap_info->update_resize_cfg = dma_info->notifier.ops->resize; dev_set_drvdata(dev, heap_info); dma_debugfs_init(dev, heap_info); if (declare_coherent_heap(&heap_info->dev, heap_info->cma_base, heap_info->cma_len, (dma_info->notifier.ops && dma_info->notifier.ops->resize) ? 0 : 1)) goto declare_fail; heap_info->dev.dma_mem->size = 0; heap_info->shrink_interval = HZ * RESIZE_DEFAULT_SHRINK_AGE; kthread_run(shrink_thread, heap_info, "%s-shrink_thread", heap_info->name); if (dma_info->notifier.ops && dma_info->notifier.ops->resize) dma_contiguous_enable_replace_pages(dma_info->cma_dev); pr_info("resizable cma heap=%s create successful", heap_info->name); return 0; declare_fail: kfree(heap_info->name); fail: kfree(heap_info); return err; #else return -EINVAL; #endif } EXPORT_SYMBOL(dma_declare_coherent_resizable_cma_memory); static int dma_assign_coherent_memory(struct device *dev, struct dma_coherent_mem *mem) { if (dev->dma_mem) return -EBUSY; dev->dma_mem = mem; /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */ return 0; } int _dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, dma_addr_t device_addr, size_t size, unsigned long alloc_shift, int flags) { struct dma_coherent_mem *mem; if (!dma_init_coherent_memory(phys_addr, device_addr, size, flags, alloc_shift, &mem)) return 0; if (dma_assign_coherent_memory(dev, mem) == 0) return flags & DMA_MEMORY_NOMAP ? DMA_MEMORY_NOMAP : flags & DMA_MEMORY_MAP ? DMA_MEMORY_MAP : DMA_MEMORY_IO; dma_release_coherent_memory(mem); return 0; } EXPORT_SYMBOL(_dma_declare_coherent_memory); unsigned long dma_get_coherent_memory_alloc_shift(struct device *dev) { if (dev->dma_mem) return dev->dma_mem->alloc_shift; return -EINVAL; } EXPORT_SYMBOL(dma_get_coherent_memory_alloc_shift); int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, dma_addr_t device_addr, size_t size, int flags) { return _dma_declare_coherent_memory(dev, phys_addr, device_addr, size, PAGE_SHIFT, flags); } EXPORT_SYMBOL(dma_declare_coherent_memory); static phys_addr_t alloc_from_contiguous_heap( struct heap_info *h, phys_addr_t base, size_t len) { size_t count; struct page *page; unsigned long order; dev_dbg(h->cma_dev, "req at base (%pa) size (0x%zx)\n", &base, len); order = get_order(len); count = PAGE_ALIGN(len) >> PAGE_SHIFT; page = dma_alloc_at_from_contiguous(h->cma_dev, count, order, base, true); if (!page) { dev_err(h->cma_dev, "dma_alloc_at_from_contiguous failed\n"); goto dma_alloc_err; } base = page_to_phys(page); dev_dbg(h->cma_dev, "allocated at base (%pa) size (0x%zx)\n", &base, len); BUG_ON(base < h->cma_base || base - h->cma_base + len > h->cma_len); return base; dma_alloc_err: return DMA_ERROR_CODE; } static void release_from_contiguous_heap( struct heap_info *h, phys_addr_t base, size_t len) { struct page *page = phys_to_page(base); size_t count = PAGE_ALIGN(len) >> PAGE_SHIFT; dma_release_from_contiguous(h->cma_dev, page, count); dev_dbg(h->cma_dev, "released at base (%pa) size (0x%zx)\n", &base, len); } static void get_first_and_last_idx(struct heap_info *h, int *first_alloc_idx, int *last_alloc_idx) { if (!h->curr_len) { *first_alloc_idx = -1; *last_alloc_idx = h->num_chunks; } else { *first_alloc_idx = div_u64(h->curr_base - h->cma_base, h->cma_chunk_size); *last_alloc_idx = div_u64(h->curr_base - h->cma_base + h->curr_len + h->cma_chunk_size - h->rem_chunk_size, h->cma_chunk_size) - 1; } } static void update_alloc_range(struct heap_info *h) { if (!h->curr_len) h->dev.dma_mem->size = 0; else h->dev.dma_mem->size = (h->curr_base - h->cma_base + h->curr_len) >> PAGE_SHIFT; } static int heap_resize_locked(struct heap_info *h, bool skip_vpr_config) { phys_addr_t base = -1; size_t len = h->cma_chunk_size; phys_addr_t prev_base = h->curr_base; size_t prev_len = h->curr_len; int alloc_at_idx = 0; int first_alloc_idx; int last_alloc_idx; phys_addr_t start_addr = h->cma_base; get_first_and_last_idx(h, &first_alloc_idx, &last_alloc_idx); pr_debug("req resize, fi=%d,li=%d\n", first_alloc_idx, last_alloc_idx); /* All chunks are in use. Can't grow it. */ if (first_alloc_idx == 0 && last_alloc_idx == h->num_chunks - 1) return -ENOMEM; /* All chunks are free. Attempt to allocate the first chunk. */ if (first_alloc_idx == -1) { base = alloc_from_contiguous_heap(h, start_addr, len); if (base == start_addr) goto alloc_success; BUG_ON(!dma_mapping_error(h->cma_dev, base)); } /* Free chunk before previously allocated chunk. Attempt * to allocate only immediate previous chunk. */ if (first_alloc_idx > 0) { alloc_at_idx = first_alloc_idx - 1; start_addr = alloc_at_idx * h->cma_chunk_size + h->cma_base; base = alloc_from_contiguous_heap(h, start_addr, len); if (base == start_addr) goto alloc_success; BUG_ON(!dma_mapping_error(h->cma_dev, base)); } /* Free chunk after previously allocated chunk. */ if (last_alloc_idx < h->num_chunks - 1) { alloc_at_idx = last_alloc_idx + 1; len = (alloc_at_idx == h->num_chunks - 1) ? h->rem_chunk_size : h->cma_chunk_size; start_addr = alloc_at_idx * h->cma_chunk_size + h->cma_base; base = alloc_from_contiguous_heap(h, start_addr, len); if (base == start_addr) goto alloc_success; BUG_ON(!dma_mapping_error(h->cma_dev, base)); } if (dma_mapping_error(h->cma_dev, base)) dev_err(&h->dev, "Failed to allocate contiguous memory on heap grow req\n"); return -ENOMEM; alloc_success: if (!h->curr_len || h->curr_base > base) h->curr_base = base; h->curr_len += len; if (!skip_vpr_config && update_vpr_config(h)) goto fail_update; dev_dbg(&h->dev, "grow heap base from=%pa to=%pa," " len from=0x%zx to=0x%zx\n", &prev_base, &h->curr_base, prev_len, h->curr_len); return 0; fail_update: release_from_contiguous_heap(h, base, len); h->curr_base = prev_base; h->curr_len = prev_len; return -ENOMEM; } static int update_vpr_config(struct heap_info *h) { /* Handle VPR configuration updates*/ if (h->update_resize_cfg) { int err = h->update_resize_cfg(h->curr_base, h->curr_len); if (err) { dev_err(&h->dev, "Failed to update heap resize\n"); return err; } dev_dbg(&h->dev, "update vpr base to %pa, size=%zx\n", &h->curr_base, h->curr_len); } update_alloc_range(h); return 0; } static inline struct page **kvzalloc_pages(u32 count) { if (count * sizeof(struct page *) <= PAGE_SIZE) return kzalloc(count * sizeof(struct page *), GFP_KERNEL); else return vzalloc(count * sizeof(struct page *)); } /* retval: !0 on success, 0 on failure */ static int dma_alloc_from_coherent_dev_at(struct device *dev, ssize_t size, dma_addr_t *dma_handle, void **ret, unsigned long attrs, ulong start) { struct dma_coherent_mem *mem; int order; unsigned long flags; int pageno, i = 0; int dma_memory_map = 0; unsigned int count; unsigned int alloc_size; unsigned long align; struct page **pages = NULL; if (!dev) return 0; mem = dev->dma_mem; if (!mem) return 0; order = get_order(size << PAGE_SHIFT >> mem->alloc_shift); if (dma_get_attr(DMA_ATTR_ALLOC_EXACT_SIZE, attrs)) count = ALIGN(size, 1 << mem->alloc_shift) >> mem->alloc_shift; else count = 1 << order; if (!count) return 0; if ((mem->flags & DMA_MEMORY_NOMAP) && dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) { alloc_size = 1; pages = kvzalloc_pages(count); if (!pages) return 0; } else { alloc_size = count; } *dma_handle = DMA_ERROR_CODE; *ret = NULL; spin_lock_irqsave(&mem->spinlock, flags); if (unlikely(size > (mem->size << mem->alloc_shift))) goto err; if ((mem->flags & DMA_MEMORY_NOMAP) && dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) { align = 0; } else { if (order > DMA_BUF_ALIGNMENT) align = (1 << DMA_BUF_ALIGNMENT) - 1; else align = (1 << order) - 1; } while (count) { pageno = bitmap_find_next_zero_area(mem->bitmap, mem->size, start, alloc_size, align); if (pageno >= mem->size) goto err; count -= alloc_size; if (pages) pages[i++] = pfn_to_page(mem->pfn_base + pageno); bitmap_set(mem->bitmap, pageno, alloc_size); } /* * Memory was found in the per-device area. */ *dma_handle = mem->device_base + (pageno << mem->alloc_shift); if (!(mem->flags & DMA_MEMORY_NOMAP)) { *ret = mem->virt_base + (pageno << mem->alloc_shift); dma_memory_map = (mem->flags & DMA_MEMORY_MAP); } else if (dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) { *ret = pages; } spin_unlock_irqrestore(&mem->spinlock, flags); if (mem->flags & DMA_MEMORY_NOMAP) /* Do nothing */; else if (dma_memory_map) memset(*ret, 0, size); else if (*ret) memset_io(*ret, 0, size); return 1; err: while (i--) bitmap_clear(mem->bitmap, page_to_pfn(pages[i]) - mem->pfn_base, alloc_size); spin_unlock_irqrestore(&mem->spinlock, flags); kvfree(pages); /* * In the case where the allocation can not be satisfied from the * per-device area, try to fall back to generic memory if the * constraints allow it. */ return mem->flags & DMA_MEMORY_EXCLUSIVE; } /** * dma_release_from_coherent_dev() - try to free the memory allocated from * per-device coherent memory pool * @dev: device from which the memory was allocated * @size: size of the memory area to free * @vaddr: virtual address of allocated pages * @attrs: DMA Attribute * * This checks whether the memory was allocated from the per-device * coherent memory pool and if so, releases that memory. * * Returns 1 if we correctly released the memory, or 0 if * dma_release_coherent_attr() should proceed with releasing memory from * generic pools. */ int dma_release_from_coherent_dev(struct device *dev, size_t size, void *vaddr, unsigned long attrs) { struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL; void *mem_addr; unsigned int count; unsigned int pageno; unsigned long flags; if (!mem) return 0; if ((mem->flags & DMA_MEMORY_NOMAP) && dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) { struct page **pages = vaddr; int i; spin_lock_irqsave(&mem->spinlock, flags); for (i = 0; i < (size >> PAGE_SHIFT); i++) { pageno = page_to_pfn(pages[i]) - mem->pfn_base; if (WARN_ONCE(pageno > mem->size, "invalid pageno:%d\n", pageno)) continue; bitmap_clear(mem->bitmap, pageno, 1); } spin_unlock_irqrestore(&mem->spinlock, flags); kvfree(pages); return 1; } if (mem->flags & DMA_MEMORY_NOMAP) mem_addr = (void *)(uintptr_t)mem->device_base; else mem_addr = mem->virt_base; if (mem && vaddr >= mem_addr && vaddr - mem_addr < mem->size << PAGE_SHIFT) { unsigned long flags; pageno = (vaddr - mem_addr) >> mem->alloc_shift; if (DMA_ATTR_ALLOC_EXACT_SIZE & attrs) count = PAGE_ALIGN(size) >> PAGE_SHIFT; else count = 1 << get_order(size); spin_lock_irqsave(&mem->spinlock, flags); bitmap_clear(mem->bitmap, pageno, count); spin_unlock_irqrestore(&mem->spinlock, flags); return 1; } return 0; } static int dma_alloc_from_coherent_dev(struct device *dev, ssize_t size, dma_addr_t *dma_handle, void **ret, unsigned long attrs) { return dma_alloc_from_coherent_dev_at(dev, size, dma_handle, ret, attrs, 0); } /* retval: !0 on success, 0 on failure */ static int dma_alloc_from_coherent_heap_dev(struct device *dev, size_t len, dma_addr_t *dma_handle, void **ret, unsigned long attrs) { struct heap_info *h = NULL; if (!dma_is_coherent_dev(dev)) return 0; *dma_handle = DMA_ERROR_CODE; h = dev_get_drvdata(dev); BUG_ON(!h); if (!h) return DMA_MEMORY_EXCLUSIVE; attrs |= DMA_ATTR_ALLOC_EXACT_SIZE; mutex_lock(&h->resize_lock); retry_alloc: /* Try allocation from already existing CMA chunks */ if (dma_alloc_from_coherent_dev_at( &h->dev, len, dma_handle, ret, attrs, (h->curr_base - h->cma_base) >> PAGE_SHIFT)) { if (*dma_handle != DMA_ERROR_CODE) { dev_dbg(&h->dev, "allocated addr %pa len 0x%zx\n", dma_handle, len); h->curr_used += len; } goto out; } if (!heap_resize_locked(h, false)) goto retry_alloc; out: mutex_unlock(&h->resize_lock); return DMA_MEMORY_EXCLUSIVE; } /* retval: !0 on success, 0 on failure */ static int dma_release_from_coherent_heap_dev(struct device *dev, size_t len, void *base, unsigned long attrs) { int idx = 0; int err = 0; struct heap_info *h = NULL; if (!dma_is_coherent_dev(dev)) return 0; h = dev_get_drvdata(dev); BUG_ON(!h); if (!h) return 1; mutex_lock(&h->resize_lock); if (!dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) { if ((uintptr_t)base < h->curr_base || len > h->curr_len || (uintptr_t)base - h->curr_base > h->curr_len - len) { BUG(); mutex_unlock(&h->resize_lock); return 1; } idx = div_u64((uintptr_t)base - h->cma_base, h->cma_chunk_size); dev_dbg(&h->dev, "req free addr (%p) size (0x%zx) idx (%d)\n", base, len, idx); } attrs |= DMA_ATTR_ALLOC_EXACT_SIZE; err = dma_release_from_coherent_dev(&h->dev, len, base, attrs); /* err = 0 on failure, !0 on successful release */ if (err && h->task) mod_timer(&h->shrink_timer, jiffies + h->shrink_interval); if (err) h->curr_used -= len; mutex_unlock(&h->resize_lock); if (err && !h->task) shrink_resizable_heap(h); return err; } static bool shrink_chunk_locked(struct heap_info *h, int idx) { size_t chunk_size; int resize_err; void *ret = NULL; dma_addr_t dev_base; unsigned long attrs = DMA_ATTR_ALLOC_EXACT_SIZE; /* check if entire chunk is free */ chunk_size = (idx == h->num_chunks - 1) ? h->rem_chunk_size : h->cma_chunk_size; resize_err = dma_alloc_from_coherent_dev_at(&h->dev, chunk_size, &dev_base, &ret, attrs, idx * h->cma_chunk_size >> PAGE_SHIFT); if (!resize_err) { goto out; } else if (dev_base != h->cma_base + idx * h->cma_chunk_size) { resize_err = dma_release_from_coherent_dev( &h->dev, chunk_size, (void *)(uintptr_t)dev_base, attrs); BUG_ON(!resize_err); goto out; } else { dev_dbg(&h->dev, "prep to remove chunk b=%pa, s=0x%zx\n", &dev_base, chunk_size); resize_err = dma_release_from_coherent_dev( &h->dev, chunk_size, (void *)(uintptr_t)dev_base, attrs); BUG_ON(!resize_err); if (!resize_err) { dev_err(&h->dev, "failed to rel mem\n"); goto out; } /* Handle VPR configuration updates */ if (h->update_resize_cfg) { phys_addr_t new_base = h->curr_base; size_t new_len = h->curr_len - chunk_size; if (h->curr_base == dev_base) new_base += chunk_size; dev_dbg(&h->dev, "update vpr base to %pa, size=%zx\n", &new_base, new_len); resize_err = h->update_resize_cfg(new_base, new_len); if (resize_err) { dev_err(&h->dev, "update resize failed\n"); goto out; } } if (h->curr_base == dev_base) h->curr_base += chunk_size; h->curr_len -= chunk_size; update_alloc_range(h); release_from_contiguous_heap(h, dev_base, chunk_size); dev_dbg(&h->dev, "removed chunk b=%pa, s=0x%zx" " new heap b=%pa, s=0x%zx\n", &dev_base, chunk_size, &h->curr_base, h->curr_len); return true; } out: return false; } static void shrink_resizable_heap(struct heap_info *h) { bool unlock = false; int first_alloc_idx, last_alloc_idx; check_next_chunk: if (unlock) { mutex_unlock(&h->resize_lock); cond_resched(); } mutex_lock(&h->resize_lock); unlock = true; if (h->curr_len <= h->floor_size) goto out_unlock; get_first_and_last_idx(h, &first_alloc_idx, &last_alloc_idx); /* All chunks are free. Exit. */ if (first_alloc_idx == -1) goto out_unlock; if (shrink_chunk_locked(h, first_alloc_idx)) goto check_next_chunk; /* Only one chunk is in use. */ if (first_alloc_idx == last_alloc_idx) goto out_unlock; if (shrink_chunk_locked(h, last_alloc_idx)) goto check_next_chunk; out_unlock: mutex_unlock(&h->resize_lock); } /* * Helper function used to manage resizable heap shrink timeouts */ static void shrink_timeout(unsigned long __data) { struct task_struct *p = (struct task_struct *) __data; wake_up_process(p); } static int shrink_thread(void *arg) { struct heap_info *h = arg; /* * Set up an interval timer which can be used to trigger a commit wakeup * after the commit interval expires */ setup_timer(&h->shrink_timer, shrink_timeout, (unsigned long)current); h->task = current; while (1) { if (kthread_should_stop()) break; shrink_resizable_heap(h); /* resize done. goto sleep */ set_current_state(TASK_INTERRUPTIBLE); schedule(); } return 0; } void dma_release_declared_memory(struct device *dev) { struct dma_coherent_mem *mem = dev->dma_mem; if (!mem) return; dev->dma_mem = NULL; if (!(mem->flags & DMA_MEMORY_NOMAP)) iounmap(mem->virt_base); kfree(mem->bitmap); kfree(mem); } EXPORT_SYMBOL(dma_release_declared_memory); void *dma_mark_declared_memory_occupied(struct device *dev, dma_addr_t device_addr, size_t size, unsigned long attrs) { struct dma_coherent_mem *mem = dev->dma_mem; int order = get_order(size); int pos, freepage; unsigned int count; unsigned long align = 0; size += device_addr & ~PAGE_MASK; if (!mem) return ERR_PTR(-EINVAL); if (!(DMA_ATTR_ALLOC_EXACT_SIZE & attrs)) { if (order > DMA_BUF_ALIGNMENT) align = (1 << DMA_BUF_ALIGNMENT) - 1; else align = (1 << order) - 1; } if (DMA_ATTR_ALLOC_EXACT_SIZE & attrs) count = PAGE_ALIGN(size) >> PAGE_SHIFT; else count = 1 << order; pos = (device_addr - mem->device_base) >> PAGE_SHIFT; freepage = bitmap_find_next_zero_area(mem->bitmap, mem->size, pos, count, align); if (pos >= mem->size || freepage != pos) return ERR_PTR(-EBUSY); bitmap_set(mem->bitmap, pos, count); return mem->virt_base + (pos << PAGE_SHIFT); } EXPORT_SYMBOL(dma_mark_declared_memory_occupied); void dma_mark_declared_memory_unoccupied(struct device *dev, dma_addr_t device_addr, size_t size, unsigned long attrs) { struct dma_coherent_mem *mem = dev->dma_mem; int pos; int count; if (!mem) return; size += device_addr & ~PAGE_MASK; if (DMA_ATTR_ALLOC_EXACT_SIZE & attrs) count = PAGE_ALIGN(size) >> PAGE_SHIFT; else count = 1 << get_order(size); pos = (device_addr - mem->device_base) >> PAGE_SHIFT; bitmap_clear(mem->bitmap, pos, count); } EXPORT_SYMBOL(dma_mark_declared_memory_unoccupied); /** * dma_alloc_from_coherent_attr() - try to allocate memory from the per-device * coherent area * * @dev: device from which we allocate memory * @size: size of requested memory area * @dma_handle: This will be filled with the correct dma handle * @ret: This pointer will be filled with the virtual address * to allocated area. * @attrs: DMA Attribute * This function should be only called from per-arch dma_alloc_coherent() * to support allocation from per-device coherent memory pools. * * Returns 0 if dma_alloc_coherent_attr should continue with allocating from * generic memory areas, or !0 if dma_alloc_coherent should return @ret. */ int dma_alloc_from_coherent_attr(struct device *dev, ssize_t size, dma_addr_t *dma_handle, void **ret, unsigned long attrs) { if (!dev) return 0; if (dev->dma_mem) return dma_alloc_from_coherent_dev(dev, size, dma_handle, ret, attrs); else return dma_alloc_from_coherent_heap_dev(dev, size, dma_handle, ret, attrs); } EXPORT_SYMBOL(dma_alloc_from_coherent_attr); /** * dma_release_from_coherent_attr() - try to free the memory allocated from * per-device coherent memory pool * @dev: device from which the memory was allocated * @size: size of the memory area to free * @vaddr: virtual address of allocated pages * @attrs: DMA Attribute * * This checks whether the memory was allocated from the per-device * coherent memory pool and if so, releases that memory. * * Returns 1 if we correctly released the memory, or 0 if * dma_release_coherent_attr() should proceed with releasing memory from * generic pools. */ int dma_release_from_coherent_attr(struct device *dev, size_t size, void *vaddr, unsigned long attrs, dma_addr_t dma_handle) { if (!dev) return 0; if (!vaddr) /* * The only possible valid case where vaddr is NULL is when * dma_alloc_attrs() is called on coherent dev which was * initialized with DMA_MEMORY_NOMAP. */ vaddr = (void *)dma_handle; if (dev->dma_mem) return dma_release_from_coherent_dev(dev, size, vaddr, attrs); else return dma_release_from_coherent_heap_dev(dev, size, vaddr, attrs); } EXPORT_SYMBOL(dma_release_from_coherent_attr); /** * dma_mmap_from_coherent() - try to mmap the memory allocated from * per-device coherent memory pool to userspace * @dev: device from which the memory was allocated * @vma: vm_area for the userspace memory * @vaddr: cpu address returned by dma_alloc_from_coherent * @size: size of the memory buffer allocated by dma_alloc_from_coherent * @ret: result from remap_pfn_range() * * This checks whether the memory was allocated from the per-device * coherent memory pool and if so, maps that memory to the provided vma. * * Returns 1 if we correctly mapped the memory, or 0 if the caller should * proceed with mapping memory from generic pools. */ int dma_mmap_from_coherent(struct device *dev, struct vm_area_struct *vma, void *vaddr, size_t size, int *ret) { struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL; void *mem_addr; if (!mem) return 0; if (mem->flags & DMA_MEMORY_NOMAP) mem_addr = (void *)(uintptr_t)mem->device_base; else mem_addr = mem->virt_base; if (mem && vaddr >= mem_addr && vaddr + size <= (mem_addr + (mem->size << PAGE_SHIFT))) { unsigned long off = vma->vm_pgoff; int start = (vaddr - mem_addr) >> PAGE_SHIFT; int user_count = vma_pages(vma); int count = PAGE_ALIGN(size) >> PAGE_SHIFT; *ret = -ENXIO; if (off < count && user_count <= count - off) { unsigned long pfn = mem->pfn_base + start + off; *ret = remap_pfn_range(vma, vma->vm_start, pfn, user_count << PAGE_SHIFT, vma->vm_page_prot); } return 1; } return 0; } EXPORT_SYMBOL(dma_mmap_from_coherent); /* * Support for reserved memory regions defined in device tree */ #ifdef CONFIG_OF_RESERVED_MEM #include #include #include static int rmem_dma_device_init(struct reserved_mem *rmem, struct device *dev) { struct dma_coherent_mem *mem = rmem->priv; if (!mem && !dma_init_coherent_memory(rmem->base, rmem->base, rmem->size, DMA_MEMORY_MAP | DMA_MEMORY_EXCLUSIVE, PAGE_SHIFT, &mem)) { pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n", &rmem->base, (unsigned long)rmem->size / SZ_1M); return -ENODEV; } rmem->priv = mem; dma_assign_coherent_memory(dev, mem); return 0; } static void rmem_dma_device_release(struct reserved_mem *rmem, struct device *dev) { dev->dma_mem = NULL; } static const struct reserved_mem_ops rmem_dma_ops = { .device_init = rmem_dma_device_init, .device_release = rmem_dma_device_release, }; static int __init rmem_dma_setup(struct reserved_mem *rmem) { unsigned long node = rmem->fdt_node; if (of_get_flat_dt_prop(node, "reusable", NULL)) return -EINVAL; #ifdef CONFIG_ARM if (!of_get_flat_dt_prop(node, "no-map", NULL)) { pr_err("Reserved memory: regions without no-map are not yet supported\n"); return -EINVAL; } #endif rmem->ops = &rmem_dma_ops; pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\n", &rmem->base, (unsigned long)rmem->size / SZ_1M); return 0; } RESERVEDMEM_OF_DECLARE(dma, "shared-dma-pool", rmem_dma_setup); #endif int dma_get_coherent_stats(struct device *dev, struct dma_coherent_stats *stats) { struct heap_info *h = NULL; struct dma_coherent_mem *mem = dev->dma_mem; if ((!dev) || !stats) return -EINVAL; h = dev_get_drvdata(dev); if (h && (h->magic == RESIZE_MAGIC)) { stats->size = h->curr_len; stats->base = h->curr_base; stats->used = h->curr_used; stats->max = h->cma_len; goto out; } if (!mem) return -EINVAL; stats->size = mem->size << PAGE_SHIFT; stats->base = mem->device_base; out: return 0; } EXPORT_SYMBOL(dma_get_coherent_stats);