tegrakernel/kernel/nvidia/drivers/video/tegra/host/pva/pva_queue.c

/*
 * PVA Task Management
 *
 * Copyright (c) 2016-2019, NVIDIA Corporation.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/delay.h>
#include <asm/ioctls.h>
#include <linux/fs.h>
#include <linux/uaccess.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/nvhost.h>
#include <linux/cvnas.h>

#include <trace/events/nvhost.h>

#ifdef CONFIG_EVENTLIB
#include <linux/keventlib.h>
#include <uapi/linux/nvdev_fence.h>
#include <uapi/linux/nvhost_events.h>
#endif


#include <uapi/linux/nvhost_pva_ioctl.h>

#include "nvhost_syncpt_unit_interface.h"
#include "../drivers/staging/android/sync.h"
#include "pva.h"
#include "pva-task.h"
#include "nvhost_buffer.h"
#include "nvhost_queue.h"
#include "pva_mailbox.h"
#include "pva_ccq.h"
#include "pva_queue.h"
#include "dev.h"
#include "hw_cfg_pva.h"
#include "t194/hardware_t194.h"
#include "pva-vpu-perf.h"

#include <trace/events/nvhost_pva.h>

#define ACTION_LIST_FENCE_SIZE 13
#define ACTION_LIST_STATUS_OPERATION_SIZE 11
#define ACTION_LIST_TERMINATION_SIZE 1
#define ACTION_LIST_STATS_SIZE 9

/*
 * The worst-case input action buffer size:
 * - Prefences trigger a word memory operation (size 13 bytes)
 * - Input status reads trigger a half-word memory operation (size 11 bytes)
 * - The action list is terminated by a null action (1 byte)
 */
#define INPUT_ACTION_BUFFER_SIZE ( \
	ALIGN((PVA_MAX_PREFENCES + 10) * ACTION_LIST_FENCE_SIZE + \
	      PVA_MAX_INPUT_STATUS * ACTION_LIST_STATUS_OPERATION_SIZE  + \
	      ACTION_LIST_TERMINATION_SIZE, 256))

/*
 * The worst-case output action buffer size:
 * - Postfences trigger a word memory operation (size 13 bytes)
 * - Output status write triggers a half-word memory operation (size 11 bytes)
 * - Output action list includes a operation for stats purpose (size 9 bytes)
 * - Output action list includes syncpoint and semaphore increments
 * - The action list is terminated by a null action (1 byte)
 */
#define OUTPUT_ACTION_BUFFER_SIZE ( \
	ALIGN((PVA_MAX_POSTFENCES + 10) * ACTION_LIST_FENCE_SIZE + \
	      PVA_MAX_OUTPUT_STATUS * ACTION_LIST_STATUS_OPERATION_SIZE  + \
	      ACTION_LIST_STATS_SIZE * 2 + \
	      ACTION_LIST_FENCE_SIZE  * 2 + \
	      ACTION_LIST_TERMINATION_SIZE, 256))

struct pva_hw_task {
	struct pva_task task;
	struct pva_action_list preaction_list;
	struct pva_action_list postaction_list;
	struct pva_task_parameter_array input_parameter_array[PVA_PARAM_LAST];
	struct pva_task_parameter_array output_parameter_array[PVA_PARAM_LAST];
	u8 preactions[INPUT_ACTION_BUFFER_SIZE];
	u8 postactions[OUTPUT_ACTION_BUFFER_SIZE];
	struct pva_task_parameter_desc input_surface_desc;
	struct pva_task_surface input_surfaces[PVA_MAX_INPUT_SURFACES];
	struct pva_task_parameter_desc output_surface_desc;
	struct pva_task_surface output_surfaces[PVA_MAX_OUTPUT_SURFACES];
	struct pva_task_statistics statistics;
	struct pva_task_vpu_perf_counter
		vpu_perf_counters[PVA_TASK_VPU_NUM_PERF_COUNTERS];
	u8 opaque_data[PVA_MAX_PRIMARY_PAYLOAD_SIZE];
};

static void pva_task_dump(struct pva_submit_task *task)
{
	int i;

	nvhost_dbg_info("task=%p, "
			"input_scalars=(handle=%u, offset=%x), "
			"input_surfaces=%p, "
			"output_scalars=(handle=%u, offset=%u), "
			"output_surfaces=%p, "
			"primary_payload=%p (size=%u)",
			task,
			task->input_scalars.handle, task->input_scalars.offset,
			task->input_surfaces,
			task->output_scalars.handle, task->output_scalars.offset,
			task->output_surfaces,
			task->primary_payload, task->primary_payload_size);

	for (i = 0; i < task->num_prefences; i++)
		nvhost_dbg_info("prefence %d: type=%u, "
				"syncpoint_index=%u, syncpoint_value=%u, "
				"sync_fd=%u, semaphore_handle=%u, "
				"semaphore_offset=%u, semaphore_value=%u", i,
				task->prefences[i].type,
				task->prefences[i].syncpoint_index,
				task->prefences[i].syncpoint_value,
				task->prefences[i].sync_fd,
				task->prefences[i].semaphore_handle,
				task->prefences[i].semaphore_offset,
				task->prefences[i].semaphore_value);

	for (i = 0; i < PVA_MAX_FENCE_TYPES; i++) {
		int j;

		for (j = 0; j < task->num_pvafences[i]; j++) {
			nvhost_dbg_info("pvafence %d: type=%u, "
				"syncpoint_index=%u, syncpoint_value=%u, "
				"sync_fd=%u, semaphore_handle=%u, "
				"semaphore_offset=%u, semaphore_value=%u", i,
				task->pvafences[i][j].fence.type,
				task->pvafences[i][j].fence.syncpoint_index,
				task->pvafences[i][j].fence.syncpoint_value,
				task->pvafences[i][j].fence.sync_fd,
				task->pvafences[i][j].fence.semaphore_handle,
				task->pvafences[i][j].fence.semaphore_offset,
				task->pvafences[i][j].fence.semaphore_value);
		}
	}

	for (i = 0; i < task->num_input_surfaces; i++)
		nvhost_dbg_info("input surface %d: format=%llu, "
				"surface_handle=%u, surface_offset=%u, "
				"roi_handle=%u, roi_offset=%u, surface_stride=%u, "
				"line_stride=%u, depth=%u, width=%u, height=%u, "
				"layout=%u", i,
				task->input_surfaces[i].format,
				task->input_surfaces[i].surface_handle,
				task->input_surfaces[i].surface_offset,
				task->input_surfaces[i].roi_handle,
				task->input_surfaces[i].roi_offset,
				task->input_surfaces[i].surface_stride,
				task->input_surfaces[i].line_stride,
				task->input_surfaces[i].depth,
				task->input_surfaces[i].width,
				task->input_surfaces[i].height,
				task->input_surfaces[i].layout);

	for (i = 0; i < task->num_output_surfaces; i++)
		nvhost_dbg_info("output surface %d: format=%llu, "
				"surface_handle=%u, surface_offset=%u, "
				"roi_handle=%u, roi_offset=%u, surface_stride=%u,"
				"line_stride=%u, depth=%u, width=%u, height=%u, "
				"layout=%u", i,
				task->output_surfaces[i].format,
				task->output_surfaces[i].surface_handle,
				task->output_surfaces[i].surface_offset,
				task->output_surfaces[i].roi_handle,
				task->output_surfaces[i].roi_offset,
				task->output_surfaces[i].surface_stride,
				task->output_surfaces[i].line_stride,
				task->output_surfaces[i].depth,
				task->output_surfaces[i].width,
				task->output_surfaces[i].height,
				task->output_surfaces[i].layout);

	for (i = 0; i < task->num_pointers; i++)
		nvhost_dbg_info("pointer %d: handle=%u, offset=%u",
				i, task->pointers[i].handle,
				task->pointers[i].offset);

	for (i = 0; i < task->num_input_task_status; i++)
		nvhost_dbg_info("input task status %d: handle=%u, offset=%u",
				i, task->input_task_status[i].handle,
				task->input_task_status[i].offset);

	for (i = 0; i < task->num_output_task_status; i++)
		nvhost_dbg_info("output task status %d: handle=%u, offset=%u",
				i, task->output_task_status[i].handle,
				task->output_task_status[i].offset);
}

static void pva_task_get_memsize(size_t *dma_size, size_t *kmem_size)
{
	/* Align task addr to 64bytes boundary for DMA use*/
	*dma_size = ALIGN(sizeof(struct pva_hw_task) + 64, 64);
	*kmem_size = sizeof(struct pva_submit_task);
}

static void pva_task_unpin_mem(struct pva_submit_task *task)
{
	int i;
	int j;

#define UNPIN_MEMORY(dst_name)						\
	do {								\
		if ((((dst_name).dmabuf) != NULL) &&			\
				((dst_name).dma_addr != 0)) {		\
			nvhost_buffer_submit_unpin(task->buffers,	\
				&((dst_name).dmabuf), 1);		\
			dma_buf_put((dst_name).dmabuf);			\
		}							\
	} while (0)

	for (i = 0; i < task->num_input_surfaces; i++) {
		UNPIN_MEMORY(task->input_surfaces_ext[i]);
		UNPIN_MEMORY(task->input_surface_rois_ext[i]);
	}


	for (i = 0; i < task->num_output_surfaces; i++) {
		UNPIN_MEMORY(task->output_surfaces_ext[i]);
		UNPIN_MEMORY(task->output_surface_rois_ext[i]);
	}

	for (i = 0; i < task->num_prefences; i++) {
		if ((task->prefences[i].type == NVDEV_FENCE_TYPE_SEMAPHORE)
			&& task->prefences[i].semaphore_handle)
			UNPIN_MEMORY(task->prefences_sema_ext[i]);
	}

	for (i = 0; i < PVA_MAX_FENCE_TYPES; i++) {
		for (j = 0; j < task->num_pvafences[i]; j++) {
			struct nvpva_fence *fence = &task->pvafences[i][j];

			if ((fence->fence.type == NVDEV_FENCE_TYPE_SEMAPHORE)
			    && fence->fence.semaphore_handle) {
				UNPIN_MEMORY(task->pvafences_sema_ext[i][j]);
			}
		}
		for (j = 0; j < task->num_pva_ts_buffers[i]; j++) {
			struct nvpva_fence *fence = &task->pvafences[i][j];

			if (fence->ts_buf_ptr.handle) {
				UNPIN_MEMORY(task->pva_ts_buffers_ext[i][j]);
			}
		}
	}

	for (i = 0; i < task->num_input_task_status; i++) {
		if (task->input_task_status[i].handle) {
			UNPIN_MEMORY(task->input_task_status_ext[i]);
		}
	}

	for (i = 0; i < task->num_output_task_status; i++) {
		if (task->output_task_status[i].handle) {
			UNPIN_MEMORY(task->output_task_status_ext[i]);
		}
	}

	for (i = 0; i < task->num_pointers; i++) {
		if (task->pointers[i].handle) {
			UNPIN_MEMORY(task->pointers_ext[i]);
		}
	}

	UNPIN_MEMORY(task->input_scalars_ext);
	UNPIN_MEMORY(task->output_scalars_ext);

#undef UNPIN_MEMORY
}

static int pva_task_pin_mem(struct pva_submit_task *task)
{
	u32 cvsram_base = nvcvnas_get_cvsram_base();
	u32 cvsram_sz = nvcvnas_get_cvsram_size();
	int err;
	int i;
	int j;

#define PIN_MEMORY(dst_name, dmabuf_fd)					\
	do {								\
		if (!(dmabuf_fd)) {					\
			err = -EFAULT;					\
			goto err_map_handle;				\
		}							\
									\
		((dst_name).dmabuf) = dma_buf_get(dmabuf_fd);		\
		if (IS_ERR_OR_NULL((dst_name).dmabuf)) {		\
			(dst_name).dmabuf = NULL;			\
			err = -EFAULT;					\
			goto err_map_handle;				\
		}							\
									\
		err = nvhost_buffer_submit_pin(task->buffers,		\
				&(dst_name).dmabuf, 1,			\
				&(dst_name).dma_addr,			\
				&(dst_name).size,			\
				&(dst_name).heap);			\
		if (err < 0)						\
			goto err_map_handle;				\
	} while (0)

	/* Pin input surfaces */
	for (i = 0; i < task->num_input_surfaces; i++) {
		/* HACK: nvmap doesn't support CVNAS yet */
		if (task->input_surfaces[i].surface_handle == 0) {
			u32 offset = task->input_surfaces[i].surface_offset;

			if (offset > cvsram_sz) {
				err = -EINVAL;
				goto err_map_handle;
			}

			task->input_surfaces_ext[i].dma_addr = cvsram_base;
			task->input_surfaces_ext[i].size = cvsram_sz - offset;
			task->input_surfaces_ext[i].heap =
				NVHOST_BUFFERS_HEAP_CVNAS;
		} else {
			PIN_MEMORY(task->input_surfaces_ext[i],
				task->input_surfaces[i].surface_handle);
		}

		if (task->input_surfaces[i].roi_handle)
			PIN_MEMORY(task->input_surface_rois_ext[i],
				task->input_surfaces[i].roi_handle);
	}

	/* ...and then output surfaces */
	for (i = 0; i < task->num_output_surfaces; i++) {
		if (task->output_surfaces[i].surface_handle == 0) {
			/* HACK: To support the MISR test
			 * Kernel is not suppose to convert the address being
			 * passed from the UMD. So setting dma_addr as  the
			 * offset passed from KMD and size to 4MB
			 */
			u32 offset = task->output_surfaces[i].surface_offset;

			/* Only root is allowed to use offsets */
			if (current_uid().val != 0) {
				err = -EINVAL;
				goto err_map_handle;
			}

			task->output_surfaces_ext[i].dma_addr = offset;
			task->output_surfaces_ext[i].size =  0x400000;
		} else {
			PIN_MEMORY(task->output_surfaces_ext[i],
				task->output_surfaces[i].surface_handle);
		}

		if (task->output_surfaces[i].roi_handle)
			PIN_MEMORY(task->output_surface_rois_ext[i],
				task->output_surfaces[i].roi_handle);
	}

	/* check fence semaphore_type before memory pin */
	for (i = 0; i < task->num_prefences; i++) {
		if ((task->prefences[i].type == NVDEV_FENCE_TYPE_SEMAPHORE)
			&& task->prefences[i].semaphore_handle) {
			PIN_MEMORY(task->prefences_sema_ext[i],
				task->prefences[i].semaphore_handle);
		}
	}

	/* check the generalized fence structures */
	for (i = 0; i < PVA_MAX_FENCE_TYPES; i++) {
		for (j = 0; j < task->num_pvafences[i]; j++) {
			struct nvpva_fence *fence = &task->pvafences[i][j];

			if ((fence->fence.type == NVDEV_FENCE_TYPE_SEMAPHORE)
			    && fence->fence.semaphore_handle) {
				PIN_MEMORY(task->pvafences_sema_ext[i][j],
					   fence->fence.semaphore_handle);
			}
		}
		for (j = 0; j < task->num_pva_ts_buffers[i]; j++) {
			struct nvpva_fence *fence = &task->pvafences[i][j];

			if (fence->ts_buf_ptr.handle) {
				PIN_MEMORY(task->pva_ts_buffers_ext[i][j],
					   fence->ts_buf_ptr.handle);
			}
		}
	}

	/* Pin the input and output action status */
	for (i = 0; i < task->num_input_task_status; i++) {
		if (task->input_task_status[i].handle) {
			PIN_MEMORY(task->input_task_status_ext[i],
				task->input_task_status[i].handle);
		}
	}

	for (i = 0; i < task->num_output_task_status; i++) {
		if (task->output_task_status[i].handle) {
			PIN_MEMORY(task->output_task_status_ext[i],
				task->output_task_status[i].handle);
		}
	}

	/* Pin task pointers */
	for (i = 0; i < task->num_pointers; i++) {
		if (task->pointers[i].handle) {
			PIN_MEMORY(task->pointers_ext[i],
				   task->pointers[i].handle);
		}
	}

	/* Pin rest */
	if (task->input_scalars.handle)
		PIN_MEMORY(task->input_scalars_ext,
			task->input_scalars.handle);

	if (task->output_scalars.handle)
		PIN_MEMORY(task->output_scalars_ext,
			task->output_scalars.handle);

#undef PIN_MEMORY

	return 0;

err_map_handle:
	pva_task_unpin_mem(task);
	return err;
}

static void pva_task_write_surfaces(struct pva_task_surface *hw_surface,
		struct pva_surface *surface,
		struct pva_parameter_ext *surface_ext,
		struct pva_parameter_ext *roi_ext,
		unsigned int count)
{
	int i;

	for (i = 0; i < count; i++) {
		hw_surface[i].address = surface_ext[i].dma_addr +
			surface[i].surface_offset;
		hw_surface[i].surface_size = surface_ext[i].size;
		hw_surface[i].roi_addr = roi_ext[i].dma_addr +
			surface[i].roi_offset;
		hw_surface[i].roi_size = roi_ext[i].size;
		hw_surface[i].format = surface[i].format;
		hw_surface[i].width = surface[i].width;
		hw_surface[i].height = surface[i].height;
		hw_surface[i].line_stride = surface[i].line_stride;
		hw_surface[i].plane_stride = surface[i].surface_stride;
		hw_surface[i].num_planes = surface[i].depth;
		hw_surface[i].layout = surface[i].layout;
		hw_surface[i].block_height_log2 = surface[i].block_height_log2;

		/* Set bit 39 for block linear surfaces in the address field.
		*  This bit is used for indicating that memory subsystem should
		*  convert the block linear format into common block linear format
		*  that is used by other engines in Tegra. Thebit in itself is
		*  dropped before making the address translation in SMMU.
		*/
		if (surface[i].layout == PVA_TASK_SURFACE_LAYOUT_BLOCK_LINEAR)
			hw_surface[i].address |= PVA_BIT64(39);

		hw_surface[i].memory = surface_ext[i].heap;
	}
}

static inline int pva_task_write_atomic_op(u8 *base, u8 action)
{
	*base = action;

	return 1;
}

static inline int
pva_task_write_struct_ptr_op(u8 *base, u8 action, u64 addr, u16 val)
{
	int i = 0;

	base[i++] = action;
	base[i++] = (u8)((addr >> 0) & 0xff);
	base[i++] = (u8)((addr >> 8) & 0xff);
	base[i++] = (u8)((addr >> 16) & 0xff);
	base[i++] = (u8)((addr >> 24) & 0xff);
	base[i++] = (u8)((addr >> 32) & 0xff);
	base[i++] = (u8)((addr >> 40) & 0xff);
	base[i++] = (u8)((addr >> 48) & 0xff);
	base[i++] = (u8)((addr >> 56) & 0xff);

	return i;
}

static inline int pva_task_write_ptr_16b_op(u8 *base, u8 action, u64 addr, u16 val)
{
	int i = 0;

	base[i++] = action;
	base[i++] = (u8)((addr >> 0) & 0xff);
	base[i++] = (u8)((addr >> 8) & 0xff);
	base[i++] = (u8)((addr >> 16) & 0xff);
	base[i++] = (u8)((addr >> 24) & 0xff);
	base[i++] = (u8)((addr >> 32) & 0xff);
	base[i++] = (u8)((addr >> 40) & 0xff);
	base[i++] = (u8)((addr >> 48) & 0xff);
	base[i++] = (u8)((addr >> 56) & 0xff);
	base[i++] = (u8)((val >> 0) & 0xff);
	base[i++] = (u8)((val >> 8) & 0xff);

	return i;
}

static inline int pva_task_write_ptr_op(u8 *base, u8 action, u64 addr, u32 val)
{
	int i = 0;

	base[i++] = action;
	base[i++] = (u8)((addr >> 0) & 0xff);
	base[i++] = (u8)((addr >> 8) & 0xff);
	base[i++] = (u8)((addr >> 16) & 0xff);
	base[i++] = (u8)((addr >> 24) & 0xff);
	base[i++] = (u8)((addr >> 32) & 0xff);
	base[i++] = (u8)((addr >> 40) & 0xff);
	base[i++] = (u8)((addr >> 48) & 0xff);
	base[i++] = (u8)((addr >> 56) & 0xff);
	base[i++] = (u8)((val >> 0) & 0xff);
	base[i++] = (u8)((val >> 8) & 0xff);
	base[i++] = (u8)((val >> 16) & 0xff);
	base[i++] = (u8)((val >> 24) & 0xff);

	return i;
}

static int pva_task_write_preactions(struct pva_submit_task *task,
				     struct pva_hw_task *hw_task)
{
	struct platform_device *host1x_pdev =
		to_platform_device(task->pva->pdev->dev.parent);
	u8 *hw_preactions = hw_task->preactions;
	int i = 0, j = 0, ptr = 0;
	u8 action_ts;
	u8 action_f;
	u32 increment;

	/* Add waits to preactions list */
	for (i = 0; i < task->num_prefences; i++) {
		struct nvdev_fence *fence = task->prefences + i;

		switch (fence->type) {
		case NVDEV_FENCE_TYPE_SYNCPT: {
			dma_addr_t syncpt_addr = nvhost_syncpt_gos_address(
							task->pva->pdev,
							fence->syncpoint_index);
			if (!syncpt_addr)
				syncpt_addr = nvhost_syncpt_address(
							task->queue->vm_pdev,
							fence->syncpoint_index);

			ptr += pva_task_write_ptr_op(&hw_preactions[ptr],
				TASK_ACT_PTR_BLK_GTREQL, syncpt_addr,
				fence->syncpoint_value);
			break;
		}
		case NVDEV_FENCE_TYPE_SEMAPHORE:
		case NVDEV_FENCE_TYPE_SEMAPHORE_TS:{
			ptr += pva_task_write_ptr_op(&hw_preactions[ptr],
				TASK_ACT_PTR_BLK_GTREQL,
				task->prefences_sema_ext[i].dma_addr  +
					fence->semaphore_offset,
				fence->semaphore_value);
			break;
		}
		case NVDEV_FENCE_TYPE_SYNC_FD: {
			int thresh, id;
			dma_addr_t syncpt_addr;
			struct sync_fence *syncfd_fence;
			struct sync_pt *pt;
			struct nvhost_master *host = nvhost_get_host(
							task->pva->pdev);
			struct nvhost_syncpt *sp = &host->syncpt;

			if (!fence->sync_fd)
				break;

			syncfd_fence = nvhost_sync_fdget(fence->sync_fd);
			if (!syncfd_fence)
				break;

			for (j = 0; j < syncfd_fence->num_fences; j++) {
				pt = sync_pt_from_fence(
					syncfd_fence->cbs[j].sync_pt);
				if (!pt)
					break;

				id = nvhost_sync_pt_id(pt);
				thresh = nvhost_sync_pt_thresh(pt);

				/* validate the synpt ids */
				if (!id ||
				!nvhost_syncpt_is_valid_hw_pt(sp, id)) {
					sync_fence_put(syncfd_fence);
					break;
				}

				if (nvhost_syncpt_is_expired(sp,
							id, thresh))
					continue;

				syncpt_addr = nvhost_syncpt_gos_address(
							task->pva->pdev, id);
				if (!syncpt_addr)
					syncpt_addr = nvhost_syncpt_address(
							task->queue->vm_pdev, id);

				ptr += pva_task_write_ptr_op(
						&hw_preactions[ptr],
						TASK_ACT_PTR_BLK_GTREQL,
						syncpt_addr, thresh);

			}
			break;
		}
		default:
			return -ENOSYS;
		}
	}

	for (i = 0; i < PVA_MAX_FENCE_TYPES; i++) {
		increment = 0;
		switch (i) {
		case PVA_FENCE_SOT_V:
			action_ts = TASK_ACT_PTR_WRITE_SOT_V_TS;
			action_f = TASK_ACT_PTR_WRITE_VAL_SOT_V;
			increment = 1;
			break;
		case PVA_FENCE_SOT_R:
			action_ts = TASK_ACT_PTR_WRITE_SOT_R_TS;
			action_f = TASK_ACT_PTR_WRITE_VAL_SOT_R;
			increment = 1;
			break;
		default:
			action_ts = 0;
			action_f = 0;
			break;
		};
		if ((action_ts == 0) || (task->num_pvafences[i] == 0))
			continue;
		for (j = 0; j < task->num_pva_ts_buffers[i]; j++) {
			if (task->pvafences[i][j].ts_buf_ptr.handle) {
				int dif;

				dif = pva_task_write_ptr_op(
				       &hw_preactions[ptr],
				       action_ts,
				       task->pva_ts_buffers_ext[i][j].dma_addr +
				       task->pvafences[i][j].ts_buf_ptr.offset,
				       1U);
				ptr += dif;
			}
		}
		for (j = 0; j < task->num_pvafences[i]; j++) {
			struct nvdev_fence *fence =
				&task->pvafences[i][j].fence;
			u32 thresh;

			switch (fence->type) {
			case NVDEV_FENCE_TYPE_SYNCPT: {
				dma_addr_t syncpt_gos_addr =
					nvhost_syncpt_gos_address(
						task->pva->pdev,
						fence->syncpoint_index);
				dma_addr_t syncpt_addr =
					nvhost_syncpt_address(
						task->queue->vm_pdev,
						task->queue->syncpt_id);

				ptr += pva_task_write_ptr_op(
					&hw_preactions[ptr],
					action_f,
					syncpt_addr,
					1U);
				task->fence_num += increment;
				/* Make a syncpoint increment */
				if (syncpt_gos_addr) {
					thresh = nvhost_syncpt_read_maxval(
						host1x_pdev,
						task->queue->syncpt_id) +
						task->fence_num;
					ptr += pva_task_write_ptr_op(
						&hw_preactions[ptr],
						TASK_ACT_PTR_WRITE_VAL,
						syncpt_gos_addr, thresh);
				}
				break;
			}
			case NVDEV_FENCE_TYPE_SEMAPHORE:
			case NVDEV_FENCE_TYPE_SEMAPHORE_TS: {
				int dif;

				dif = pva_task_write_ptr_op(&hw_preactions[ptr],
				       action_f,
				       task->pvafences_sema_ext[i][j].dma_addr +
				       fence->semaphore_offset,
				       fence->semaphore_value);
				ptr += dif;
				break;
			}
			case NVDEV_FENCE_TYPE_SYNC_FD:
				/* TODO XXX*/
			default:
				return -ENOSYS;
			}
		}
	}

	/* Perform input status checks */
	for (i = 0; i < task->num_input_task_status; i++) {
		struct pva_status_handle *input_status =
					task->input_task_status + i;
		dma_addr_t input_status_addr =
				task->input_task_status_ext[i].dma_addr  +
				input_status->offset;

		ptr += pva_task_write_ptr_16b_op(
					&hw_preactions[ptr],
					TASK_ACT_READ_STATUS,
					input_status_addr, 0);
	}

	ptr += pva_task_write_atomic_op(&hw_preactions[ptr],
		TASK_ACT_TERMINATE);

	/* Store the preaction list */
	hw_task->preaction_list.offset = offsetof(struct pva_hw_task, preactions);
	hw_task->preaction_list.length = ptr;

	nvhost_dbg_info("preaction buffer alloted size %d: used size %d",
				INPUT_ACTION_BUFFER_SIZE, ptr);
	return 0;
}

static void pva_task_write_postactions(struct pva_submit_task *task,
				       struct pva_hw_task *hw_task)
{
	dma_addr_t syncpt_addr = nvhost_syncpt_address(task->queue->vm_pdev,
				task->queue->syncpt_id);
	dma_addr_t syncpt_gos_addr = nvhost_syncpt_gos_address(task->pva->pdev,
				task->queue->syncpt_id);
	u8 *hw_postactions = hw_task->postactions;
	int ptr = 0, i = 0;
	struct platform_device *host1x_pdev =
			to_platform_device(task->pva->pdev->dev.parent);
	dma_addr_t output_status_addr;
	u32 thresh;
	u8 action_ts;
	int j;

	/* Write Output action status */
	for (i = 0; i < task->num_output_task_status; i++) {
		struct pva_status_handle *output_status =
					task->output_task_status + i;

		output_status_addr = task->output_task_status_ext[i].dma_addr +
				     output_status->offset;
		ptr += pva_task_write_ptr_16b_op(
					&hw_postactions[ptr],
					TASK_ACT_WRITE_STATUS,
					output_status_addr, 1);
	}

	for (i = 0; i < PVA_MAX_FENCE_TYPES; i++) {
		switch (i) {
		case PVA_FENCE_EOT_V:
			action_ts = TASK_ACT_PTR_WRITE_EOT_V_TS;
			break;
		case PVA_FENCE_EOT_R:
			action_ts = TASK_ACT_PTR_WRITE_EOT_R_TS;
			break;
		case PVA_FENCE_POST:
			action_ts = TASK_ACT_PTR_WRITE_TS;
			break;
		default:
			action_ts = 0;
			break;
		};
		if (action_ts == 0)
			continue;
		for (j = 0; j < task->num_pva_ts_buffers[i]; j++) {
			if (task->pvafences[i][j].ts_buf_ptr.handle) {
				int dif;

				dif = pva_task_write_ptr_op(
				       &hw_postactions[ptr],
				       action_ts,
				       task->pva_ts_buffers_ext[i][j].dma_addr +
				       task->pvafences[i][j].ts_buf_ptr.offset,
				       1U);

				ptr += dif;
			}
		}
	}

	/* Add postactions list for semaphore */
	j = PVA_FENCE_POST;
	for (i = 0; i < task->num_pvafences[j]; i++) {
		struct nvdev_fence *fence = &task->pvafences[j][i].fence;

		if (fence->type == NVDEV_FENCE_TYPE_SEMAPHORE) {
			ptr += pva_task_write_ptr_op(&hw_postactions[ptr],
				TASK_ACT_PTR_WRITE_VAL,
				task->pvafences_sema_ext[j][i].dma_addr +
				fence->semaphore_offset,
				fence->semaphore_value);
		} else if (fence->type == NVDEV_FENCE_TYPE_SEMAPHORE_TS) {
			/*
			* Timestamp will be filled by ucode hence making the
			* place holder for timestamp size, sizeof(u64).
			*/
			ptr += sizeof(u64) +
			       pva_task_write_ptr_op(
				&hw_postactions[ptr],
				TASK_ACT_PTR_WRITE_VAL_TS,
				task->pvafences_sema_ext[j][i].dma_addr +
				fence->semaphore_offset,
				fence->semaphore_value);
		}
	}

	task->fence_num += 1;

	/* Make a syncpoint increment */
	if (syncpt_gos_addr) {
		thresh = nvhost_syncpt_read_maxval(
			host1x_pdev,
			task->queue->syncpt_id) + task->fence_num;
		ptr += pva_task_write_ptr_op(
			&hw_postactions[ptr],
			TASK_ACT_PTR_WRITE_VAL,
			syncpt_gos_addr, thresh);
	}

	ptr += pva_task_write_ptr_op(&hw_postactions[ptr],
		TASK_ACT_PTR_WRITE_VAL, syncpt_addr, 1);


	output_status_addr = task->dma_addr +
			     offsetof(struct pva_hw_task, statistics);
	ptr += pva_task_write_struct_ptr_op(
				&hw_postactions[ptr],
				TASK_ACT_PVA_STATISTICS,
				output_status_addr, 1);

	if (task->pva->vpu_perf_counters_enable) {
		ptr += pva_task_write_struct_ptr_op(
				&hw_postactions[ptr],
				TASK_ACT_PVA_VPU_PERF_COUNTERS,
				task->dma_addr + offsetof(struct pva_hw_task,
					vpu_perf_counters),
				1);
	}

	ptr += pva_task_write_atomic_op(&hw_postactions[ptr],
		TASK_ACT_TERMINATE);

	/* Store the postaction list */
	hw_task->postaction_list.offset = offsetof(struct pva_hw_task,
						   postactions);
	hw_task->postaction_list.length = ptr;

	nvhost_dbg_info("postaction buffer alloted size %d: used size %d",
				OUTPUT_ACTION_BUFFER_SIZE, ptr);

}

static void pva_task_write_output_surfaces(struct pva_submit_task *task,
					   struct pva_hw_task *hw_task)
{
	struct pva_task_parameter_array *surface_parameter;

	if (task->num_output_surfaces == 0)
		return;

	surface_parameter = hw_task->output_parameter_array +
			    hw_task->task.num_output_parameters;

	/* Write parameter descriptor */
	surface_parameter->address = task->dma_addr +
				     offsetof(struct pva_hw_task,
					      output_surface_desc);
	surface_parameter->type = PVA_PARAM_SURFACE_LIST;
	surface_parameter->size = sizeof(struct pva_task_parameter_desc) +
				  sizeof(struct pva_task_surface) *
				  task->num_output_surfaces;
	hw_task->task.num_output_parameters++;

	/* Write the surface descriptor base information */
	hw_task->output_surface_desc.num_parameters = task->num_output_surfaces;
	hw_task->output_surface_desc.reserved = 0;

	/* Write the output surfaces */
	pva_task_write_surfaces(hw_task->output_surfaces,
			task->output_surfaces,
			task->output_surfaces_ext,
			task->output_surface_rois_ext,
			task->num_output_surfaces);
}

static void pva_task_write_input_surfaces(struct pva_submit_task *task,
					  struct pva_hw_task *hw_task)
{
	struct pva_task_parameter_array *surface_parameter;

	if (task->num_input_surfaces == 0)
		return;

	surface_parameter = hw_task->input_parameter_array +
			    hw_task->task.num_input_parameters;

	/* Write parameter descriptor */
	surface_parameter->address = task->dma_addr +
				     offsetof(struct pva_hw_task,
					      input_surface_desc);
	surface_parameter->type = PVA_PARAM_SURFACE_LIST;
	surface_parameter->size = sizeof(struct pva_task_parameter_desc) +
				  sizeof(struct pva_task_surface) *
				  task->num_input_surfaces;
	hw_task->task.num_input_parameters++;

	/* Write the surface descriptor base information */
	hw_task->input_surface_desc.num_parameters = task->num_input_surfaces;
	hw_task->input_surface_desc.reserved = 0;

	/* Write the input surfaces */
	pva_task_write_surfaces(hw_task->input_surfaces,
			task->input_surfaces,
			task->input_surfaces_ext,
			task->input_surface_rois_ext,
			task->num_input_surfaces);
}

static void pva_task_write_non_surfaces(struct pva_submit_task *task,
					struct pva_hw_task *hw_task)
{
	struct pva_task_parameter_array *hw_input_parameters =
			hw_task->input_parameter_array;
	struct pva_task_parameter_array *hw_output_parameters =
			hw_task->output_parameter_array;

#define COPY_PARAMETER(target, name, name_ext, param_type, count)	\
	do {								\
		if ((name).handle) {					\
			target[(count)].address = (name_ext).dma_addr + \
						  (name).offset;	\
			target[(count)].size = (name_ext).size -	\
					       (name).offset;		\
			target[(count)].type = (param_type);		\
			(count)++;					\
		}							\
	} while (0)

	COPY_PARAMETER(hw_input_parameters, task->input_scalars,
		       task->input_scalars_ext,
		       PVA_PARAM_SCALAR_LIST,
		       hw_task->task.num_input_parameters);
	COPY_PARAMETER(hw_output_parameters, task->output_scalars,
		       task->output_scalars_ext,
		       PVA_PARAM_SCALAR_LIST,
		       hw_task->task.num_output_parameters);
#undef COPY_PARAMETER
}

static int pva_task_write_opaque_data(struct pva_submit_task *task,
				      struct pva_hw_task *hw_task)
{
	struct pva_task_parameter_array *opaque_parameter;
	struct pva_task_opaque_data_desc *opaque_desc;
	struct pva_parameter_ext *handle_ext;
	unsigned int primary_payload_offset;
	struct pva_memory_handle *handle;
	unsigned int pointer_list_offset;
	struct pva_task_pointer pointer;
	u8 *primary_payload, *pointers;
	unsigned int num_bytes;
	u64 aux, size, flags;
	unsigned int i;

	if (task->num_pointers == 0 && task->primary_payload_size == 0)
		return 0;

	/* Calculate size of the opaque data */
	num_bytes = sizeof(struct pva_task_opaque_data_desc);
	num_bytes += task->primary_payload_size;
	num_bytes += sizeof(struct pva_task_pointer) * task->num_pointers;

	if (num_bytes > PVA_MAX_PRIMARY_PAYLOAD_SIZE)
		return -ENOMEM;

	/* Opaque parameter resides always in the input parameter block */
	opaque_parameter = hw_task->input_parameter_array +
			   hw_task->task.num_input_parameters;

	/* Write parameter descriptor */
	opaque_parameter->address = task->dma_addr +
				     offsetof(struct pva_hw_task,
					      opaque_data);
	opaque_parameter->type = PVA_PARAM_OPAQUE_DATA;
	opaque_parameter->size = num_bytes;
	hw_task->task.num_input_parameters++;

	/* Determine offset to the primary_payload start */
	primary_payload_offset = sizeof(struct pva_task_opaque_data_desc);
	primary_payload = hw_task->opaque_data + primary_payload_offset;

	/* Determine offset to the start of the pointer list */
	pointer_list_offset = primary_payload_offset +
		task->primary_payload_size;
	pointers = hw_task->opaque_data + pointer_list_offset;

	/* Initialize the opaque data descriptor */
	opaque_desc = (void *)hw_task->opaque_data;
	opaque_desc->primary_payload_size = task->primary_payload_size;

	/* Copy the primary_payload */
	memcpy(primary_payload,
	       task->primary_payload,
	       task->primary_payload_size);

	/* Copy the pointers */
	for (i = 0; i < task->num_pointers; i++) {
		handle = task->pointers + i;
		handle_ext = task->pointers_ext + i;

		size = handle_ext->size & PVA_TASK_POINTER_AUX_SIZE_MASK;
		if (size != handle_ext->size) {
			return -EINVAL;
		}

		flags = 0;
		if (handle_ext->heap == NVHOST_BUFFERS_HEAP_CVNAS)
			flags |= PVA_TASK_POINTER_AUX_FLAGS_CVNAS;

		aux = (size << PVA_TASK_POINTER_AUX_SIZE_SHIFT) |
		      (flags << PVA_TASK_POINTER_AUX_FLAGS_SHIFT);

		pointer.address = handle_ext->dma_addr + handle->offset;
		pointer.aux = aux;

		/* The data might be unaligned. copy it byte-by-byte */
		memcpy(pointers, &pointer, sizeof(pointer));
		pointers += sizeof(pointer);
	}

	return 0;
}

static int pva_task_write(struct pva_submit_task *task, bool atomic)
{
	struct pva_hw_task *hw_task;
	int err;
	int i;

	/* Task start from the memory base */
	hw_task = task->va;

	/* Write the preaction list */
	err = pva_task_write_preactions(task, hw_task);
	if (err < 0)
		return err;

	/* Write the postaction list */
	pva_task_write_postactions(task, hw_task);

	/* Initialize parameters */
	pva_task_write_non_surfaces(task, hw_task);

	/* Write the pointers and the primary payload */
	err = pva_task_write_opaque_data(task, hw_task);
	if (err < 0)
		return err;

	/* Write input surfaces */
	pva_task_write_input_surfaces(task, hw_task);

	/* Write output surfaces */
	pva_task_write_output_surfaces(task, hw_task);

	hw_task->task.input_parameters = offsetof(struct pva_hw_task,
						  input_parameter_array);
	hw_task->task.output_parameters = offsetof(struct pva_hw_task,
						  output_parameter_array);
	hw_task->task.gen_task.versionid = TASK_VERSION_ID;
	hw_task->task.gen_task.engineid = PVA_ENGINE_ID;
	hw_task->task.gen_task.sequence = 0;
	hw_task->task.gen_task.length = offsetof(struct pva_hw_task,
						 input_surface_desc);
	hw_task->task.gen_task.n_preaction_lists = 1;
	hw_task->task.gen_task.preaction_lists_p = offsetof(struct pva_hw_task,
							    preaction_list);
	hw_task->task.gen_task.n_postaction_lists = 1;
	hw_task->task.gen_task.postaction_lists_p = offsetof(struct pva_hw_task,
							     postaction_list);
	hw_task->task.runlist_version = PVA_TASK_VERSION_ID;
	hw_task->task.queue_id = task->queue->id;
	hw_task->task.flags = atomic ? PVA_TASK_FL_ATOMIC : 0;
	hw_task->task.operation = task->operation;
	hw_task->task.timeout = task->timeout;

	/* Set flags to debug the vpu application if debugfs node is set for vpu id */
	if (task->pva->dbg_vpu_app_id == task->operation)
			hw_task->task.flags |= PVA_TASK_FL_VPU_DEBUG;

	/* This should be delivered from userspace - hard-code
	 * until the mechanism is in place.
	 */
	hw_task->task.operation_version = 1;

	for (i = 0; i < roundup(sizeof(struct pva_hw_task), 16) / 16; i++) {
		u8 *task_va = task->va;
		u32 base = i * 16;

		nvhost_dbg_info("%02x, %02x, %02x, %02x, %02x, %02x, %02x %02x, "
				"%02x, %02x, %02x, %02x, %02x, %02x, %02x %02x",
				task_va[base],
				task_va[base + 1],
				task_va[base + 2],
				task_va[base + 3],
				task_va[base + 4],
				task_va[base + 5],
				task_va[base + 6],
				task_va[base + 7],
				task_va[base + 8],
				task_va[base + 9],
				task_va[base + 10],
				task_va[base + 11],
				task_va[base + 12],
				task_va[base + 13],
				task_va[base + 14],
				task_va[base + 15]);
	}

	return 0;
}

#ifdef CONFIG_EVENTLIB
static void
pva_eventlib_record_perf_counter(struct platform_device *pdev,
				 u32 syncpt_id,
				 u32 syncpt_thresh,
				 u32 operation,
				 u32 tag,
				 u32 count,
				 u32 sum,
				 u64 sum_squared,
				 u32 min,
				 u32 max,
				 u64 timestamp_begin,
				 u64 timestamp_end)
{
	struct nvhost_device_data *pdata = platform_get_drvdata(pdev);
	struct nvhost_vpu_perf_counter perf_counter;

	if (!pdata->eventlib_id)
		return;

	perf_counter.class_id = pdata->class;
	perf_counter.syncpt_id = syncpt_id;
	perf_counter.syncpt_thresh = syncpt_thresh;
	perf_counter.operation = operation;
	perf_counter.tag = tag;
	perf_counter.count = count;
	perf_counter.average = sum / count;
	perf_counter.variance =
		((u64)count * sum_squared - (u64)sum * (u64)sum)
			/ (u64)count / (u64)count;
	perf_counter.minimum = min;
	perf_counter.maximum = max;

	keventlib_write(pdata->eventlib_id,
			&perf_counter,
			sizeof(perf_counter),
			NVHOST_VPU_PERF_COUNTER_BEGIN,
			timestamp_begin);

	keventlib_write(pdata->eventlib_id,
			&perf_counter,
			sizeof(perf_counter),
			NVHOST_VPU_PERF_COUNTER_END,
			timestamp_end);
}
static void
pva_eventlib_record_r5_states(struct platform_device *pdev,
			      u32 syncpt_id,
			      u32 syncpt_thresh,
			      struct pva_task_statistics *stats,
			      u32 operation)
{
	struct nvhost_device_data *pdata = platform_get_drvdata(pdev);
	struct nvhost_pva_task_state state;

	if (!pdata->eventlib_id)
		return;

	state.class_id = pdata->class;
	state.syncpt_id = syncpt_id;
	state.syncpt_thresh = syncpt_thresh;
	state.operation = operation;

	keventlib_write(pdata->eventlib_id,
			&state,
			sizeof(state),
			NVHOST_PVA_QUEUE_BEGIN,
			stats->queued_time);

	keventlib_write(pdata->eventlib_id,
			&state,
			sizeof(state),
			NVHOST_PVA_QUEUE_END,
			stats->vpu_assigned_time);

	keventlib_write(pdata->eventlib_id,
			&state,
			sizeof(state),
			NVHOST_PVA_PREPARE_BEGIN,
			stats->vpu_assigned_time);

	keventlib_write(pdata->eventlib_id,
			&state,
			sizeof(state),
			NVHOST_PVA_PREPARE_END,
			stats->vpu_start_time);

	keventlib_write(pdata->eventlib_id,
			&state,
			sizeof(state),
			stats->vpu_assigned == 0 ? NVHOST_PVA_VPU0_BEGIN
						 : NVHOST_PVA_VPU1_BEGIN,
			stats->vpu_start_time);

	keventlib_write(pdata->eventlib_id,
			&state,
			sizeof(state),
			stats->vpu_assigned == 0 ? NVHOST_PVA_VPU0_END
						 : NVHOST_PVA_VPU1_END,
			stats->vpu_complete_time);

	keventlib_write(pdata->eventlib_id,
			&state,
			sizeof(state),
			NVHOST_PVA_POST_BEGIN,
			stats->vpu_complete_time);

	keventlib_write(pdata->eventlib_id,
			&state,
			sizeof(state),
			NVHOST_PVA_POST_END,
			stats->complete_time);
}
#else
static void
pva_eventlib_record_perf_counter(struct platform_device *pdev,
				 u32 syncpt_id,
				 u32 syncpt_thresh,
				 u32 operation,
				 u32 tag,
				 u32 count,
				 u32 sum,
				 u64 sum_squared,
				 u32 min,
				 u32 max,
				 u64 timestamp_begin,
				 u64 timestamp_end)
{
}
static void
pva_eventlib_record_r5_states(struct platform_device *pdev,
			      struct pva_task_statistics *stats,
			      u32 operation)
{
}
#endif

static void pva_task_update(struct pva_submit_task *task)
{
	struct nvhost_queue *queue = task->queue;
	struct pva_hw_task *hw_task = task->va;
	struct pva *pva = task->pva;
	struct platform_device *pdev = pva->pdev;
	struct nvhost_device_data *pdata = platform_get_drvdata(pdev);
	struct pva_task_statistics *stats = &hw_task->statistics;
	struct pva_task_vpu_perf_counter *perf;
	u32 idx;

	trace_nvhost_task_timestamp(dev_name(&pdev->dev),
				    pdata->class,
				    queue->syncpt_id,
				    task->syncpt_thresh,
				    stats->vpu_assigned_time,
				    stats->complete_time);
	nvhost_eventlib_log_task(pdev,
				 queue->syncpt_id,
				 task->syncpt_thresh,
				 stats->vpu_assigned_time,
				 stats->complete_time);
	nvhost_dbg_info("Completed task %p (0x%llx), start_time=%llu, end_time=%llu",
			task, (u64)task->dma_addr,
			stats->vpu_assigned_time,
			stats->complete_time);

	trace_nvhost_pva_task_stats(pdev->name,
			stats->queued_time,
			stats->head_time,
			stats->input_actions_complete,
			stats->vpu_assigned_time,
			stats->vpu_start_time,
			stats->vpu_complete_time,
			stats->complete_time,
			stats->vpu_assigned);

	nvhost_dbg_info("QueuedTime %llu, HeadTime 0x%llu, "
			"InputActionComplete %llu, VpuAssignedTime %llu, "
			"VpuStartTime %llu, VpuCompleteTime %llu, "
			"TaskCompeteTime %llu, AssignedVpu %d",
			stats->queued_time,
			stats->head_time,
			stats->input_actions_complete,
			stats->vpu_assigned_time,
			stats->vpu_start_time,
			stats->vpu_complete_time,
			stats->complete_time,
			stats->vpu_assigned);

	pva_eventlib_record_r5_states(pdev,
			queue->syncpt_id,
			task->syncpt_thresh,
			stats, task->operation);

	/* Record task postfences */
	nvhost_eventlib_log_fences(pdev,
			queue->syncpt_id,
			task->syncpt_thresh,
			&(task->pvafences[PVA_FENCE_POST][0].fence),
			1,
			NVDEV_FENCE_KIND_POST,
			stats->complete_time);

	if (task->pva->vpu_perf_counters_enable) {
		for (idx = 0; idx < PVA_TASK_VPU_NUM_PERF_COUNTERS; idx++) {
			perf = &hw_task->vpu_perf_counters[idx];
			if (perf->count != 0) {
				trace_nvhost_pva_task_vpu_perf(
					pdev->name, idx, perf->count,
					perf->sum, perf->sum_squared,
					perf->min, perf->max);
				pva_eventlib_record_perf_counter(
					pdev, queue->syncpt_id,
					task->syncpt_thresh,
					task->operation, idx, perf->count,
					perf->sum, perf->sum_squared,
					perf->min, perf->max,
					stats->vpu_assigned_time,
					stats->complete_time);
			}
		}
	}

	/* Unpin job memory. PVA shouldn't be using it anymore */
	pva_task_unpin_mem(task);

	/* Drop PM runtime reference of PVA */
	nvhost_module_idle(task->pva->pdev);

	/* remove the task from the queue */
	list_del(&task->node);

	/* Not linked anymore so drop the reference */
	kref_put(&task->ref, pva_task_free);

	/* Drop queue reference to allow reusing it */
	nvhost_queue_put(queue);
}

void pva_task_free(struct kref *ref)
{
	struct pva_submit_task *task =
		container_of(ref, struct pva_submit_task, ref);
	/* Release memory that was allocated for the task */
	nvhost_queue_free_task_memory(task->queue, task->pool_index);
}

static void pva_queue_update(void *priv, int nr_completed)
{
	struct nvhost_queue *queue = priv;
	struct pva_submit_task *task, *n;
	struct list_head completed;

	INIT_LIST_HEAD(&completed);

	/* Move completed tasks to a separate list */
	mutex_lock(&queue->list_lock);
	list_for_each_entry_safe(task, n, &queue->tasklist, node) {
		if (!nvhost_syncpt_is_expired_ext(queue->pool->pdev,
						  queue->syncpt_id,
						  task->syncpt_thresh))
			break;

		list_move_tail(&task->node, &completed);
	}
	mutex_unlock(&queue->list_lock);

	/* Handle completed tasks */
	list_for_each_entry_safe(task, n, &completed, node)
		pva_task_update(task);
}

static void pva_queue_dump(struct nvhost_queue *queue, struct seq_file *s)
{
	struct pva_submit_task *task;
	int i = 0;
	int k = PVA_FENCE_POST;

	seq_printf(s, "Queue %u, Tasks\n", queue->id);

	mutex_lock(&queue->list_lock);
	list_for_each_entry(task, &queue->tasklist, node) {
		int j;

		seq_printf(s, "    #%u: Operation = %u\n",
				i++, task->operation);

		for (j = 0; j < task->num_prefences; j++)
			seq_printf(s, "    prefence %d: \n\t"
				"syncpoint_index=%u, syncpoint_value=%u\n",
				j,
				task->prefences[j].syncpoint_index,
				task->prefences[j].syncpoint_value);

		for (j = 0; j < task->num_pvafences[k]; j++)
			seq_printf(s, "    postfence %d: \n\t"
				"syncpoint_index=%u, syncpoint_value=%u\n",
				j,
				task->pvafences[k][j].fence.syncpoint_index,
				task->pvafences[k][j].fence.syncpoint_value);


	}
	mutex_unlock(&queue->list_lock);
}

static int pva_task_submit_channel_ccq(struct pva_submit_task *task,
				       u32 *thresh)
{
	struct nvhost_queue *queue = task->queue;
	u64 fifo_flags = PVA_FIFO_INT_ON_ERR;
	u64 fifo_cmd = pva_fifo_submit(queue->id,
				       task->dma_addr,
				       fifo_flags);
	u32 syncpt_wait_ids[PVA_MAX_PREFENCES];
	u32 syncpt_wait_thresh[PVA_MAX_PREFENCES];
	unsigned int i;
	u32 cmdbuf[4];
	int err = 0;

	/* Pick up fences... */
	for (i = 0; i < task->num_prefences; i++) {
		/* ..and ensure that we have only syncpoints present */
		if (task->prefences[i].type != NVDEV_FENCE_TYPE_SYNCPT)
			return -EINVAL;

		/* Put fences into a separate array */
		syncpt_wait_ids[i] =
			task->prefences[i].syncpoint_index;
		syncpt_wait_thresh[i] =
			task->prefences[i].syncpoint_value;
	}

	/* A simple command buffer: Write two words into the ccq
	 * register
	 */
	cmdbuf[0] = nvhost_opcode_setpayload(2);
	cmdbuf[1] = nvhost_opcode_nonincr_w(cfg_ccq_r() >> 2);
	cmdbuf[2] = (u32)(fifo_cmd >> 32);
	cmdbuf[3] = (u32)(fifo_cmd & 0xffffffff);

	/* Submit the command buffer and waits to channel */
	err = nvhost_queue_submit_to_host1x(queue,
					    cmdbuf,
					    ARRAY_SIZE(cmdbuf),
					    1,
					    syncpt_wait_ids,
					    syncpt_wait_thresh,
					    task->num_prefences,
					    thresh);
	return err;
}

static int pva_task_submit_mmio_ccq(struct pva_submit_task *task,
				    u32 *thresh)
{
	struct platform_device *host1x_pdev =
			to_platform_device(task->pva->pdev->dev.parent);
	struct nvhost_queue *queue = task->queue;
	u32 old_maxval, new_maxval;
	u64 fifo_flags = PVA_FIFO_INT_ON_ERR;
	u64 fifo_cmd = pva_fifo_submit(queue->id,
				       task->dma_addr,
				       fifo_flags);
	int err = 0;

	/* Increment syncpoint to capture threshold */
	old_maxval = nvhost_syncpt_read_maxval(host1x_pdev, queue->syncpt_id);
	new_maxval = nvhost_syncpt_incr_max_ext(host1x_pdev,
						queue->syncpt_id,
						task->fence_num);

	err = pva_ccq_send(task->pva, fifo_cmd);
	if (err < 0)
		goto err_submit;

	*thresh = new_maxval;

	return 0;

err_submit:
	nvhost_syncpt_set_maxval(host1x_pdev, queue->syncpt_id, old_maxval);

	return err;
}

static int pva_task_submit_mailbox(struct pva_submit_task *task,
				   u32 *thresh)
{
	struct platform_device *host1x_pdev =
			to_platform_device(task->pva->pdev->dev.parent);
	struct nvhost_queue *queue = task->queue;
	struct pva_mailbox_status_regs status;
	u32 old_maxval, new_maxval;
	struct pva_cmd cmd;
	u32 flags, nregs;
	int err = 0;

	/* Construct submit command */
	flags = PVA_CMD_INT_ON_ERR | PVA_CMD_INT_ON_COMPLETE;
	nregs = pva_cmd_submit(&cmd, queue->id,
				task->dma_addr, flags);

	/* Increment syncpoint to capture threshold */
	old_maxval = nvhost_syncpt_read_maxval(host1x_pdev, queue->syncpt_id);
	new_maxval = nvhost_syncpt_incr_max_ext(host1x_pdev,
						queue->syncpt_id,
						task->fence_num);

	/* Submit request to PVA and wait for response */
	err = pva_mailbox_send_cmd_sync(task->pva, &cmd, nregs, &status);
	if (err < 0) {
		nvhost_warn(&task->pva->pdev->dev,
			"Failed to submit task: %d", err);
		goto err_submit;
	}

	/* Ensure that response is valid */
	if (status.error != PVA_ERR_NO_ERROR) {
		nvhost_warn(&task->pva->pdev->dev, "PVA task rejected: %u",
			status.error);
		err = -EINVAL;
		goto err_submit;
	}

	*thresh = new_maxval;

	return 0;

err_submit:
	nvhost_syncpt_set_maxval(host1x_pdev, queue->syncpt_id, old_maxval);

	return err;
}

static int pva_task_submit(struct pva_submit_task *task,
			   u32 *task_thresh)
{
	struct platform_device *host1x_pdev =
			to_platform_device(task->pva->pdev->dev.parent);
	struct nvhost_queue *queue = task->queue;
	u32 thresh = 0;
	u64 timestamp;
	int err = 0;

	nvhost_dbg_info("Submitting task %p (0x%llx)", task,
			(u64)task->dma_addr);

	/* Get a reference of the queue to avoid it being reused. It
	 * gets freed in the callback...
	 */
	nvhost_queue_get(queue);

	/* Turn on the hardware */
	err = nvhost_module_busy(task->pva->pdev);
	if (err)
		goto err_module_busy;

	/*
	 * TSC timestamp is same as CNTVCT. Task statistics are being
	 * reported in TSC ticks.
	 */
	timestamp = arch_counter_get_cntvct();

	/* Choose the submit policy based on the mode */
	switch (task->pva->submit_mode) {
	case PVA_SUBMIT_MODE_MAILBOX:
		err = pva_task_submit_mailbox(task, &thresh);
		break;

	case PVA_SUBMIT_MODE_MMIO_CCQ:
		err = pva_task_submit_mmio_ccq(task, &thresh);
		break;

	case PVA_SUBMIT_MODE_CHANNEL_CCQ:
		err = pva_task_submit_channel_ccq(task, &thresh);
		break;
	}

	if (err < 0)
		goto err_submit;

	/* Record task prefences */
	nvhost_eventlib_log_fences(task->pva->pdev,
				   queue->syncpt_id,
				   thresh,
				   task->prefences,
				   task->num_prefences,
				   NVDEV_FENCE_KIND_PRE,
				   timestamp);

	nvhost_eventlib_log_submit(task->pva->pdev,
				   queue->syncpt_id,
				   thresh,
				   timestamp);

	task->syncpt_thresh = thresh;

	nvhost_dbg_info("Postfence id=%u, value=%u",
			queue->syncpt_id, thresh);

	*task_thresh = thresh;

	/* Going to be linked so obtain the reference */
	kref_get(&task->ref);

	/*
	 * Tasks in the queue list can be modified by the interrupt handler.
	 * Adding the task into the list must be the last step before
	 * registering the interrupt handler.
	 */
	mutex_lock(&queue->list_lock);
	list_add_tail(&task->node, &queue->tasklist);
	mutex_unlock(&queue->list_lock);

	/*
	 * Register the interrupt handler. This must be done after adding
	 * the tasks into the queue since otherwise we may miss the completion
	 * event.
	 */
	WARN_ON(nvhost_intr_register_notifier(host1x_pdev,
					      queue->syncpt_id, thresh,
					      pva_queue_update, queue));

	return err;

err_submit:
	nvhost_module_idle(task->pva->pdev);
err_module_busy:
	nvhost_queue_put(queue);
	return err;
}

static int pva_queue_submit(struct nvhost_queue *queue, void *args)
{
	struct pva_submit_tasks *task_header = args;
	int err = 0;
	int i;

	for (i = 0; i < task_header->num_tasks; i++) {
		struct pva_submit_task *task = task_header->tasks[i];
		u32 *thresh = &task_header->task_thresh[i];

		task->fence_num = 0;

		/* First, dump the task that we are submitting */
		pva_task_dump(task);

		/* Pin job memory */
		err = pva_task_pin_mem(task);
		if (err < 0)
			break;

		/* Write the task data */
		pva_task_write(task, false);

		err = pva_task_submit(task, thresh);
		if (err < 0)
			break;
	}

	return err;
}

static int pva_queue_set_attribute(struct nvhost_queue *queue, void *args)
{
	uint32_t flags = PVA_CMD_INT_ON_ERR | PVA_CMD_INT_ON_COMPLETE;
	struct pva_queue_set_attribute *set_attr = args;
	struct pva_queue_attribute *attr = set_attr->attr;
	struct pva_mailbox_status_regs status;
	struct pva_cmd cmd;
	int err = 0;
	u32 nregs;

	nregs = pva_cmd_set_queue_attributes(&cmd, queue->id, attr->id,
			attr->value,
			flags);

	/* Submit request to PVA and wait for response */
	if (set_attr->bootup)
		err = pva_mailbox_send_cmd_sync_locked(set_attr->pva,
						       &cmd,
						       nregs,
						       &status);
	else
		err = pva_mailbox_send_cmd_sync(set_attr->pva,
						&cmd,
						nregs,
						&status);
	if (err < 0) {
		nvhost_warn(&set_attr->pva->pdev->dev,
			    "Failed to set attributes: %d\n",
			    err);
		goto end;
	}

	/* Ensure that response is valid */
	if (status.error != PVA_ERR_NO_ERROR) {
		nvhost_warn(&set_attr->pva->pdev->dev,
			    "PVA Q attribute rejected: %u\n",
			    status.error);
		err = -EINVAL;
	}

 end:
	return err;
}

static void pva_queue_cleanup_fence(struct nvdev_fence *fence,
				    struct pva_parameter_ext *fence_ext)
{
	struct dma_buf *dmabuf;
	u8 *dmabuf_cpuva;
	u32 *fence_cpuva;

	if (fence->type != NVDEV_FENCE_TYPE_SEMAPHORE)
		return;

	dmabuf = fence_ext->dmabuf;
	dmabuf_cpuva = dma_buf_vmap(dmabuf);

	if (!dmabuf_cpuva)
		return;

	if (!(fence->semaphore_offset % 4))
		return;

	fence_cpuva = (void *)&dmabuf_cpuva[fence->semaphore_offset];
	*fence_cpuva = fence->semaphore_value;

	dma_buf_vunmap(dmabuf, dmabuf_cpuva);
}

static void pva_queue_cleanup_status(struct pva_status_handle *status_h,
				     struct pva_parameter_ext *status_h_ext)
{
	struct dma_buf *dmabuf = status_h_ext->dmabuf;
	u8 *dmabuf_cpuva = dma_buf_vmap(dmabuf);
	struct nvhost_notification *status_ptr;

	if (!dmabuf_cpuva)
		return;

	status_ptr = (void *)&dmabuf_cpuva[status_h->offset];
	status_ptr->status = 0x8888;

	dma_buf_vunmap(dmabuf, dmabuf_cpuva);
}

static void pva_queue_cleanup(struct nvhost_queue *queue,
			      struct pva_submit_task *task)
{
	struct platform_device *pdev = queue->pool->pdev;
	struct nvhost_master *host = nvhost_get_host(pdev);
	bool expired = nvhost_syncpt_is_expired(&host->syncpt,
						queue->syncpt_id,
						task->syncpt_thresh);
	unsigned int i;
	unsigned int j;

	/*
	 * Ensure that there won't be communication with PVA for
	 * checking the task status
	 */
	task->invalid = true;

	/* Ignore expired fences */
	if (expired)
		return;

	/* Write task status first */
	for (i = 0; i < task->num_output_task_status; i++)
		pva_queue_cleanup_status(task->output_task_status,
					 task->output_task_status_ext);

	/* Finish up non-syncpoint fences */
	for (i = 0; i < PVA_MAX_FENCE_TYPES; i++) {
		for (j = 0; j < task->num_pvafences[i]; j++) {
			pva_queue_cleanup_fence(&task->pvafences[i][j].fence,
					&task->pvafences_sema_ext[i][j]);
		}
	}

	/* Finish syncpoint increments to release waiters */
	nvhost_syncpt_cpu_incr_ext(pdev, queue->syncpt_id);
}

static int pva_queue_abort(struct nvhost_queue *queue)
{
	struct pva_submit_task *task;

	mutex_lock(&queue->list_lock);

	list_for_each_entry(task, &queue->tasklist, node)
		pva_queue_cleanup(queue, task);

	mutex_unlock(&queue->list_lock);

	return 0;
}

struct nvhost_queue_ops pva_queue_ops = {
	.abort = pva_queue_abort,
	.submit = pva_queue_submit,
	.get_task_size = pva_task_get_memsize,
	.dump = pva_queue_dump,
	.set_attribute = pva_queue_set_attribute,
};