276 lines
6.3 KiB
C
276 lines
6.3 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Copyright 2018-2020 Broadcom.
|
|
*/
|
|
#include <linux/dma-mapping.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/pagemap.h>
|
|
#include <linux/pgtable.h>
|
|
#include <linux/vmalloc.h>
|
|
|
|
#include <asm/page.h>
|
|
#include <asm/unaligned.h>
|
|
|
|
#include <uapi/linux/misc/bcm_vk.h>
|
|
|
|
#include "bcm_vk.h"
|
|
#include "bcm_vk_msg.h"
|
|
#include "bcm_vk_sg.h"
|
|
|
|
/*
|
|
* Valkyrie has a hardware limitation of 16M transfer size.
|
|
* So limit the SGL chunks to 16M.
|
|
*/
|
|
#define BCM_VK_MAX_SGL_CHUNK SZ_16M
|
|
|
|
static int bcm_vk_dma_alloc(struct device *dev,
|
|
struct bcm_vk_dma *dma,
|
|
int dir,
|
|
struct _vk_data *vkdata);
|
|
static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma);
|
|
|
|
/* Uncomment to dump SGLIST */
|
|
/* #define BCM_VK_DUMP_SGLIST */
|
|
|
|
static int bcm_vk_dma_alloc(struct device *dev,
|
|
struct bcm_vk_dma *dma,
|
|
int direction,
|
|
struct _vk_data *vkdata)
|
|
{
|
|
dma_addr_t addr, sg_addr;
|
|
int err;
|
|
int i;
|
|
int offset;
|
|
u32 size;
|
|
u32 remaining_size;
|
|
u32 transfer_size;
|
|
u64 data;
|
|
unsigned long first, last;
|
|
struct _vk_data *sgdata;
|
|
|
|
/* Get 64-bit user address */
|
|
data = get_unaligned(&vkdata->address);
|
|
|
|
/* offset into first page */
|
|
offset = offset_in_page(data);
|
|
|
|
/* Calculate number of pages */
|
|
first = (data & PAGE_MASK) >> PAGE_SHIFT;
|
|
last = ((data + vkdata->size - 1) & PAGE_MASK) >> PAGE_SHIFT;
|
|
dma->nr_pages = last - first + 1;
|
|
|
|
/* Allocate DMA pages */
|
|
dma->pages = kmalloc_array(dma->nr_pages,
|
|
sizeof(struct page *),
|
|
GFP_KERNEL);
|
|
if (!dma->pages)
|
|
return -ENOMEM;
|
|
|
|
dev_dbg(dev, "Alloc DMA Pages [0x%llx+0x%x => %d pages]\n",
|
|
data, vkdata->size, dma->nr_pages);
|
|
|
|
dma->direction = direction;
|
|
|
|
/* Get user pages into memory */
|
|
err = get_user_pages_fast(data & PAGE_MASK,
|
|
dma->nr_pages,
|
|
direction == DMA_FROM_DEVICE,
|
|
dma->pages);
|
|
if (err != dma->nr_pages) {
|
|
dma->nr_pages = (err >= 0) ? err : 0;
|
|
dev_err(dev, "get_user_pages_fast, err=%d [%d]\n",
|
|
err, dma->nr_pages);
|
|
return err < 0 ? err : -EINVAL;
|
|
}
|
|
|
|
/* Max size of sg list is 1 per mapped page + fields at start */
|
|
dma->sglen = (dma->nr_pages * sizeof(*sgdata)) +
|
|
(sizeof(u32) * SGLIST_VKDATA_START);
|
|
|
|
/* Allocate sglist */
|
|
dma->sglist = dma_alloc_coherent(dev,
|
|
dma->sglen,
|
|
&dma->handle,
|
|
GFP_KERNEL);
|
|
if (!dma->sglist)
|
|
return -ENOMEM;
|
|
|
|
dma->sglist[SGLIST_NUM_SG] = 0;
|
|
dma->sglist[SGLIST_TOTALSIZE] = vkdata->size;
|
|
remaining_size = vkdata->size;
|
|
sgdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];
|
|
|
|
/* Map all pages into DMA */
|
|
size = min_t(size_t, PAGE_SIZE - offset, remaining_size);
|
|
remaining_size -= size;
|
|
sg_addr = dma_map_page(dev,
|
|
dma->pages[0],
|
|
offset,
|
|
size,
|
|
dma->direction);
|
|
transfer_size = size;
|
|
if (unlikely(dma_mapping_error(dev, sg_addr))) {
|
|
__free_page(dma->pages[0]);
|
|
return -EIO;
|
|
}
|
|
|
|
for (i = 1; i < dma->nr_pages; i++) {
|
|
size = min_t(size_t, PAGE_SIZE, remaining_size);
|
|
remaining_size -= size;
|
|
addr = dma_map_page(dev,
|
|
dma->pages[i],
|
|
0,
|
|
size,
|
|
dma->direction);
|
|
if (unlikely(dma_mapping_error(dev, addr))) {
|
|
__free_page(dma->pages[i]);
|
|
return -EIO;
|
|
}
|
|
|
|
/*
|
|
* Compress SG list entry when pages are contiguous
|
|
* and transfer size less or equal to BCM_VK_MAX_SGL_CHUNK
|
|
*/
|
|
if ((addr == (sg_addr + transfer_size)) &&
|
|
((transfer_size + size) <= BCM_VK_MAX_SGL_CHUNK)) {
|
|
/* pages are contiguous, add to same sg entry */
|
|
transfer_size += size;
|
|
} else {
|
|
/* pages are not contiguous, write sg entry */
|
|
sgdata->size = transfer_size;
|
|
put_unaligned(sg_addr, (u64 *)&sgdata->address);
|
|
dma->sglist[SGLIST_NUM_SG]++;
|
|
|
|
/* start new sg entry */
|
|
sgdata++;
|
|
sg_addr = addr;
|
|
transfer_size = size;
|
|
}
|
|
}
|
|
/* Write last sg list entry */
|
|
sgdata->size = transfer_size;
|
|
put_unaligned(sg_addr, (u64 *)&sgdata->address);
|
|
dma->sglist[SGLIST_NUM_SG]++;
|
|
|
|
/* Update pointers and size field to point to sglist */
|
|
put_unaligned((u64)dma->handle, &vkdata->address);
|
|
vkdata->size = (dma->sglist[SGLIST_NUM_SG] * sizeof(*sgdata)) +
|
|
(sizeof(u32) * SGLIST_VKDATA_START);
|
|
|
|
#ifdef BCM_VK_DUMP_SGLIST
|
|
dev_dbg(dev,
|
|
"sgl 0x%llx handle 0x%llx, sglen: 0x%x sgsize: 0x%x\n",
|
|
(u64)dma->sglist,
|
|
dma->handle,
|
|
dma->sglen,
|
|
vkdata->size);
|
|
for (i = 0; i < vkdata->size / sizeof(u32); i++)
|
|
dev_dbg(dev, "i:0x%x 0x%x\n", i, dma->sglist[i]);
|
|
#endif
|
|
|
|
return 0;
|
|
}
|
|
|
|
int bcm_vk_sg_alloc(struct device *dev,
|
|
struct bcm_vk_dma *dma,
|
|
int dir,
|
|
struct _vk_data *vkdata,
|
|
int num)
|
|
{
|
|
int i;
|
|
int rc = -EINVAL;
|
|
|
|
/* Convert user addresses to DMA SG List */
|
|
for (i = 0; i < num; i++) {
|
|
if (vkdata[i].size && vkdata[i].address) {
|
|
/*
|
|
* If both size and address are non-zero
|
|
* then DMA alloc.
|
|
*/
|
|
rc = bcm_vk_dma_alloc(dev,
|
|
&dma[i],
|
|
dir,
|
|
&vkdata[i]);
|
|
} else if (vkdata[i].size ||
|
|
vkdata[i].address) {
|
|
/*
|
|
* If one of size and address are zero
|
|
* there is a problem.
|
|
*/
|
|
dev_err(dev,
|
|
"Invalid vkdata %x 0x%x 0x%llx\n",
|
|
i, vkdata[i].size, vkdata[i].address);
|
|
rc = -EINVAL;
|
|
} else {
|
|
/*
|
|
* If size and address are both zero
|
|
* don't convert, but return success.
|
|
*/
|
|
rc = 0;
|
|
}
|
|
|
|
if (rc)
|
|
goto fail_alloc;
|
|
}
|
|
return rc;
|
|
|
|
fail_alloc:
|
|
while (i > 0) {
|
|
i--;
|
|
if (dma[i].sglist)
|
|
bcm_vk_dma_free(dev, &dma[i]);
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma)
|
|
{
|
|
dma_addr_t addr;
|
|
int i;
|
|
int num_sg;
|
|
u32 size;
|
|
struct _vk_data *vkdata;
|
|
|
|
dev_dbg(dev, "free sglist=%p sglen=0x%x\n", dma->sglist, dma->sglen);
|
|
|
|
/* Unmap all pages in the sglist */
|
|
num_sg = dma->sglist[SGLIST_NUM_SG];
|
|
vkdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];
|
|
for (i = 0; i < num_sg; i++) {
|
|
size = vkdata[i].size;
|
|
addr = get_unaligned(&vkdata[i].address);
|
|
|
|
dma_unmap_page(dev, addr, size, dma->direction);
|
|
}
|
|
|
|
/* Free allocated sglist */
|
|
dma_free_coherent(dev, dma->sglen, dma->sglist, dma->handle);
|
|
|
|
/* Release lock on all pages */
|
|
for (i = 0; i < dma->nr_pages; i++)
|
|
put_page(dma->pages[i]);
|
|
|
|
/* Free allocated dma pages */
|
|
kfree(dma->pages);
|
|
dma->sglist = NULL;
|
|
|
|
return 0;
|
|
}
|
|
|
|
int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num,
|
|
int *proc_cnt)
|
|
{
|
|
int i;
|
|
|
|
*proc_cnt = 0;
|
|
/* Unmap and free all pages and sglists */
|
|
for (i = 0; i < num; i++) {
|
|
if (dma[i].sglist) {
|
|
bcm_vk_dma_free(dev, &dma[i]);
|
|
*proc_cnt += 1;
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|