/* linux/drivers/dma/pl330.c * * Copyright (C) 2010 Samsung Electronics Co. Ltd. * Jaswinder Singh <jassi.brar@samsung.com> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. */ #include <linux/io.h> #include <linux/init.h> #include <linux/slab.h> #include <linux/module.h> #include <linux/dmaengine.h> #include <linux/interrupt.h> #include <linux/amba/bus.h> #include <linux/amba/pl330.h> #define NR_DEFAULT_DESC 16 enum desc_status { /* In the DMAC pool */ FREE, /* * Allocted to some channel during prep_xxx * Also may be sitting on the work_list. */ PREP, /* * Sitting on the work_list and already submitted * to the PL330 core. Not more than two descriptors * of a channel can be BUSY at any time. */ BUSY, /* * Sitting on the channel work_list but xfer done * by PL330 core */ DONE, }; struct dma_pl330_chan { /* Schedule desc completion */ struct tasklet_struct task; /* DMA-Engine Channel */ struct dma_chan chan; /* Last completed cookie */ dma_cookie_t completed; /* List of to be xfered descriptors */ struct list_head work_list; /* Pointer to the DMAC that manages this channel, * NULL if the channel is available to be acquired. * As the parent, this DMAC also provides descriptors * to the channel. */ struct dma_pl330_dmac *dmac; /* To protect channel manipulation */ spinlock_t lock; /* Token of a hardware channel thread of PL330 DMAC * NULL if the channel is available to be acquired. */ void *pl330_chid; }; struct dma_pl330_dmac { struct pl330_info pif; /* DMA-Engine Device */ struct dma_device ddma; /* Pool of descriptors available for the DMAC's channels */ struct list_head desc_pool; /* To protect desc_pool manipulation */ spinlock_t pool_lock; /* Peripheral channels connected to this DMAC */ struct dma_pl330_chan peripherals[0]; /* keep at end */ }; struct dma_pl330_desc { /* To attach to a queue as child */ struct list_head node; /* Descriptor for the DMA Engine API */ struct dma_async_tx_descriptor txd; /* Xfer for PL330 core */ struct pl330_xfer px; struct pl330_reqcfg rqcfg; struct pl330_req req; enum desc_status status; /* The channel which currently holds this desc */ struct dma_pl330_chan *pchan; }; static inline struct dma_pl330_chan * to_pchan(struct dma_chan *ch) { if (!ch) return NULL; return container_of(ch, struct dma_pl330_chan, chan); } static inline struct dma_pl330_desc * to_desc(struct dma_async_tx_descriptor *tx) { return container_of(tx, struct dma_pl330_desc, txd); } static inline void free_desc_list(struct list_head *list) { struct dma_pl330_dmac *pdmac; struct dma_pl330_desc *desc; struct dma_pl330_chan *pch; unsigned long flags; if (list_empty(list)) return; /* Finish off the work list */ list_for_each_entry(desc, list, node) { dma_async_tx_callback callback; void *param; /* All desc in a list belong to same channel */ pch = desc->pchan; callback = desc->txd.callback; param = desc->txd.callback_param; if (callback) callback(param); desc->pchan = NULL; } pdmac = pch->dmac; spin_lock_irqsave(&pdmac->pool_lock, flags); list_splice_tail_init(list, &pdmac->desc_pool); spin_unlock_irqrestore(&pdmac->pool_lock, flags); } static inline void fill_queue(struct dma_pl330_chan *pch) { struct dma_pl330_desc *desc; int ret; list_for_each_entry(desc, &pch->work_list, node) { /* If already submitted */ if (desc->status == BUSY) break; ret = pl330_submit_req(pch->pl330_chid, &desc->req); if (!ret) { desc->status = BUSY; break; } else if (ret == -EAGAIN) { /* QFull or DMAC Dying */ break; } else { /* Unacceptable request */ desc->status = DONE; dev_err(pch->dmac->pif.dev, "%s:%d Bad Desc(%d)\n", __func__, __LINE__, desc->txd.cookie); tasklet_schedule(&pch->task); } } } static void pl330_tasklet(unsigned long data) { struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data; struct dma_pl330_desc *desc, *_dt; unsigned long flags; LIST_HEAD(list); spin_lock_irqsave(&pch->lock, flags); /* Pick up ripe tomatoes */ list_for_each_entry_safe(desc, _dt, &pch->work_list, node) if (desc->status == DONE) { pch->completed = desc->txd.cookie; list_move_tail(&desc->node, &list); } /* Try to submit a req imm. next to the last completed cookie */ fill_queue(pch); /* Make sure the PL330 Channel thread is active */ pl330_chan_ctrl(pch->pl330_chid, PL330_OP_START); spin_unlock_irqrestore(&pch->lock, flags); free_desc_list(&list); } static void dma_pl330_rqcb(void *token, enum pl330_op_err err) { struct dma_pl330_desc *desc = token; struct dma_pl330_chan *pch = desc->pchan; unsigned long flags; /* If desc aborted */ if (!pch) return; spin_lock_irqsave(&pch->lock, flags); desc->status = DONE; spin_unlock_irqrestore(&pch->lock, flags); tasklet_schedule(&pch->task); } static int pl330_alloc_chan_resources(struct dma_chan *chan) { struct dma_pl330_chan *pch = to_pchan(chan); struct dma_pl330_dmac *pdmac = pch->dmac; unsigned long flags; spin_lock_irqsave(&pch->lock, flags); pch->completed = chan->cookie = 1; pch->pl330_chid = pl330_request_channel(&pdmac->pif); if (!pch->pl330_chid) { spin_unlock_irqrestore(&pch->lock, flags); return 0; } tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch); spin_unlock_irqrestore(&pch->lock, flags); return 1; } static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg) { struct dma_pl330_chan *pch = to_pchan(chan); struct dma_pl330_desc *desc; unsigned long flags; /* Only supports DMA_TERMINATE_ALL */ if (cmd != DMA_TERMINATE_ALL) return -ENXIO; spin_lock_irqsave(&pch->lock, flags); /* FLUSH the PL330 Channel thread */ pl330_chan_ctrl(pch->pl330_chid, PL330_OP_FLUSH); /* Mark all desc done */ list_for_each_entry(desc, &pch->work_list, node) desc->status = DONE; spin_unlock_irqrestore(&pch->lock, flags); pl330_tasklet((unsigned long) pch); return 0; } static void pl330_free_chan_resources(struct dma_chan *chan) { struct dma_pl330_chan *pch = to_pchan(chan); unsigned long flags; spin_lock_irqsave(&pch->lock, flags); tasklet_kill(&pch->task); pl330_release_channel(pch->pl330_chid); pch->pl330_chid = NULL; spin_unlock_irqrestore(&pch->lock, flags); } static enum dma_status pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate) { struct dma_pl330_chan *pch = to_pchan(chan); dma_cookie_t last_done, last_used; int ret; last_done = pch->completed; last_used = chan->cookie; ret = dma_async_is_complete(cookie, last_done, last_used); dma_set_tx_state(txstate, last_done, last_used, 0); return ret; } static void pl330_issue_pending(struct dma_chan *chan) { pl330_tasklet((unsigned long) to_pchan(chan)); } /* * We returned the last one of the circular list of descriptor(s) * from prep_xxx, so the argument to submit corresponds to the last * descriptor of the list. */ static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx) { struct dma_pl330_desc *desc, *last = to_desc(tx); struct dma_pl330_chan *pch = to_pchan(tx->chan); dma_cookie_t cookie; unsigned long flags; spin_lock_irqsave(&pch->lock, flags); /* Assign cookies to all nodes */ cookie = tx->chan->cookie; while (!list_empty(&last->node)) { desc = list_entry(last->node.next, struct dma_pl330_desc, node); if (++cookie < 0) cookie = 1; desc->txd.cookie = cookie; list_move_tail(&desc->node, &pch->work_list); } if (++cookie < 0) cookie = 1; last->txd.cookie = cookie; list_add_tail(&last->node, &pch->work_list); tx->chan->cookie = cookie; spin_unlock_irqrestore(&pch->lock, flags); return cookie; } static inline void _init_desc(struct dma_pl330_desc *desc) { desc->pchan = NULL; desc->req.x = &desc->px; desc->req.token = desc; desc->rqcfg.swap = SWAP_NO; desc->rqcfg.privileged = 0; desc->rqcfg.insnaccess = 0; desc->rqcfg.scctl = SCCTRL0; desc->rqcfg.dcctl = DCCTRL0; desc->req.cfg = &desc->rqcfg; desc->req.xfer_cb = dma_pl330_rqcb; desc->txd.tx_submit = pl330_tx_submit; INIT_LIST_HEAD(&desc->node); } /* Returns the number of descriptors added to the DMAC pool */ int add_desc(struct dma_pl330_dmac *pdmac, gfp_t flg, int count) { struct dma_pl330_desc *desc; unsigned long flags; int i; if (!pdmac) return 0; desc = kmalloc(count * sizeof(*desc), flg); if (!desc) return 0; spin_lock_irqsave(&pdmac->pool_lock, flags); for (i = 0; i < count; i++) { _init_desc(&desc[i]); list_add_tail(&desc[i].node, &pdmac->desc_pool); } spin_unlock_irqrestore(&pdmac->pool_lock, flags); return count; } static struct dma_pl330_desc * pluck_desc(struct dma_pl330_dmac *pdmac) { struct dma_pl330_desc *desc = NULL; unsigned long flags; if (!pdmac) return NULL; spin_lock_irqsave(&pdmac->pool_lock, flags); if (!list_empty(&pdmac->desc_pool)) { desc = list_entry(pdmac->desc_pool.next, struct dma_pl330_desc, node); list_del_init(&desc->node); desc->status = PREP; desc->txd.callback = NULL; } spin_unlock_irqrestore(&pdmac->pool_lock, flags); return desc; } static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch) { struct dma_pl330_dmac *pdmac = pch->dmac; struct dma_pl330_peri *peri = pch->chan.private; struct dma_pl330_desc *desc; /* Pluck one desc from the pool of DMAC */ desc = pluck_desc(pdmac); /* If the DMAC pool is empty, alloc new */ if (!desc) { if (!add_desc(pdmac, GFP_ATOMIC, 1)) return NULL; /* Try again */ desc = pluck_desc(pdmac); if (!desc) { dev_err(pch->dmac->pif.dev, "%s:%d ALERT!\n", __func__, __LINE__); return NULL; } } /* Initialize the descriptor */ desc->pchan = pch; desc->txd.cookie = 0; async_tx_ack(&desc->txd); desc->req.rqtype = peri->rqtype; desc->req.peri = peri->peri_id; dma_async_tx_descriptor_init(&desc->txd, &pch->chan); return desc; } static inline void fill_px(struct pl330_xfer *px, dma_addr_t dst, dma_addr_t src, size_t len) { px->next = NULL; px->bytes = len; px->dst_addr = dst; px->src_addr = src; } static struct dma_pl330_desc * __pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst, dma_addr_t src, size_t len) { struct dma_pl330_desc *desc = pl330_get_desc(pch); if (!desc) { dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n", __func__, __LINE__); return NULL; } /* * Ideally we should lookout for reqs bigger than * those that can be programmed with 256 bytes of * MC buffer, but considering a req size is seldom * going to be word-unaligned and more than 200MB, * we take it easy. * Also, should the limit is reached we'd rather * have the platform increase MC buffer size than * complicating this API driver. */ fill_px(&desc->px, dst, src, len); return desc; } /* Call after fixing burst size */ static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len) { struct dma_pl330_chan *pch = desc->pchan; struct pl330_info *pi = &pch->dmac->pif; int burst_len; burst_len = pi->pcfg.data_bus_width / 8; burst_len *= pi->pcfg.data_buf_dep; burst_len >>= desc->rqcfg.brst_size; /* src/dst_burst_len can't be more than 16 */ if (burst_len > 16) burst_len = 16; while (burst_len > 1) { if (!(len % (burst_len << desc->rqcfg.brst_size))) break; burst_len--; } return burst_len; } static struct dma_async_tx_descriptor * pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, size_t len, unsigned long flags) { struct dma_pl330_desc *desc; struct dma_pl330_chan *pch = to_pchan(chan); struct dma_pl330_peri *peri = chan->private; struct pl330_info *pi; int burst; if (unlikely(!pch || !len || !peri)) return NULL; if (peri->rqtype != MEMTOMEM) return NULL; pi = &pch->dmac->pif; desc = __pl330_prep_dma_memcpy(pch, dst, src, len); if (!desc) return NULL; desc->rqcfg.src_inc = 1; desc->rqcfg.dst_inc = 1; /* Select max possible burst size */ burst = pi->pcfg.data_bus_width / 8; while (burst > 1) { if (!(len % burst)) break; burst /= 2; } desc->rqcfg.brst_size = 0; while (burst != (1 << desc->rqcfg.brst_size)) desc->rqcfg.brst_size++; desc->rqcfg.brst_len = get_burst_len(desc, len); desc->txd.flags = flags; return &desc->txd; } static struct dma_async_tx_descriptor * pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_data_direction direction, unsigned long flg) { struct dma_pl330_desc *first, *desc = NULL; struct dma_pl330_chan *pch = to_pchan(chan); struct dma_pl330_peri *peri = chan->private; struct scatterlist *sg; unsigned long flags; int i, burst_size; dma_addr_t addr; if (unlikely(!pch || !sgl || !sg_len)) return NULL; /* Make sure the direction is consistent */ if ((direction == DMA_TO_DEVICE && peri->rqtype != MEMTODEV) || (direction == DMA_FROM_DEVICE && peri->rqtype != DEVTOMEM)) { dev_err(pch->dmac->pif.dev, "%s:%d Invalid Direction\n", __func__, __LINE__); return NULL; } addr = peri->fifo_addr; burst_size = peri->burst_sz; first = NULL; for_each_sg(sgl, sg, sg_len, i) { desc = pl330_get_desc(pch); if (!desc) { struct dma_pl330_dmac *pdmac = pch->dmac; dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n", __func__, __LINE__); if (!first) return NULL; spin_lock_irqsave(&pdmac->pool_lock, flags); while (!list_empty(&first->node)) { desc = list_entry(first->node.next, struct dma_pl330_desc, node); list_move_tail(&desc->node, &pdmac->desc_pool); } list_move_tail(&first->node, &pdmac->desc_pool); spin_unlock_irqrestore(&pdmac->pool_lock, flags); return NULL; } if (!first) first = desc; else list_add_tail(&desc->node, &first->node); if (direction == DMA_TO_DEVICE) { desc->rqcfg.src_inc = 1; desc->rqcfg.dst_inc = 0; fill_px(&desc->px, addr, sg_dma_address(sg), sg_dma_len(sg)); } else { desc->rqcfg.src_inc = 0; desc->rqcfg.dst_inc = 1; fill_px(&desc->px, sg_dma_address(sg), addr, sg_dma_len(sg)); } desc->rqcfg.brst_size = burst_size; desc->rqcfg.brst_len = 1; } /* Return the last desc in the chain */ desc->txd.flags = flg; return &desc->txd; } static irqreturn_t pl330_irq_handler(int irq, void *data) { if (pl330_update(data)) return IRQ_HANDLED; else return IRQ_NONE; } static int __devinit pl330_probe(struct amba_device *adev, struct amba_id *id) { struct dma_pl330_platdata *pdat; struct dma_pl330_dmac *pdmac; struct dma_pl330_chan *pch; struct pl330_info *pi; struct dma_device *pd; struct resource *res; int i, ret, irq; pdat = adev->dev.platform_data; if (!pdat || !pdat->nr_valid_peri) { dev_err(&adev->dev, "platform data missing\n"); return -ENODEV; } /* Allocate a new DMAC and its Channels */ pdmac = kzalloc(pdat->nr_valid_peri * sizeof(*pch) + sizeof(*pdmac), GFP_KERNEL); if (!pdmac) { dev_err(&adev->dev, "unable to allocate mem\n"); return -ENOMEM; } pi = &pdmac->pif; pi->dev = &adev->dev; pi->pl330_data = NULL; pi->mcbufsz = pdat->mcbuf_sz; res = &adev->res; request_mem_region(res->start, resource_size(res), "dma-pl330"); pi->base = ioremap(res->start, resource_size(res)); if (!pi->base) { ret = -ENXIO; goto probe_err1; } irq = adev->irq[0]; ret = request_irq(irq, pl330_irq_handler, 0, dev_name(&adev->dev), pi); if (ret) goto probe_err2; ret = pl330_add(pi); if (ret) goto probe_err3; INIT_LIST_HEAD(&pdmac->desc_pool); spin_lock_init(&pdmac->pool_lock); /* Create a descriptor pool of default size */ if (!add_desc(pdmac, GFP_KERNEL, NR_DEFAULT_DESC)) dev_warn(&adev->dev, "unable to allocate desc\n"); pd = &pdmac->ddma; INIT_LIST_HEAD(&pd->channels); /* Initialize channel parameters */ for (i = 0; i < pdat->nr_valid_peri; i++) { struct dma_pl330_peri *peri = &pdat->peri[i]; pch = &pdmac->peripherals[i]; switch (peri->rqtype) { case MEMTOMEM: dma_cap_set(DMA_MEMCPY, pd->cap_mask); break; case MEMTODEV: case DEVTOMEM: dma_cap_set(DMA_SLAVE, pd->cap_mask); break; default: dev_err(&adev->dev, "DEVTODEV Not Supported\n"); continue; } INIT_LIST_HEAD(&pch->work_list); spin_lock_init(&pch->lock); pch->pl330_chid = NULL; pch->chan.private = peri; pch->chan.device = pd; pch->chan.chan_id = i; pch->dmac = pdmac; /* Add the channel to the DMAC list */ pd->chancnt++; list_add_tail(&pch->chan.device_node, &pd->channels); } pd->dev = &adev->dev; pd->device_alloc_chan_resources = pl330_alloc_chan_resources; pd->device_free_chan_resources = pl330_free_chan_resources; pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy; pd->device_tx_status = pl330_tx_status; pd->device_prep_slave_sg = pl330_prep_slave_sg; pd->device_control = pl330_control; pd->device_issue_pending = pl330_issue_pending; ret = dma_async_device_register(pd); if (ret) { dev_err(&adev->dev, "unable to register DMAC\n"); goto probe_err4; } amba_set_drvdata(adev, pdmac); dev_info(&adev->dev, "Loaded driver for PL330 DMAC-%d\n", adev->periphid); dev_info(&adev->dev, "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n", pi->pcfg.data_buf_dep, pi->pcfg.data_bus_width / 8, pi->pcfg.num_chan, pi->pcfg.num_peri, pi->pcfg.num_events); return 0; probe_err4: pl330_del(pi); probe_err3: free_irq(irq, pi); probe_err2: iounmap(pi->base); probe_err1: release_mem_region(res->start, resource_size(res)); kfree(pdmac); return ret; } static int __devexit pl330_remove(struct amba_device *adev) { struct dma_pl330_dmac *pdmac = amba_get_drvdata(adev); struct dma_pl330_chan *pch, *_p; struct pl330_info *pi; struct resource *res; int irq; if (!pdmac) return 0; amba_set_drvdata(adev, NULL); /* Idle the DMAC */ list_for_each_entry_safe(pch, _p, &pdmac->ddma.channels, chan.device_node) { /* Remove the channel */ list_del(&pch->chan.device_node); /* Flush the channel */ pl330_control(&pch->chan, DMA_TERMINATE_ALL, 0); pl330_free_chan_resources(&pch->chan); } pi = &pdmac->pif; pl330_del(pi); irq = adev->irq[0]; free_irq(irq, pi); iounmap(pi->base); res = &adev->res; release_mem_region(res->start, resource_size(res)); kfree(pdmac); return 0; } static struct amba_id pl330_ids[] = { { .id = 0x00041330, .mask = 0x000fffff, }, { 0, 0 }, }; static struct amba_driver pl330_driver = { .drv = { .owner = THIS_MODULE, .name = "dma-pl330", }, .id_table = pl330_ids, .probe = pl330_probe, .remove = pl330_remove, }; static int __init pl330_init(void) { return amba_driver_register(&pl330_driver); } module_init(pl330_init); static void __exit pl330_exit(void) { amba_driver_unregister(&pl330_driver); return; } module_exit(pl330_exit); MODULE_AUTHOR("Jaswinder Singh <jassi.brar@samsung.com>"); MODULE_DESCRIPTION("API Driver for PL330 DMAC"); MODULE_LICENSE("GPL");