2 * offload engine driver for the Marvell XOR engine
3 * Copyright (C) 2007, 2008, Marvell International Ltd.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 #include <linux/init.h>
16 #include <linux/slab.h>
17 #include <linux/delay.h>
18 #include <linux/dma-mapping.h>
19 #include <linux/spinlock.h>
20 #include <linux/interrupt.h>
21 #include <linux/of_device.h>
22 #include <linux/platform_device.h>
23 #include <linux/memory.h>
24 #include <linux/clk.h>
26 #include <linux/of_irq.h>
27 #include <linux/irqdomain.h>
28 #include <linux/cpumask.h>
29 #include <linux/platform_data/dma-mv_xor.h>
31 #include "dmaengine.h"
45 static void mv_xor_issue_pending(struct dma_chan *chan);
47 #define to_mv_xor_chan(chan) \
48 container_of(chan, struct mv_xor_chan, dmachan)
50 #define to_mv_xor_slot(tx) \
51 container_of(tx, struct mv_xor_desc_slot, async_tx)
53 #define mv_chan_to_devp(chan) \
56 static void mv_desc_init(struct mv_xor_desc_slot *desc,
57 dma_addr_t addr, u32 byte_count,
58 enum dma_ctrl_flags flags)
60 struct mv_xor_desc *hw_desc = desc->hw_desc;
62 hw_desc->status = XOR_DESC_DMA_OWNED;
63 hw_desc->phy_next_desc = 0;
64 /* Enable end-of-descriptor interrupts only for DMA_PREP_INTERRUPT */
65 hw_desc->desc_command = (flags & DMA_PREP_INTERRUPT) ?
66 XOR_DESC_EOD_INT_EN : 0;
67 hw_desc->phy_dest_addr = addr;
68 hw_desc->byte_count = byte_count;
71 static void mv_desc_set_mode(struct mv_xor_desc_slot *desc)
73 struct mv_xor_desc *hw_desc = desc->hw_desc;
78 hw_desc->desc_command |= XOR_DESC_OPERATION_XOR;
81 hw_desc->desc_command |= XOR_DESC_OPERATION_MEMCPY;
89 static void mv_desc_set_next_desc(struct mv_xor_desc_slot *desc,
92 struct mv_xor_desc *hw_desc = desc->hw_desc;
93 BUG_ON(hw_desc->phy_next_desc);
94 hw_desc->phy_next_desc = next_desc_addr;
97 static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc,
98 int index, dma_addr_t addr)
100 struct mv_xor_desc *hw_desc = desc->hw_desc;
101 hw_desc->phy_src_addr[mv_phy_src_idx(index)] = addr;
102 if (desc->type == DMA_XOR)
103 hw_desc->desc_command |= (1 << index);
106 static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan)
108 return readl_relaxed(XOR_CURR_DESC(chan));
111 static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan,
114 writel_relaxed(next_desc_addr, XOR_NEXT_DESC(chan));
117 static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan)
119 u32 val = readl_relaxed(XOR_INTR_MASK(chan));
120 val |= XOR_INTR_MASK_VALUE << (chan->idx * 16);
121 writel_relaxed(val, XOR_INTR_MASK(chan));
124 static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan)
126 u32 intr_cause = readl_relaxed(XOR_INTR_CAUSE(chan));
127 intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF;
131 static void mv_chan_clear_eoc_cause(struct mv_xor_chan *chan)
135 val = XOR_INT_END_OF_DESC | XOR_INT_END_OF_CHAIN | XOR_INT_STOPPED;
136 val = ~(val << (chan->idx * 16));
137 dev_dbg(mv_chan_to_devp(chan), "%s, val 0x%08x\n", __func__, val);
138 writel_relaxed(val, XOR_INTR_CAUSE(chan));
141 static void mv_chan_clear_err_status(struct mv_xor_chan *chan)
143 u32 val = 0xFFFF0000 >> (chan->idx * 16);
144 writel_relaxed(val, XOR_INTR_CAUSE(chan));
147 static void mv_chan_set_mode(struct mv_xor_chan *chan,
150 u32 config = readl_relaxed(XOR_CONFIG(chan));
155 #if defined(__BIG_ENDIAN)
156 config |= XOR_DESCRIPTOR_SWAP;
158 config &= ~XOR_DESCRIPTOR_SWAP;
161 writel_relaxed(config, XOR_CONFIG(chan));
164 static void mv_chan_activate(struct mv_xor_chan *chan)
166 dev_dbg(mv_chan_to_devp(chan), " activate chan.\n");
168 /* writel ensures all descriptors are flushed before activation */
169 writel(BIT(0), XOR_ACTIVATION(chan));
172 static char mv_chan_is_busy(struct mv_xor_chan *chan)
174 u32 state = readl_relaxed(XOR_ACTIVATION(chan));
176 state = (state >> 4) & 0x3;
178 return (state == 1) ? 1 : 0;
182 * mv_chan_start_new_chain - program the engine to operate on new
183 * chain headed by sw_desc
184 * Caller must hold &mv_chan->lock while calling this function
186 static void mv_chan_start_new_chain(struct mv_xor_chan *mv_chan,
187 struct mv_xor_desc_slot *sw_desc)
189 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: sw_desc %p\n",
190 __func__, __LINE__, sw_desc);
192 /* set the hardware chain */
193 mv_chan_set_next_descriptor(mv_chan, sw_desc->async_tx.phys);
196 mv_xor_issue_pending(&mv_chan->dmachan);
200 mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc,
201 struct mv_xor_chan *mv_chan,
204 BUG_ON(desc->async_tx.cookie < 0);
206 if (desc->async_tx.cookie > 0) {
207 cookie = desc->async_tx.cookie;
209 /* call the callback (must not sleep or submit new
210 * operations to this channel)
212 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
213 dma_descriptor_unmap(&desc->async_tx);
216 /* run dependent operations */
217 dma_run_dependencies(&desc->async_tx);
223 mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan)
225 struct mv_xor_desc_slot *iter, *_iter;
227 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
228 list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
231 if (async_tx_test_ack(&iter->async_tx))
232 list_move_tail(&iter->node, &mv_chan->free_slots);
238 mv_desc_clean_slot(struct mv_xor_desc_slot *desc,
239 struct mv_xor_chan *mv_chan)
241 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: desc %p flags %d\n",
242 __func__, __LINE__, desc, desc->async_tx.flags);
244 /* the client is allowed to attach dependent operations
247 if (!async_tx_test_ack(&desc->async_tx))
248 /* move this slot to the completed_slots */
249 list_move_tail(&desc->node, &mv_chan->completed_slots);
251 list_move_tail(&desc->node, &mv_chan->free_slots);
256 /* This function must be called with the mv_xor_chan spinlock held */
257 static void mv_chan_slot_cleanup(struct mv_xor_chan *mv_chan)
259 struct mv_xor_desc_slot *iter, *_iter;
260 dma_cookie_t cookie = 0;
261 int busy = mv_chan_is_busy(mv_chan);
262 u32 current_desc = mv_chan_get_current_desc(mv_chan);
263 int current_cleaned = 0;
264 struct mv_xor_desc *hw_desc;
266 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
267 dev_dbg(mv_chan_to_devp(mv_chan), "current_desc %x\n", current_desc);
268 mv_chan_clean_completed_slots(mv_chan);
270 /* free completed slots from the chain starting with
271 * the oldest descriptor
274 list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
277 /* clean finished descriptors */
278 hw_desc = iter->hw_desc;
279 if (hw_desc->status & XOR_DESC_SUCCESS) {
280 cookie = mv_desc_run_tx_complete_actions(iter, mv_chan,
283 /* done processing desc, clean slot */
284 mv_desc_clean_slot(iter, mv_chan);
286 /* break if we did cleaned the current */
287 if (iter->async_tx.phys == current_desc) {
292 if (iter->async_tx.phys == current_desc) {
299 if ((busy == 0) && !list_empty(&mv_chan->chain)) {
300 if (current_cleaned) {
302 * current descriptor cleaned and removed, run
305 iter = list_entry(mv_chan->chain.next,
306 struct mv_xor_desc_slot,
308 mv_chan_start_new_chain(mv_chan, iter);
310 if (!list_is_last(&iter->node, &mv_chan->chain)) {
312 * descriptors are still waiting after
313 * current, trigger them
315 iter = list_entry(iter->node.next,
316 struct mv_xor_desc_slot,
318 mv_chan_start_new_chain(mv_chan, iter);
321 * some descriptors are still waiting
324 tasklet_schedule(&mv_chan->irq_tasklet);
330 mv_chan->dmachan.completed_cookie = cookie;
333 static void mv_xor_tasklet(unsigned long data)
335 struct mv_xor_chan *chan = (struct mv_xor_chan *) data;
337 spin_lock_bh(&chan->lock);
338 mv_chan_slot_cleanup(chan);
339 spin_unlock_bh(&chan->lock);
342 static struct mv_xor_desc_slot *
343 mv_chan_alloc_slot(struct mv_xor_chan *mv_chan)
345 struct mv_xor_desc_slot *iter;
347 spin_lock_bh(&mv_chan->lock);
349 if (!list_empty(&mv_chan->free_slots)) {
350 iter = list_first_entry(&mv_chan->free_slots,
351 struct mv_xor_desc_slot,
354 list_move_tail(&iter->node, &mv_chan->allocated_slots);
356 spin_unlock_bh(&mv_chan->lock);
358 /* pre-ack descriptor */
359 async_tx_ack(&iter->async_tx);
360 iter->async_tx.cookie = -EBUSY;
366 spin_unlock_bh(&mv_chan->lock);
368 /* try to free some slots if the allocation fails */
369 tasklet_schedule(&mv_chan->irq_tasklet);
374 /************************ DMA engine API functions ****************************/
376 mv_xor_tx_submit(struct dma_async_tx_descriptor *tx)
378 struct mv_xor_desc_slot *sw_desc = to_mv_xor_slot(tx);
379 struct mv_xor_chan *mv_chan = to_mv_xor_chan(tx->chan);
380 struct mv_xor_desc_slot *old_chain_tail;
382 int new_hw_chain = 1;
384 dev_dbg(mv_chan_to_devp(mv_chan),
385 "%s sw_desc %p: async_tx %p\n",
386 __func__, sw_desc, &sw_desc->async_tx);
388 spin_lock_bh(&mv_chan->lock);
389 cookie = dma_cookie_assign(tx);
391 if (list_empty(&mv_chan->chain))
392 list_move_tail(&sw_desc->node, &mv_chan->chain);
396 old_chain_tail = list_entry(mv_chan->chain.prev,
397 struct mv_xor_desc_slot,
399 list_move_tail(&sw_desc->node, &mv_chan->chain);
401 dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %pa\n",
402 &old_chain_tail->async_tx.phys);
404 /* fix up the hardware chain */
405 mv_desc_set_next_desc(old_chain_tail, sw_desc->async_tx.phys);
407 /* if the channel is not busy */
408 if (!mv_chan_is_busy(mv_chan)) {
409 u32 current_desc = mv_chan_get_current_desc(mv_chan);
411 * and the curren desc is the end of the chain before
412 * the append, then we need to start the channel
414 if (current_desc == old_chain_tail->async_tx.phys)
420 mv_chan_start_new_chain(mv_chan, sw_desc);
422 spin_unlock_bh(&mv_chan->lock);
427 /* returns the number of allocated descriptors */
428 static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
433 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
434 struct mv_xor_desc_slot *slot = NULL;
435 int num_descs_in_pool = MV_XOR_POOL_SIZE/MV_XOR_SLOT_SIZE;
437 /* Allocate descriptor slots */
438 idx = mv_chan->slots_allocated;
439 while (idx < num_descs_in_pool) {
440 slot = kzalloc(sizeof(*slot), GFP_KERNEL);
442 dev_info(mv_chan_to_devp(mv_chan),
443 "channel only initialized %d descriptor slots",
447 virt_desc = mv_chan->dma_desc_pool_virt;
448 slot->hw_desc = virt_desc + idx * MV_XOR_SLOT_SIZE;
450 dma_async_tx_descriptor_init(&slot->async_tx, chan);
451 slot->async_tx.tx_submit = mv_xor_tx_submit;
452 INIT_LIST_HEAD(&slot->node);
453 dma_desc = mv_chan->dma_desc_pool;
454 slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE;
457 spin_lock_bh(&mv_chan->lock);
458 mv_chan->slots_allocated = idx;
459 list_add_tail(&slot->node, &mv_chan->free_slots);
460 spin_unlock_bh(&mv_chan->lock);
463 dev_dbg(mv_chan_to_devp(mv_chan),
464 "allocated %d descriptor slots\n",
465 mv_chan->slots_allocated);
467 return mv_chan->slots_allocated ? : -ENOMEM;
470 static struct dma_async_tx_descriptor *
471 mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
472 unsigned int src_cnt, size_t len, unsigned long flags)
474 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
475 struct mv_xor_desc_slot *sw_desc;
477 if (unlikely(len < MV_XOR_MIN_BYTE_COUNT))
480 BUG_ON(len > MV_XOR_MAX_BYTE_COUNT);
482 dev_dbg(mv_chan_to_devp(mv_chan),
483 "%s src_cnt: %d len: %zu dest %pad flags: %ld\n",
484 __func__, src_cnt, len, &dest, flags);
486 sw_desc = mv_chan_alloc_slot(mv_chan);
488 sw_desc->type = DMA_XOR;
489 sw_desc->async_tx.flags = flags;
490 mv_desc_init(sw_desc, dest, len, flags);
491 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
492 mv_desc_set_mode(sw_desc);
494 mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]);
497 dev_dbg(mv_chan_to_devp(mv_chan),
498 "%s sw_desc %p async_tx %p \n",
499 __func__, sw_desc, &sw_desc->async_tx);
500 return sw_desc ? &sw_desc->async_tx : NULL;
503 static struct dma_async_tx_descriptor *
504 mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
505 size_t len, unsigned long flags)
508 * A MEMCPY operation is identical to an XOR operation with only
509 * a single source address.
511 return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
514 static struct dma_async_tx_descriptor *
515 mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
517 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
518 dma_addr_t src, dest;
521 src = mv_chan->dummy_src_addr;
522 dest = mv_chan->dummy_dst_addr;
523 len = MV_XOR_MIN_BYTE_COUNT;
526 * We implement the DMA_INTERRUPT operation as a minimum sized
527 * XOR operation with a single dummy source address.
529 return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
532 static void mv_xor_free_chan_resources(struct dma_chan *chan)
534 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
535 struct mv_xor_desc_slot *iter, *_iter;
536 int in_use_descs = 0;
538 spin_lock_bh(&mv_chan->lock);
540 mv_chan_slot_cleanup(mv_chan);
542 list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
545 list_move_tail(&iter->node, &mv_chan->free_slots);
547 list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
550 list_move_tail(&iter->node, &mv_chan->free_slots);
552 list_for_each_entry_safe(iter, _iter, &mv_chan->allocated_slots,
555 list_move_tail(&iter->node, &mv_chan->free_slots);
557 list_for_each_entry_safe_reverse(
558 iter, _iter, &mv_chan->free_slots, node) {
559 list_del(&iter->node);
561 mv_chan->slots_allocated--;
564 dev_dbg(mv_chan_to_devp(mv_chan), "%s slots_allocated %d\n",
565 __func__, mv_chan->slots_allocated);
566 spin_unlock_bh(&mv_chan->lock);
569 dev_err(mv_chan_to_devp(mv_chan),
570 "freeing %d in use descriptors!\n", in_use_descs);
574 * mv_xor_status - poll the status of an XOR transaction
575 * @chan: XOR channel handle
576 * @cookie: XOR transaction identifier
577 * @txstate: XOR transactions state holder (or NULL)
579 static enum dma_status mv_xor_status(struct dma_chan *chan,
581 struct dma_tx_state *txstate)
583 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
586 ret = dma_cookie_status(chan, cookie, txstate);
587 if (ret == DMA_COMPLETE)
590 spin_lock_bh(&mv_chan->lock);
591 mv_chan_slot_cleanup(mv_chan);
592 spin_unlock_bh(&mv_chan->lock);
594 return dma_cookie_status(chan, cookie, txstate);
597 static void mv_chan_dump_regs(struct mv_xor_chan *chan)
601 val = readl_relaxed(XOR_CONFIG(chan));
602 dev_err(mv_chan_to_devp(chan), "config 0x%08x\n", val);
604 val = readl_relaxed(XOR_ACTIVATION(chan));
605 dev_err(mv_chan_to_devp(chan), "activation 0x%08x\n", val);
607 val = readl_relaxed(XOR_INTR_CAUSE(chan));
608 dev_err(mv_chan_to_devp(chan), "intr cause 0x%08x\n", val);
610 val = readl_relaxed(XOR_INTR_MASK(chan));
611 dev_err(mv_chan_to_devp(chan), "intr mask 0x%08x\n", val);
613 val = readl_relaxed(XOR_ERROR_CAUSE(chan));
614 dev_err(mv_chan_to_devp(chan), "error cause 0x%08x\n", val);
616 val = readl_relaxed(XOR_ERROR_ADDR(chan));
617 dev_err(mv_chan_to_devp(chan), "error addr 0x%08x\n", val);
620 static void mv_chan_err_interrupt_handler(struct mv_xor_chan *chan,
623 if (intr_cause & XOR_INT_ERR_DECODE) {
624 dev_dbg(mv_chan_to_devp(chan), "ignoring address decode error\n");
628 dev_err(mv_chan_to_devp(chan), "error on chan %d. intr cause 0x%08x\n",
629 chan->idx, intr_cause);
631 mv_chan_dump_regs(chan);
635 static irqreturn_t mv_xor_interrupt_handler(int irq, void *data)
637 struct mv_xor_chan *chan = data;
638 u32 intr_cause = mv_chan_get_intr_cause(chan);
640 dev_dbg(mv_chan_to_devp(chan), "intr cause %x\n", intr_cause);
642 if (intr_cause & XOR_INTR_ERRORS)
643 mv_chan_err_interrupt_handler(chan, intr_cause);
645 tasklet_schedule(&chan->irq_tasklet);
647 mv_chan_clear_eoc_cause(chan);
652 static void mv_xor_issue_pending(struct dma_chan *chan)
654 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
656 if (mv_chan->pending >= MV_XOR_THRESHOLD) {
657 mv_chan->pending = 0;
658 mv_chan_activate(mv_chan);
663 * Perform a transaction to verify the HW works.
666 static int mv_chan_memcpy_self_test(struct mv_xor_chan *mv_chan)
670 dma_addr_t src_dma, dest_dma;
671 struct dma_chan *dma_chan;
673 struct dma_async_tx_descriptor *tx;
674 struct dmaengine_unmap_data *unmap;
677 src = kmalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL);
681 dest = kzalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL);
687 /* Fill in src buffer */
688 for (i = 0; i < PAGE_SIZE; i++)
689 ((u8 *) src)[i] = (u8)i;
691 dma_chan = &mv_chan->dmachan;
692 if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
697 unmap = dmaengine_get_unmap_data(dma_chan->device->dev, 2, GFP_KERNEL);
703 src_dma = dma_map_page(dma_chan->device->dev, virt_to_page(src),
704 (size_t)src & ~PAGE_MASK, PAGE_SIZE,
706 unmap->addr[0] = src_dma;
708 ret = dma_mapping_error(dma_chan->device->dev, src_dma);
715 dest_dma = dma_map_page(dma_chan->device->dev, virt_to_page(dest),
716 (size_t)dest & ~PAGE_MASK, PAGE_SIZE,
718 unmap->addr[1] = dest_dma;
720 ret = dma_mapping_error(dma_chan->device->dev, dest_dma);
726 unmap->len = PAGE_SIZE;
728 tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
731 dev_err(dma_chan->device->dev,
732 "Self-test cannot prepare operation, disabling\n");
737 cookie = mv_xor_tx_submit(tx);
738 if (dma_submit_error(cookie)) {
739 dev_err(dma_chan->device->dev,
740 "Self-test submit error, disabling\n");
745 mv_xor_issue_pending(dma_chan);
749 if (mv_xor_status(dma_chan, cookie, NULL) !=
751 dev_err(dma_chan->device->dev,
752 "Self-test copy timed out, disabling\n");
757 dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
758 PAGE_SIZE, DMA_FROM_DEVICE);
759 if (memcmp(src, dest, PAGE_SIZE)) {
760 dev_err(dma_chan->device->dev,
761 "Self-test copy failed compare, disabling\n");
767 dmaengine_unmap_put(unmap);
768 mv_xor_free_chan_resources(dma_chan);
775 #define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */
777 mv_chan_xor_self_test(struct mv_xor_chan *mv_chan)
781 struct page *xor_srcs[MV_XOR_NUM_SRC_TEST];
782 dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST];
784 struct dma_async_tx_descriptor *tx;
785 struct dmaengine_unmap_data *unmap;
786 struct dma_chan *dma_chan;
791 int src_count = MV_XOR_NUM_SRC_TEST;
793 for (src_idx = 0; src_idx < src_count; src_idx++) {
794 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
795 if (!xor_srcs[src_idx]) {
797 __free_page(xor_srcs[src_idx]);
802 dest = alloc_page(GFP_KERNEL);
805 __free_page(xor_srcs[src_idx]);
809 /* Fill in src buffers */
810 for (src_idx = 0; src_idx < src_count; src_idx++) {
811 u8 *ptr = page_address(xor_srcs[src_idx]);
812 for (i = 0; i < PAGE_SIZE; i++)
813 ptr[i] = (1 << src_idx);
816 for (src_idx = 0; src_idx < src_count; src_idx++)
817 cmp_byte ^= (u8) (1 << src_idx);
819 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
820 (cmp_byte << 8) | cmp_byte;
822 memset(page_address(dest), 0, PAGE_SIZE);
824 dma_chan = &mv_chan->dmachan;
825 if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
830 unmap = dmaengine_get_unmap_data(dma_chan->device->dev, src_count + 1,
838 for (i = 0; i < src_count; i++) {
839 unmap->addr[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
840 0, PAGE_SIZE, DMA_TO_DEVICE);
841 dma_srcs[i] = unmap->addr[i];
842 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[i]);
850 unmap->addr[src_count] = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE,
852 dest_dma = unmap->addr[src_count];
853 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[src_count]);
859 unmap->len = PAGE_SIZE;
861 tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
862 src_count, PAGE_SIZE, 0);
864 dev_err(dma_chan->device->dev,
865 "Self-test cannot prepare operation, disabling\n");
870 cookie = mv_xor_tx_submit(tx);
871 if (dma_submit_error(cookie)) {
872 dev_err(dma_chan->device->dev,
873 "Self-test submit error, disabling\n");
878 mv_xor_issue_pending(dma_chan);
882 if (mv_xor_status(dma_chan, cookie, NULL) !=
884 dev_err(dma_chan->device->dev,
885 "Self-test xor timed out, disabling\n");
890 dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
891 PAGE_SIZE, DMA_FROM_DEVICE);
892 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
893 u32 *ptr = page_address(dest);
894 if (ptr[i] != cmp_word) {
895 dev_err(dma_chan->device->dev,
896 "Self-test xor failed compare, disabling. index %d, data %x, expected %x\n",
897 i, ptr[i], cmp_word);
904 dmaengine_unmap_put(unmap);
905 mv_xor_free_chan_resources(dma_chan);
909 __free_page(xor_srcs[src_idx]);
914 static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan)
916 struct dma_chan *chan, *_chan;
917 struct device *dev = mv_chan->dmadev.dev;
919 dma_async_device_unregister(&mv_chan->dmadev);
921 dma_free_coherent(dev, MV_XOR_POOL_SIZE,
922 mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
923 dma_unmap_single(dev, mv_chan->dummy_src_addr,
924 MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
925 dma_unmap_single(dev, mv_chan->dummy_dst_addr,
926 MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
928 list_for_each_entry_safe(chan, _chan, &mv_chan->dmadev.channels,
930 list_del(&chan->device_node);
933 free_irq(mv_chan->irq, mv_chan);
938 static struct mv_xor_chan *
939 mv_xor_channel_add(struct mv_xor_device *xordev,
940 struct platform_device *pdev,
941 int idx, dma_cap_mask_t cap_mask, int irq)
944 struct mv_xor_chan *mv_chan;
945 struct dma_device *dma_dev;
947 mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL);
949 return ERR_PTR(-ENOMEM);
953 if (xordev->xor_type == XOR_ORION)
954 mv_chan->op_in_desc = XOR_MODE_IN_REG;
956 mv_chan->op_in_desc = XOR_MODE_IN_DESC;
958 dma_dev = &mv_chan->dmadev;
961 * These source and destination dummy buffers are used to implement
962 * a DMA_INTERRUPT operation as a minimum-sized XOR operation.
963 * Hence, we only need to map the buffers at initialization-time.
965 mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev,
966 mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
967 mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev,
968 mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
970 /* allocate coherent memory for hardware descriptors
971 * note: writecombine gives slightly better performance, but
972 * requires that we explicitly flush the writes
974 mv_chan->dma_desc_pool_virt =
975 dma_alloc_wc(&pdev->dev, MV_XOR_POOL_SIZE, &mv_chan->dma_desc_pool,
977 if (!mv_chan->dma_desc_pool_virt)
978 return ERR_PTR(-ENOMEM);
980 /* discover transaction capabilites from the platform data */
981 dma_dev->cap_mask = cap_mask;
983 INIT_LIST_HEAD(&dma_dev->channels);
985 /* set base routines */
986 dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources;
987 dma_dev->device_free_chan_resources = mv_xor_free_chan_resources;
988 dma_dev->device_tx_status = mv_xor_status;
989 dma_dev->device_issue_pending = mv_xor_issue_pending;
990 dma_dev->dev = &pdev->dev;
992 /* set prep routines based on capability */
993 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
994 dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt;
995 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
996 dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy;
997 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
998 dma_dev->max_xor = 8;
999 dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor;
1002 mv_chan->mmr_base = xordev->xor_base;
1003 mv_chan->mmr_high_base = xordev->xor_high_base;
1004 tasklet_init(&mv_chan->irq_tasklet, mv_xor_tasklet, (unsigned long)
1007 /* clear errors before enabling interrupts */
1008 mv_chan_clear_err_status(mv_chan);
1010 ret = request_irq(mv_chan->irq, mv_xor_interrupt_handler,
1011 0, dev_name(&pdev->dev), mv_chan);
1015 mv_chan_unmask_interrupts(mv_chan);
1017 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
1018 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_IN_DESC);
1020 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_XOR);
1022 spin_lock_init(&mv_chan->lock);
1023 INIT_LIST_HEAD(&mv_chan->chain);
1024 INIT_LIST_HEAD(&mv_chan->completed_slots);
1025 INIT_LIST_HEAD(&mv_chan->free_slots);
1026 INIT_LIST_HEAD(&mv_chan->allocated_slots);
1027 mv_chan->dmachan.device = dma_dev;
1028 dma_cookie_init(&mv_chan->dmachan);
1030 list_add_tail(&mv_chan->dmachan.device_node, &dma_dev->channels);
1032 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1033 ret = mv_chan_memcpy_self_test(mv_chan);
1034 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1039 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1040 ret = mv_chan_xor_self_test(mv_chan);
1041 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1046 dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n",
1047 mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode",
1048 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1049 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1050 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1052 dma_async_device_register(dma_dev);
1056 free_irq(mv_chan->irq, mv_chan);
1058 dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE,
1059 mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
1060 return ERR_PTR(ret);
1064 mv_xor_conf_mbus_windows(struct mv_xor_device *xordev,
1065 const struct mbus_dram_target_info *dram)
1067 void __iomem *base = xordev->xor_high_base;
1071 for (i = 0; i < 8; i++) {
1072 writel(0, base + WINDOW_BASE(i));
1073 writel(0, base + WINDOW_SIZE(i));
1075 writel(0, base + WINDOW_REMAP_HIGH(i));
1078 for (i = 0; i < dram->num_cs; i++) {
1079 const struct mbus_dram_window *cs = dram->cs + i;
1081 writel((cs->base & 0xffff0000) |
1082 (cs->mbus_attr << 8) |
1083 dram->mbus_dram_target_id, base + WINDOW_BASE(i));
1084 writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i));
1086 win_enable |= (1 << i);
1087 win_enable |= 3 << (16 + (2 * i));
1090 writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1091 writel(win_enable, base + WINDOW_BAR_ENABLE(1));
1092 writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1093 writel(0, base + WINDOW_OVERRIDE_CTRL(1));
1097 mv_xor_conf_mbus_windows_a3700(struct mv_xor_device *xordev)
1099 void __iomem *base = xordev->xor_high_base;
1103 for (i = 0; i < 8; i++) {
1104 writel(0, base + WINDOW_BASE(i));
1105 writel(0, base + WINDOW_SIZE(i));
1107 writel(0, base + WINDOW_REMAP_HIGH(i));
1110 * For Armada3700 open default 4GB Mbus window. The dram
1111 * related configuration are done at AXIS level.
1113 writel(0xffff0000, base + WINDOW_SIZE(0));
1115 win_enable |= 3 << 16;
1117 writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1118 writel(win_enable, base + WINDOW_BAR_ENABLE(1));
1119 writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1120 writel(0, base + WINDOW_OVERRIDE_CTRL(1));
1124 * Since this XOR driver is basically used only for RAID5, we don't
1125 * need to care about synchronizing ->suspend with DMA activity,
1126 * because the DMA engine will naturally be quiet due to the block
1127 * devices being suspended.
1129 static int mv_xor_suspend(struct platform_device *pdev, pm_message_t state)
1131 struct mv_xor_device *xordev = platform_get_drvdata(pdev);
1134 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1135 struct mv_xor_chan *mv_chan = xordev->channels[i];
1140 mv_chan->saved_config_reg =
1141 readl_relaxed(XOR_CONFIG(mv_chan));
1142 mv_chan->saved_int_mask_reg =
1143 readl_relaxed(XOR_INTR_MASK(mv_chan));
1149 static int mv_xor_resume(struct platform_device *dev)
1151 struct mv_xor_device *xordev = platform_get_drvdata(dev);
1152 const struct mbus_dram_target_info *dram;
1155 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1156 struct mv_xor_chan *mv_chan = xordev->channels[i];
1161 writel_relaxed(mv_chan->saved_config_reg,
1162 XOR_CONFIG(mv_chan));
1163 writel_relaxed(mv_chan->saved_int_mask_reg,
1164 XOR_INTR_MASK(mv_chan));
1167 if (xordev->xor_type == XOR_ARMADA_37XX) {
1168 mv_xor_conf_mbus_windows_a3700(xordev);
1172 dram = mv_mbus_dram_info();
1174 mv_xor_conf_mbus_windows(xordev, dram);
1179 static const struct of_device_id mv_xor_dt_ids[] = {
1180 { .compatible = "marvell,orion-xor", .data = (void *)XOR_ORION },
1181 { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_ARMADA_38X },
1182 { .compatible = "marvell,armada-3700-xor", .data = (void *)XOR_ARMADA_37XX },
1186 static unsigned int mv_xor_engine_count;
1188 static int mv_xor_probe(struct platform_device *pdev)
1190 const struct mbus_dram_target_info *dram;
1191 struct mv_xor_device *xordev;
1192 struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev);
1193 struct resource *res;
1194 unsigned int max_engines, max_channels;
1197 dev_notice(&pdev->dev, "Marvell shared XOR driver\n");
1199 xordev = devm_kzalloc(&pdev->dev, sizeof(*xordev), GFP_KERNEL);
1203 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1207 xordev->xor_base = devm_ioremap(&pdev->dev, res->start,
1208 resource_size(res));
1209 if (!xordev->xor_base)
1212 res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1216 xordev->xor_high_base = devm_ioremap(&pdev->dev, res->start,
1217 resource_size(res));
1218 if (!xordev->xor_high_base)
1221 platform_set_drvdata(pdev, xordev);
1225 * We need to know which type of XOR device we use before
1226 * setting up. In non-dt case it can only be the legacy one.
1228 xordev->xor_type = XOR_ORION;
1229 if (pdev->dev.of_node) {
1230 const struct of_device_id *of_id =
1231 of_match_device(mv_xor_dt_ids,
1234 xordev->xor_type = (uintptr_t)of_id->data;
1238 * (Re-)program MBUS remapping windows if we are asked to.
1240 if (xordev->xor_type == XOR_ARMADA_37XX) {
1241 mv_xor_conf_mbus_windows_a3700(xordev);
1243 dram = mv_mbus_dram_info();
1245 mv_xor_conf_mbus_windows(xordev, dram);
1248 /* Not all platforms can gate the clock, so it is not
1249 * an error if the clock does not exists.
1251 xordev->clk = clk_get(&pdev->dev, NULL);
1252 if (!IS_ERR(xordev->clk))
1253 clk_prepare_enable(xordev->clk);
1256 * We don't want to have more than one channel per CPU in
1257 * order for async_tx to perform well. So we limit the number
1258 * of engines and channels so that we take into account this
1259 * constraint. Note that we also want to use channels from
1260 * separate engines when possible. For dual-CPU Armada 3700
1261 * SoC with single XOR engine allow using its both channels.
1263 max_engines = num_present_cpus();
1264 if (xordev->xor_type == XOR_ARMADA_37XX)
1265 max_channels = num_present_cpus();
1267 max_channels = min_t(unsigned int,
1268 MV_XOR_MAX_CHANNELS,
1269 DIV_ROUND_UP(num_present_cpus(), 2));
1271 if (mv_xor_engine_count >= max_engines)
1274 if (pdev->dev.of_node) {
1275 struct device_node *np;
1278 for_each_child_of_node(pdev->dev.of_node, np) {
1279 struct mv_xor_chan *chan;
1280 dma_cap_mask_t cap_mask;
1283 if (i >= max_channels)
1286 dma_cap_zero(cap_mask);
1287 dma_cap_set(DMA_MEMCPY, cap_mask);
1288 dma_cap_set(DMA_XOR, cap_mask);
1289 dma_cap_set(DMA_INTERRUPT, cap_mask);
1291 irq = irq_of_parse_and_map(np, 0);
1294 goto err_channel_add;
1297 chan = mv_xor_channel_add(xordev, pdev, i,
1300 ret = PTR_ERR(chan);
1301 irq_dispose_mapping(irq);
1302 goto err_channel_add;
1305 xordev->channels[i] = chan;
1308 } else if (pdata && pdata->channels) {
1309 for (i = 0; i < max_channels; i++) {
1310 struct mv_xor_channel_data *cd;
1311 struct mv_xor_chan *chan;
1314 cd = &pdata->channels[i];
1317 goto err_channel_add;
1320 irq = platform_get_irq(pdev, i);
1323 goto err_channel_add;
1326 chan = mv_xor_channel_add(xordev, pdev, i,
1329 ret = PTR_ERR(chan);
1330 goto err_channel_add;
1333 xordev->channels[i] = chan;
1340 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++)
1341 if (xordev->channels[i]) {
1342 mv_xor_channel_remove(xordev->channels[i]);
1343 if (pdev->dev.of_node)
1344 irq_dispose_mapping(xordev->channels[i]->irq);
1347 if (!IS_ERR(xordev->clk)) {
1348 clk_disable_unprepare(xordev->clk);
1349 clk_put(xordev->clk);
1355 static struct platform_driver mv_xor_driver = {
1356 .probe = mv_xor_probe,
1357 .suspend = mv_xor_suspend,
1358 .resume = mv_xor_resume,
1360 .name = MV_XOR_NAME,
1361 .of_match_table = of_match_ptr(mv_xor_dt_ids),
1366 static int __init mv_xor_init(void)
1368 return platform_driver_register(&mv_xor_driver);
1370 device_initcall(mv_xor_init);
1373 MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>");
1374 MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine");
1375 MODULE_LICENSE("GPL");