diff options
| -rw-r--r-- | drivers/dma/bcm-sba-raid.c | 544 |
1 files changed, 295 insertions, 249 deletions
diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c index e41bbc7cb094..6c2c44724637 100644 --- a/drivers/dma/bcm-sba-raid.c +++ b/drivers/dma/bcm-sba-raid.c @@ -36,6 +36,7 @@ */ #include <linux/bitops.h> +#include <linux/debugfs.h> #include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/list.h> @@ -48,7 +49,8 @@ #include "dmaengine.h" -/* SBA command related defines */ +/* ====== Driver macros and defines ===== */ + #define SBA_TYPE_SHIFT 48 #define SBA_TYPE_MASK GENMASK(1, 0) #define SBA_TYPE_A 0x0 @@ -82,39 +84,40 @@ #define SBA_CMD_WRITE_BUFFER 0xc #define SBA_CMD_GALOIS 0xe +#define SBA_MAX_REQ_PER_MBOX_CHANNEL 8192 + /* Driver helper macros */ #define to_sba_request(tx) \ container_of(tx, struct sba_request, tx) #define to_sba_device(dchan) \ container_of(dchan, struct sba_device, dma_chan) -enum sba_request_state { - SBA_REQUEST_STATE_FREE = 1, - SBA_REQUEST_STATE_ALLOCED = 2, - SBA_REQUEST_STATE_PENDING = 3, - SBA_REQUEST_STATE_ACTIVE = 4, - SBA_REQUEST_STATE_RECEIVED = 5, - SBA_REQUEST_STATE_COMPLETED = 6, - SBA_REQUEST_STATE_ABORTED = 7, +/* ===== Driver data structures ===== */ + +enum sba_request_flags { + SBA_REQUEST_STATE_FREE = 0x001, + SBA_REQUEST_STATE_ALLOCED = 0x002, + SBA_REQUEST_STATE_PENDING = 0x004, + SBA_REQUEST_STATE_ACTIVE = 0x008, + SBA_REQUEST_STATE_ABORTED = 0x010, + SBA_REQUEST_STATE_MASK = 0x0ff, + SBA_REQUEST_FENCE = 0x100, }; struct sba_request { /* Global state */ struct list_head node; struct sba_device *sba; - enum sba_request_state state; - bool fence; + u32 flags; /* Chained requests management */ struct sba_request *first; struct list_head next; - unsigned int next_count; atomic_t next_pending_count; /* BRCM message data */ - void *resp; - dma_addr_t resp_dma; - struct brcm_sba_command *cmds; struct brcm_message msg; struct dma_async_tx_descriptor tx; + /* SBA commands */ + struct brcm_sba_command cmds[0]; }; enum sba_version { @@ -152,19 +155,18 @@ struct sba_device { void *cmds_base; dma_addr_t cmds_dma_base; spinlock_t reqs_lock; - struct sba_request *reqs; bool reqs_fence; struct list_head reqs_alloc_list; struct list_head reqs_pending_list; struct list_head reqs_active_list; - struct list_head reqs_received_list; - struct list_head reqs_completed_list; struct list_head reqs_aborted_list; struct list_head reqs_free_list; - int reqs_free_count; + /* DebugFS directory entries */ + struct dentry *root; + struct dentry *stats; }; -/* ====== SBA command helper routines ===== */ +/* ====== Command helper routines ===== */ static inline u64 __pure sba_cmd_enc(u64 cmd, u32 val, u32 shift, u32 mask) { @@ -196,32 +198,50 @@ static inline u32 __pure sba_cmd_pq_c_mdata(u32 d, u32 b1, u32 b0) ((d & SBA_C_MDATA_DNUM_MASK) << SBA_C_MDATA_DNUM_SHIFT); } -/* ====== Channel resource management routines ===== */ +/* ====== General helper routines ===== */ + +static void sba_peek_mchans(struct sba_device *sba) +{ + int mchan_idx; + + for (mchan_idx = 0; mchan_idx < sba->mchans_count; mchan_idx++) + mbox_client_peek_data(sba->mchans[mchan_idx]); +} static struct sba_request *sba_alloc_request(struct sba_device *sba) { + bool found = false; unsigned long flags; struct sba_request *req = NULL; spin_lock_irqsave(&sba->reqs_lock, flags); + list_for_each_entry(req, &sba->reqs_free_list, node) { + if (async_tx_test_ack(&req->tx)) { + list_move_tail(&req->node, &sba->reqs_alloc_list); + found = true; + break; + } + } + spin_unlock_irqrestore(&sba->reqs_lock, flags); - req = list_first_entry_or_null(&sba->reqs_free_list, - struct sba_request, node); - if (req) { - list_move_tail(&req->node, &sba->reqs_alloc_list); - req->state = SBA_REQUEST_STATE_ALLOCED; - req->fence = false; - req->first = req; - INIT_LIST_HEAD(&req->next); - req->next_count = 1; - atomic_set(&req->next_pending_count, 1); - - sba->reqs_free_count--; - - dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan); + if (!found) { + /* + * We have no more free requests so, we peek + * mailbox channels hoping few active requests + * would have completed which will create more + * room for new requests. + */ + sba_peek_mchans(sba); + return NULL; } - spin_unlock_irqrestore(&sba->reqs_lock, flags); + req->flags = SBA_REQUEST_STATE_ALLOCED; + req->first = req; + INIT_LIST_HEAD(&req->next); + atomic_set(&req->next_pending_count, 1); + + dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan); + async_tx_ack(&req->tx); return req; } @@ -231,7 +251,8 @@ static void _sba_pending_request(struct sba_device *sba, struct sba_request *req) { lockdep_assert_held(&sba->reqs_lock); - req->state = SBA_REQUEST_STATE_PENDING; + req->flags &= ~SBA_REQUEST_STATE_MASK; + req->flags |= SBA_REQUEST_STATE_PENDING; list_move_tail(&req->node, &sba->reqs_pending_list); if (list_empty(&sba->reqs_active_list)) sba->reqs_fence = false; @@ -246,9 +267,10 @@ static bool _sba_active_request(struct sba_device *sba, sba->reqs_fence = false; if (sba->reqs_fence) return false; - req->state = SBA_REQUEST_STATE_ACTIVE; + req->flags &= ~SBA_REQUEST_STATE_MASK; + req->flags |= SBA_REQUEST_STATE_ACTIVE; list_move_tail(&req->node, &sba->reqs_active_list); - if (req->fence) + if (req->flags & SBA_REQUEST_FENCE) sba->reqs_fence = true; return true; } @@ -258,7 +280,8 @@ static void _sba_abort_request(struct sba_device *sba, struct sba_request *req) { lockdep_assert_held(&sba->reqs_lock); - req->state = SBA_REQUEST_STATE_ABORTED; + req->flags &= ~SBA_REQUEST_STATE_MASK; + req->flags |= SBA_REQUEST_STATE_ABORTED; list_move_tail(&req->node, &sba->reqs_aborted_list); if (list_empty(&sba->reqs_active_list)) sba->reqs_fence = false; @@ -269,42 +292,11 @@ static void _sba_free_request(struct sba_device *sba, struct sba_request *req) { lockdep_assert_held(&sba->reqs_lock); - req->state = SBA_REQUEST_STATE_FREE; + req->flags &= ~SBA_REQUEST_STATE_MASK; + req->flags |= SBA_REQUEST_STATE_FREE; list_move_tail(&req->node, &sba->reqs_free_list); if (list_empty(&sba->reqs_active_list)) sba->reqs_fence = false; - sba->reqs_free_count++; -} - -static void sba_received_request(struct sba_request *req) -{ - unsigned long flags; - struct sba_device *sba = req->sba; - - spin_lock_irqsave(&sba->reqs_lock, flags); - req->state = SBA_REQUEST_STATE_RECEIVED; - list_move_tail(&req->node, &sba->reqs_received_list); - spin_unlock_irqrestore(&sba->reqs_lock, flags); -} - -static void sba_complete_chained_requests(struct sba_request *req) -{ - unsigned long flags; - struct sba_request *nreq; - struct sba_device *sba = req->sba; - - spin_lock_irqsave(&sba->reqs_lock, flags); - - req->state = SBA_REQUEST_STATE_COMPLETED; - list_move_tail(&req->node, &sba->reqs_completed_list); - list_for_each_entry(nreq, &req->next, next) { - nreq->state = SBA_REQUEST_STATE_COMPLETED; - list_move_tail(&nreq->node, &sba->reqs_completed_list); - } - if (list_empty(&sba->reqs_active_list)) - sba->reqs_fence = false; - - spin_unlock_irqrestore(&sba->reqs_lock, flags); } static void sba_free_chained_requests(struct sba_request *req) @@ -332,8 +324,7 @@ static void sba_chain_request(struct sba_request *first, list_add_tail(&req->next, &first->next); req->first = first; - first->next_count++; - atomic_set(&first->next_pending_count, first->next_count); + atomic_inc(&first->next_pending_count); spin_unlock_irqrestore(&sba->reqs_lock, flags); } @@ -349,14 +340,6 @@ static void sba_cleanup_nonpending_requests(struct sba_device *sba) list_for_each_entry_safe(req, req1, &sba->reqs_alloc_list, node) _sba_free_request(sba, req); - /* Freeup all received request */ - list_for_each_entry_safe(req, req1, &sba->reqs_received_list, node) - _sba_free_request(sba, req); - - /* Freeup all completed request */ - list_for_each_entry_safe(req, req1, &sba->reqs_completed_list, node) - _sba_free_request(sba, req); - /* Set all active requests as aborted */ list_for_each_entry_safe(req, req1, &sba->reqs_active_list, node) _sba_abort_request(sba, req); @@ -383,26 +366,6 @@ static void sba_cleanup_pending_requests(struct sba_device *sba) spin_unlock_irqrestore(&sba->reqs_lock, flags); } -/* ====== DMAENGINE callbacks ===== */ - -static void sba_free_chan_resources(struct dma_chan *dchan) -{ - /* - * Channel resources are pre-alloced so we just free-up - * whatever we can so that we can re-use pre-alloced - * channel resources next time. - */ - sba_cleanup_nonpending_requests(to_sba_device(dchan)); -} - -static int sba_device_terminate_all(struct dma_chan *dchan) -{ - /* Cleanup all pending requests */ - sba_cleanup_pending_requests(to_sba_device(dchan)); - - return 0; -} - static int sba_send_mbox_request(struct sba_device *sba, struct sba_request *req) { @@ -419,42 +382,156 @@ static int sba_send_mbox_request(struct sba_device *sba, dev_err(sba->dev, "send message failed with error %d", ret); return ret; } + + /* Check error returned by mailbox controller */ ret = req->msg.error; if (ret < 0) { dev_err(sba->dev, "message error %d", ret); - return ret; } - return 0; + /* Signal txdone for mailbox channel */ + mbox_client_txdone(sba->mchans[mchans_idx], ret); + + return ret; } -static void sba_issue_pending(struct dma_chan *dchan) +/* Note: Must be called with sba->reqs_lock held */ +static void _sba_process_pending_requests(struct sba_device *sba) { int ret; - unsigned long flags; - struct sba_request *req, *req1; - struct sba_device *sba = to_sba_device(dchan); + u32 count; + struct sba_request *req; - spin_lock_irqsave(&sba->reqs_lock, flags); + /* + * Process few pending requests + * + * For now, we process (<number_of_mailbox_channels> * 8) + * number of requests at a time. + */ + count = sba->mchans_count * 8; + while (!list_empty(&sba->reqs_pending_list) && count) { + /* Get the first pending request */ + req = list_first_entry(&sba->reqs_pending_list, + struct sba_request, node); - /* Process all pending request */ - list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node) { /* Try to make request active */ if (!_sba_active_request(sba, req)) break; /* Send request to mailbox channel */ - spin_unlock_irqrestore(&sba->reqs_lock, flags); ret = sba_send_mbox_request(sba, req); - spin_lock_irqsave(&sba->reqs_lock, flags); - - /* If something went wrong then keep request pending */ if (ret < 0) { _sba_pending_request(sba, req); break; } + + count--; + } +} + +static void sba_process_received_request(struct sba_device *sba, + struct sba_request *req) +{ + unsigned long flags; + struct dma_async_tx_descriptor *tx; + struct sba_request *nreq, *first = req->first; + + /* Process only after all chained requests are received */ + if (!atomic_dec_return(&first->next_pending_count)) { + tx = &first->tx; + + WARN_ON(tx->cookie < 0); + if (tx->cookie > 0) { + dma_cookie_complete(tx); + dmaengine_desc_get_callback_invoke(tx, NULL); + dma_descriptor_unmap(tx); + tx->callback = NULL; + tx->callback_result = NULL; + } + + dma_run_dependencies(tx); + + spin_lock_irqsave(&sba->reqs_lock, flags); + + /* Free all requests chained to first request */ + list_for_each_entry(nreq, &first->next, next) + _sba_free_request(sba, nreq); + INIT_LIST_HEAD(&first->next); + + /* Free the first request */ + _sba_free_request(sba, first); + + /* Process pending requests */ + _sba_process_pending_requests(sba); + + spin_unlock_irqrestore(&sba->reqs_lock, flags); } +} + +static void sba_write_stats_in_seqfile(struct sba_device *sba, + struct seq_file *file) +{ + unsigned long flags; + struct sba_request *req; + u32 free_count = 0, alloced_count = 0; + u32 pending_count = 0, active_count = 0, aborted_count = 0; + + spin_lock_irqsave(&sba->reqs_lock, flags); + + list_for_each_entry(req, &sba->reqs_free_list, node) + if (async_tx_test_ack(&req->tx)) + free_count++; + + list_for_each_entry(req, &sba->reqs_alloc_list, node) + alloced_count++; + + list_for_each_entry(req, &sba->reqs_pending_list, node) + pending_count++; + + list_for_each_entry(req, &sba->reqs_active_list, node) + active_count++; + list_for_each_entry(req, &sba->reqs_aborted_list, node) + aborted_count++; + + spin_unlock_irqrestore(&sba->reqs_lock, flags); + + seq_printf(file, "maximum requests = %d\n", sba->max_req); + seq_printf(file, "free requests = %d\n", free_count); + seq_printf(file, "alloced requests = %d\n", alloced_count); + seq_printf(file, "pending requests = %d\n", pending_count); + seq_printf(file, "active requests = %d\n", active_count); + seq_printf(file, "aborted requests = %d\n", aborted_count); +} + +/* ====== DMAENGINE callbacks ===== */ + +static void sba_free_chan_resources(struct dma_chan *dchan) +{ + /* + * Channel resources are pre-alloced so we just free-up + * whatever we can so that we can re-use pre-alloced + * channel resources next time. + */ + sba_cleanup_nonpending_requests(to_sba_device(dchan)); +} + +static int sba_device_terminate_all(struct dma_chan *dchan) +{ + /* Cleanup all pending requests */ + sba_cleanup_pending_requests(to_sba_device(dchan)); + + return 0; +} + +static void sba_issue_pending(struct dma_chan *dchan) +{ + unsigned long flags; + struct sba_device *sba = to_sba_device(dchan); + + /* Process pending requests */ + spin_lock_irqsave(&sba->reqs_lock, flags); + _sba_process_pending_requests(sba); spin_unlock_irqrestore(&sba->reqs_lock, flags); } @@ -486,17 +563,15 @@ static enum dma_status sba_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, struct dma_tx_state *txstate) { - int mchan_idx; enum dma_status ret; struct sba_device *sba = to_sba_device(dchan); - for (mchan_idx = 0; mchan_idx < sba->mchans_count; mchan_idx++) - mbox_client_peek_data(sba->mchans[mchan_idx]); - ret = dma_cookie_status(dchan, cookie, txstate); if (ret == DMA_COMPLETE) return ret; + sba_peek_mchans(sba); + return dma_cookie_status(dchan, cookie, txstate); } @@ -506,6 +581,7 @@ static void sba_fillup_interrupt_msg(struct sba_request *req, { u64 cmd; u32 c_mdata; + dma_addr_t resp_dma = req->tx.phys; struct brcm_sba_command *cmdsp = cmds; /* Type-B command to load dummy data into buf0 */ @@ -521,7 +597,7 @@ static void sba_fillup_interrupt_msg(struct sba_request *req, cmdsp->cmd = cmd; *cmdsp->cmd_dma = cpu_to_le64(cmd); cmdsp->flags = BRCM_SBA_CMD_TYPE_B; - cmdsp->data = req->resp_dma; + cmdsp->data = resp_dma; cmdsp->data_len = req->sba->hw_resp_size; cmdsp++; @@ -542,11 +618,11 @@ static void sba_fillup_interrupt_msg(struct sba_request *req, cmdsp->flags = BRCM_SBA_CMD_TYPE_A; if (req->sba->hw_resp_size) { cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; - cmdsp->resp = req->resp_dma; + cmdsp->resp = resp_dma; cmdsp->resp_len = req->sba->hw_resp_size; } cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; - cmdsp->data = req->resp_dma; + cmdsp->data = resp_dma; cmdsp->data_len = req->sba->hw_resp_size; cmdsp++; @@ -573,7 +649,7 @@ sba_prep_dma_interrupt(struct dma_chan *dchan, unsigned long flags) * Force fence so that no requests are submitted * until DMA callback for this request is invoked. */ - req->fence = true; + req->flags |= SBA_REQUEST_FENCE; /* Fillup request message */ sba_fillup_interrupt_msg(req, req->cmds, &req->msg); @@ -593,6 +669,7 @@ static void sba_fillup_memcpy_msg(struct sba_request *req, { u64 cmd; u32 c_mdata; + dma_addr_t resp_dma = req->tx.phys; struct brcm_sba_command *cmdsp = cmds; /* Type-B command to load data into buf0 */ @@ -629,7 +706,7 @@ static void sba_fillup_memcpy_msg(struct sba_request *req, cmdsp->flags = BRCM_SBA_CMD_TYPE_A; if (req->sba->hw_resp_size) { cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; - cmdsp->resp = req->resp_dma; + cmdsp->resp = resp_dma; cmdsp->resp_len = req->sba->hw_resp_size; } cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; @@ -656,7 +733,8 @@ sba_prep_dma_memcpy_req(struct sba_device *sba, req = sba_alloc_request(sba); if (!req) return NULL; - req->fence = (flags & DMA_PREP_FENCE) ? true : false; + if (flags & DMA_PREP_FENCE) + req->flags |= SBA_REQUEST_FENCE; /* Fillup request message */ sba_fillup_memcpy_msg(req, req->cmds, &req->msg, @@ -711,6 +789,7 @@ static void sba_fillup_xor_msg(struct sba_request *req, u64 cmd; u32 c_mdata; unsigned int i; + dma_addr_t resp_dma = req->tx.phys; struct brcm_sba_command *cmdsp = cmds; /* Type-B command to load data into buf0 */ @@ -766,7 +845,7 @@ static void sba_fillup_xor_msg(struct sba_request *req, cmdsp->flags = BRCM_SBA_CMD_TYPE_A; if (req->sba->hw_resp_size) { cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; - cmdsp->resp = req->resp_dma; + cmdsp->resp = resp_dma; cmdsp->resp_len = req->sba->hw_resp_size; } cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; @@ -782,7 +861,7 @@ static void sba_fillup_xor_msg(struct sba_request *req, msg->error = 0; } -struct sba_request * +static struct sba_request * sba_prep_dma_xor_req(struct sba_device *sba, dma_addr_t off, dma_addr_t dst, dma_addr_t *src, u32 src_cnt, size_t len, unsigned long flags) @@ -793,7 +872,8 @@ sba_prep_dma_xor_req(struct sba_device *sba, req = sba_alloc_request(sba); if (!req) return NULL; - req->fence = (flags & DMA_PREP_FENCE) ? true : false; + if (flags & DMA_PREP_FENCE) + req->flags |= SBA_REQUEST_FENCE; /* Fillup request message */ sba_fillup_xor_msg(req, req->cmds, &req->msg, @@ -854,6 +934,7 @@ static void sba_fillup_pq_msg(struct sba_request *req, u64 cmd; u32 c_mdata; unsigned int i; + dma_addr_t resp_dma = req->tx.phys; struct brcm_sba_command *cmdsp = cmds; if (pq_continue) { @@ -947,7 +1028,7 @@ static void sba_fillup_pq_msg(struct sba_request *req, cmdsp->flags = BRCM_SBA_CMD_TYPE_A; if (req->sba->hw_resp_size) { cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; - cmdsp->resp = req->resp_dma; + cmdsp->resp = resp_dma; cmdsp->resp_len = req->sba->hw_resp_size; } cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; @@ -974,7 +1055,7 @@ static void sba_fillup_pq_msg(struct sba_request *req, cmdsp->flags = BRCM_SBA_CMD_TYPE_A; if (req->sba->hw_resp_size) { cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; - cmdsp->resp = req->resp_dma; + cmdsp->resp = resp_dma; cmdsp->resp_len = req->sba->hw_resp_size; } cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; @@ -991,7 +1072,7 @@ static void sba_fillup_pq_msg(struct sba_request *req, msg->error = 0; } -struct sba_request * +static struct sba_request * sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off, dma_addr_t *dst_p, dma_addr_t *dst_q, dma_addr_t *src, u32 src_cnt, const u8 *scf, size_t len, unsigned long flags) @@ -1002,7 +1083,8 @@ sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off, req = sba_alloc_request(sba); if (!req) return NULL; - req->fence = (flags & DMA_PREP_FENCE) ? true : false; + if (flags & DMA_PREP_FENCE) + req->flags |= SBA_REQUEST_FENCE; /* Fillup request messages */ sba_fillup_pq_msg(req, dmaf_continue(flags), @@ -1027,6 +1109,7 @@ static void sba_fillup_pq_single_msg(struct sba_request *req, u64 cmd; u32 c_mdata; u8 pos, dpos = raid6_gflog[scf]; + dma_addr_t resp_dma = req->tx.phys; struct brcm_sba_command *cmdsp = cmds; if (!dst_p) @@ -1105,7 +1188,7 @@ static void sba_fillup_pq_single_msg(struct sba_request *req, cmdsp->flags = BRCM_SBA_CMD_TYPE_A; if (req->sba->hw_resp_size) { cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; - cmdsp->resp = req->resp_dma; + cmdsp->resp = resp_dma; cmdsp->resp_len = req->sba->hw_resp_size; } cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; @@ -1226,7 +1309,7 @@ skip_q_computation: cmdsp->flags = BRCM_SBA_CMD_TYPE_A; if (req->sba->hw_resp_size) { cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; - cmdsp->resp = req->resp_dma; + cmdsp->resp = resp_dma; cmdsp->resp_len = req->sba->hw_resp_size; } cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; @@ -1243,7 +1326,7 @@ skip_q: msg->error = 0; } -struct sba_request * +static struct sba_request * sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off, dma_addr_t *dst_p, dma_addr_t *dst_q, dma_addr_t src, u8 scf, size_t len, @@ -1255,7 +1338,8 @@ sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off, req = sba_alloc_request(sba); if (!req) return NULL; - req->fence = (flags & DMA_PREP_FENCE) ? true : false; + if (flags & DMA_PREP_FENCE) + req->flags |= SBA_REQUEST_FENCE; /* Fillup request messages */ sba_fillup_pq_single_msg(req, dmaf_continue(flags), @@ -1370,40 +1454,10 @@ fail: /* ====== Mailbox callbacks ===== */ -static void sba_dma_tx_actions(struct sba_request *req) -{ - struct dma_async_tx_descriptor *tx = &req->tx; - - WARN_ON(tx->cookie < 0); - - if (tx->cookie > 0) { - dma_cookie_complete(tx); - - /* - * Call the callback (must not sleep or submit new - * operations to this channel) - */ - if (tx->callback) - tx->callback(tx->callback_param); - - dma_descriptor_unmap(tx); - } - - /* Run dependent operations */ - dma_run_dependencies(tx); - - /* If waiting for 'ack' then move to completed list */ - if (!async_tx_test_ack(&req->tx)) - sba_complete_chained_requests(req); - else - sba_free_chained_requests(req); -} - static void sba_receive_message(struct mbox_client *cl, void *msg) { - unsigned long flags; struct brcm_message *m = msg; - struct sba_request *req = m->ctx, *req1; + struct sba_request *req = m->ctx; struct sba_device *sba = req->sba; /* Error count if message has error */ @@ -1411,52 +1465,37 @@ static void sba_receive_message(struct mbox_client *cl, void *msg) dev_err(sba->dev, "%s got message with error %d", dma_chan_name(&sba->dma_chan), m->error); - /* Mark request as received */ - sba_received_request(req); - - /* Wait for all chained requests to be completed */ - if (atomic_dec_return(&req->first->next_pending_count)) - goto done; - - /* Point to first request */ - req = req->first; - - /* Update request */ - if (req->state == SBA_REQUEST_STATE_RECEIVED) - sba_dma_tx_actions(req); - else - sba_free_chained_requests(req); + /* Process received request */ + sba_process_received_request(sba, req); +} - spin_lock_irqsave(&sba->reqs_lock, flags); +/* ====== Debugfs callbacks ====== */ - /* Re-check all completed request waiting for 'ack' */ - list_for_each_entry_safe(req, req1, &sba->reqs_completed_list, node) { - spin_unlock_irqrestore(&sba->reqs_lock, flags); - sba_dma_tx_actions(req); - spin_lock_irqsave(&sba->reqs_lock, flags); - } +static int sba_debugfs_stats_show(struct seq_file *file, void *offset) +{ + struct platform_device *pdev = to_platform_device(file->private); + struct sba_device *sba = platform_get_drvdata(pdev); - spin_unlock_irqrestore(&sba->reqs_lock, flags); + /* Write stats in file */ + sba_write_stats_in_seqfile(sba, file); -done: - /* Try to submit pending request */ - sba_issue_pending(&sba->dma_chan); + return 0; } /* ====== Platform driver routines ===== */ static int sba_prealloc_channel_resources(struct sba_device *sba) { - int i, j, p, ret = 0; + int i, j, ret = 0; struct sba_request *req = NULL; - sba->resp_base = dma_alloc_coherent(sba->dma_dev.dev, + sba->resp_base = dma_alloc_coherent(sba->mbox_dev, sba->max_resp_pool_size, &sba->resp_dma_base, GFP_KERNEL); if (!sba->resp_base) return -ENOMEM; - sba->cmds_base = dma_alloc_coherent(sba->dma_dev.dev, + sba->cmds_base = dma_alloc_coherent(sba->mbox_dev, sba->max_cmds_pool_size, &sba->cmds_dma_base, GFP_KERNEL); if (!sba->cmds_base) { @@ -1469,36 +1508,23 @@ static int sba_prealloc_channel_resources(struct sba_device *sba) INIT_LIST_HEAD(&sba->reqs_alloc_list); INIT_LIST_HEAD(&sba->reqs_pending_list); INIT_LIST_HEAD(&sba->reqs_active_list); - INIT_LIST_HEAD(&sba->reqs_received_list); - INIT_LIST_HEAD(&sba->reqs_completed_list); INIT_LIST_HEAD(&sba->reqs_aborted_list); INIT_LIST_HEAD(&sba->reqs_free_list); - sba->reqs = devm_kcalloc(sba->dev, sba->max_req, - sizeof(*req), GFP_KERNEL); - if (!sba->reqs) { - ret = -ENOMEM; - goto fail_free_cmds_pool; - } - - for (i = 0, p = 0; i < sba->max_req; i++) { - req = &sba->reqs[i]; + for (i = 0; i < sba->max_req; i++) { + req = devm_kzalloc(sba->dev, + sizeof(*req) + + sba->max_cmd_per_req * sizeof(req->cmds[0]), + GFP_KERNEL); + if (!req) { + ret = -ENOMEM; + goto fail_free_cmds_pool; + } INIT_LIST_HEAD(&req->node); req->sba = sba; - req->state = SBA_REQUEST_STATE_FREE; + req->flags = SBA_REQUEST_STATE_FREE; INIT_LIST_HEAD(&req->next); - req->next_count = 1; atomic_set(&req->next_pending_count, 0); - req->fence = false; - req->resp = sba->resp_base + p; - req->resp_dma = sba->resp_dma_base + p; - p += sba->hw_resp_size; - req->cmds = devm_kcalloc(sba->dev, sba->max_cmd_per_req, - sizeof(*req->cmds), GFP_KERNEL); - if (!req->cmds) { - ret = -ENOMEM; - goto fail_free_cmds_pool; - } for (j = 0; j < sba->max_cmd_per_req; j++) { req->cmds[j].cmd = 0; req->cmds[j].cmd_dma = sba->cmds_base + @@ -1509,21 +1535,20 @@ static int sba_prealloc_channel_resources(struct sba_device *sba) } memset(&req->msg, 0, sizeof(req->msg)); dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan); + async_tx_ack(&req->tx); req->tx.tx_submit = sba_tx_submit; - req->tx.phys = req->resp_dma; + req->tx.phys = sba->resp_dma_base + i * sba->hw_resp_size; list_add_tail(&req->node, &sba->reqs_free_list); } - sba->reqs_free_count = sba->max_req; - return 0; fail_free_cmds_pool: - dma_free_coherent(sba->dma_dev.dev, + dma_free_coherent(sba->mbox_dev, sba->max_cmds_pool_size, sba->cmds_base, sba->cmds_dma_base); fail_free_resp_pool: - dma_free_coherent(sba->dma_dev.dev, + dma_free_coherent(sba->mbox_dev, sba->max_resp_pool_size, sba->resp_base, sba->resp_dma_base); return ret; @@ -1532,9 +1557,9 @@ fail_free_resp_pool: static void sba_freeup_channel_resources(struct sba_device *sba) { dmaengine_terminate_all(&sba->dma_chan); - dma_free_coherent(sba->dma_dev.dev, sba->max_cmds_pool_size, + dma_free_coherent(sba->mbox_dev, sba->max_cmds_pool_size, sba->cmds_base, sba->cmds_dma_base); - dma_free_coherent(sba->dma_dev.dev, sba->max_resp_pool_size, + dma_free_coherent(sba->mbox_dev, sba->max_resp_pool_size, sba->resp_base, sba->resp_dma_base); sba->resp_base = NULL; sba->resp_dma_base = 0; @@ -1625,6 +1650,13 @@ static int sba_probe(struct platform_device *pdev) sba->dev = &pdev->dev; platform_set_drvdata(pdev, sba); + /* Number of channels equals number of mailbox channels */ + ret = of_count_phandle_with_args(pdev->dev.of_node, + "mboxes", "#mbox-cells"); + if (ret <= 0) + return -ENODEV; + mchans_count = ret; + /* Determine SBA version from DT compatible string */ if (of_device_is_compatible(sba->dev->of_node, "brcm,iproc-sba")) sba->ver = SBA_VER_1; @@ -1637,14 +1669,12 @@ static int sba_probe(struct platform_device *pdev) /* Derived Configuration parameters */ switch (sba->ver) { case SBA_VER_1: - sba->max_req = 1024; sba->hw_buf_size = 4096; sba->hw_resp_size = 8; sba->max_pq_coefs = 6; sba->max_pq_srcs = 6; break; case SBA_VER_2: - sba->max_req = 1024; sba->hw_buf_size = 4096; sba->hw_resp_size = 8; sba->max_pq_coefs = 30; @@ -1658,6 +1688,7 @@ static int sba_probe(struct platform_device *pdev) default: return -EINVAL; } + sba->max_req = SBA_MAX_REQ_PER_MBOX_CHANNEL * mchans_count; sba->max_cmd_per_req = sba->max_pq_srcs + 3; sba->max_xor_srcs = sba->max_cmd_per_req - 1; sba->max_resp_pool_size = sba->max_req * sba->hw_resp_size; @@ -1668,25 +1699,17 @@ static int sba_probe(struct platform_device *pdev) sba->client.dev = &pdev->dev; sba->client.rx_callback = sba_receive_message; sba->client.tx_block = false; - sba->client.knows_txdone = false; + sba->client.knows_txdone = true; sba->client.tx_tout = 0; - /* Number of channels equals number of mailbox channels */ - ret = of_count_phandle_with_args(pdev->dev.of_node, - "mboxes", "#mbox-cells"); - if (ret <= 0) - return -ENODEV; - mchans_count = ret; - sba->mchans_count = 0; - atomic_set(&sba->mchans_current, 0); - /* Allocate mailbox channel array */ - sba->mchans = devm_kcalloc(&pdev->dev, sba->mchans_count, + sba->mchans = devm_kcalloc(&pdev->dev, mchans_count, sizeof(*sba->mchans), GFP_KERNEL); if (!sba->mchans) return -ENOMEM; /* Request mailbox channels */ + sba->mchans_count = 0; for (i = 0; i < mchans_count; i++) { sba->mchans[i] = mbox_request_channel(&sba->client, i); if (IS_ERR(sba->mchans[i])) { @@ -1695,6 +1718,7 @@ static int sba_probe(struct platform_device *pdev) } sba->mchans_count++; } + atomic_set(&sba->mchans_current, 0); /* Find-out underlying mailbox device */ ret = of_parse_phandle_with_args(pdev->dev.of_node, @@ -1723,15 +1747,34 @@ static int sba_probe(struct platform_device *pdev) } } - /* Register DMA device with linux async framework */ - ret = sba_async_register(sba); + /* Prealloc channel resource */ + ret = sba_prealloc_channel_resources(sba); if (ret) goto fail_free_mchans; - /* Prealloc channel resource */ - ret = sba_prealloc_channel_resources(sba); + /* Check availability of debugfs */ + if (!debugfs_initialized()) + goto skip_debugfs; + + /* Create debugfs root entry */ + sba->root = debugfs_create_dir(dev_name(sba->dev), NULL); + if (IS_ERR_OR_NULL(sba->root)) { + dev_err(sba->dev, "failed to create debugfs root entry\n"); + sba->root = NULL; + goto skip_debugfs; + } + + /* Create debugfs stats entry */ + sba->stats = debugfs_create_devm_seqfile(sba->dev, "stats", sba->root, + sba_debugfs_stats_show); + if (IS_ERR_OR_NULL(sba->stats)) + dev_err(sba->dev, "failed to create debugfs stats file\n"); +skip_debugfs: + + /* Register DMA device with Linux async framework */ + ret = sba_async_register(sba); if (ret) - goto fail_async_dev_unreg; + goto fail_free_resources; /* Print device info */ dev_info(sba->dev, "%s using SBAv%d and %d mailbox channels", @@ -1740,8 +1783,9 @@ static int sba_probe(struct platform_device *pdev) return 0; -fail_async_dev_unreg: - dma_async_device_unregister(&sba->dma_dev); +fail_free_resources: + debugfs_remove_recursive(sba->root); + sba_freeup_channel_resources(sba); fail_free_mchans: for (i = 0; i < sba->mchans_count; i++) mbox_free_channel(sba->mchans[i]); @@ -1753,10 +1797,12 @@ static int sba_remove(struct platform_device *pdev) int i; struct sba_device *sba = platform_get_drvdata(pdev); - sba_freeup_channel_resources(sba); - dma_async_device_unregister(&sba->dma_dev); + debugfs_remove_recursive(sba->root); + + sba_freeup_channel_resources(sba); + for (i = 0; i < sba->mchans_count; i++) mbox_free_channel(sba->mchans[i]); |