xprtrmda: Reduce calls to ib_poll_cq() in completion handlers

Change the completion handlers to grab up to 16 items per
ib_poll_cq() call. No extra ib_poll_cq() is needed if fewer than 16
items are returned.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>

diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index c7d5281..b8caee9 100644 (file)
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -162,14 +162,23 @@ rpcrdma_sendcq_process_wc(struct ib_wc *wc)
 }
 
 static int
-rpcrdma_sendcq_poll(struct ib_cq *cq)
+rpcrdma_sendcq_poll(struct ib_cq *cq, struct rpcrdma_ep *ep)
 {
-       struct ib_wc wc;
-       int rc;
+       struct ib_wc *wcs;
+       int count, rc;
 
-       while ((rc = ib_poll_cq(cq, 1, &wc)) == 1)
-               rpcrdma_sendcq_process_wc(&wc);
-       return rc;
+       do {
+               wcs = ep->rep_send_wcs;
+
+               rc = ib_poll_cq(cq, RPCRDMA_POLLSIZE, wcs);
+               if (rc <= 0)
+                       return rc;
+
+               count = rc;
+               while (count-- > 0)
+                       rpcrdma_sendcq_process_wc(wcs++);
+       } while (rc == RPCRDMA_POLLSIZE);
+       return 0;
 }
 
 /*
@@ -183,9 +192,10 @@ rpcrdma_sendcq_poll(struct ib_cq *cq)
 static void
 rpcrdma_sendcq_upcall(struct ib_cq *cq, void *cq_context)
 {
+       struct rpcrdma_ep *ep = (struct rpcrdma_ep *)cq_context;
        int rc;
 
-       rc = rpcrdma_sendcq_poll(cq);
+       rc = rpcrdma_sendcq_poll(cq, ep);
        if (rc) {
                dprintk("RPC:       %s: ib_poll_cq failed: %i\n",
                        __func__, rc);
@@ -202,7 +212,7 @@ rpcrdma_sendcq_upcall(struct ib_cq *cq, void *cq_context)
                return;
        }
 
-       rpcrdma_sendcq_poll(cq);
+       rpcrdma_sendcq_poll(cq, ep);
 }
 
 static void
@@ -241,14 +251,23 @@ out_schedule:
 }
 
 static int
-rpcrdma_recvcq_poll(struct ib_cq *cq)
+rpcrdma_recvcq_poll(struct ib_cq *cq, struct rpcrdma_ep *ep)
 {
-       struct ib_wc wc;
-       int rc;
+       struct ib_wc *wcs;
+       int count, rc;
 
-       while ((rc = ib_poll_cq(cq, 1, &wc)) == 1)
-               rpcrdma_recvcq_process_wc(&wc);
-       return rc;
+       do {
+               wcs = ep->rep_recv_wcs;
+
+               rc = ib_poll_cq(cq, RPCRDMA_POLLSIZE, wcs);
+               if (rc <= 0)
+                       return rc;
+
+               count = rc;
+               while (count-- > 0)
+                       rpcrdma_recvcq_process_wc(wcs++);
+       } while (rc == RPCRDMA_POLLSIZE);
+       return 0;
 }
 
 /*
@@ -266,9 +285,10 @@ rpcrdma_recvcq_poll(struct ib_cq *cq)
 static void
 rpcrdma_recvcq_upcall(struct ib_cq *cq, void *cq_context)
 {
+       struct rpcrdma_ep *ep = (struct rpcrdma_ep *)cq_context;
        int rc;
 
-       rc = rpcrdma_recvcq_poll(cq);
+       rc = rpcrdma_recvcq_poll(cq, ep);
        if (rc) {
                dprintk("RPC:       %s: ib_poll_cq failed: %i\n",
                        __func__, rc);
@@ -285,7 +305,7 @@ rpcrdma_recvcq_upcall(struct ib_cq *cq, void *cq_context)
                return;
        }
 
-       rpcrdma_recvcq_poll(cq);
+       rpcrdma_recvcq_poll(cq, ep);
 }
 
 #ifdef RPC_DEBUG
@@ -721,7 +741,7 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
        INIT_DELAYED_WORK(&ep->rep_connect_worker, rpcrdma_connect_worker);
 
        sendcq = ib_create_cq(ia->ri_id->device, rpcrdma_sendcq_upcall,
-                                 rpcrdma_cq_async_error_upcall, NULL,
+                                 rpcrdma_cq_async_error_upcall, ep,
                                  ep->rep_attr.cap.max_send_wr + 1, 0);
        if (IS_ERR(sendcq)) {
                rc = PTR_ERR(sendcq);
@@ -738,7 +758,7 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
        }
 
        recvcq = ib_create_cq(ia->ri_id->device, rpcrdma_recvcq_upcall,
-                                 rpcrdma_cq_async_error_upcall, NULL,
+                                 rpcrdma_cq_async_error_upcall, ep,
                                  ep->rep_attr.cap.max_recv_wr + 1, 0);
        if (IS_ERR(recvcq)) {
                rc = PTR_ERR(recvcq);

diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index 334ab6e..cb4c882 100644 (file)
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -74,6 +74,8 @@ struct rpcrdma_ia {
  * RDMA Endpoint -- one per transport instance
  */
 
+#define RPCRDMA_POLLSIZE       (16)
+
 struct rpcrdma_ep {
        atomic_t                rep_cqcount;
        int                     rep_cqinit;
@@ -88,6 +90,8 @@ struct rpcrdma_ep {
        struct rdma_conn_param  rep_remote_cma;
        struct sockaddr_storage rep_remote_addr;
        struct delayed_work     rep_connect_worker;
+       struct ib_wc            rep_send_wcs[RPCRDMA_POLLSIZE];
+       struct ib_wc            rep_recv_wcs[RPCRDMA_POLLSIZE];
 };
 
 #define INIT_CQCOUNT(ep) atomic_set(&(ep)->rep_cqcount, (ep)->rep_cqinit)