int src_niov, const lnet_kiov_t *src,
unsigned int offset, unsigned int len);
-void lnet_copy_iov2iov(unsigned int ndiov, const struct kvec *diov,
- unsigned int doffset,
+void lnet_copy_iov2iter(struct iov_iter *to,
unsigned int nsiov, const struct kvec *siov,
unsigned int soffset, unsigned int nob);
-void lnet_copy_kiov2iov(unsigned int niov, const struct kvec *iov,
- unsigned int iovoffset,
+void lnet_copy_kiov2iter(struct iov_iter *to,
unsigned int nkiov, const lnet_kiov_t *kiov,
unsigned int kiovoffset, unsigned int nob);
-void lnet_copy_iov2kiov(unsigned int nkiov, const lnet_kiov_t *kiov,
- unsigned int kiovoffset,
- unsigned int niov, const struct kvec *iov,
- unsigned int iovoffset, unsigned int nob);
-void lnet_copy_kiov2kiov(unsigned int ndkiov, const lnet_kiov_t *dkiov,
- unsigned int doffset,
- unsigned int nskiov, const lnet_kiov_t *skiov,
- unsigned int soffset, unsigned int nob);
-
-static inline void
-lnet_copy_iov2flat(int dlen, void *dest, unsigned int doffset,
- unsigned int nsiov, const struct kvec *siov, unsigned int soffset,
- unsigned int nob)
-{
- struct kvec diov = {/*.iov_base = */ dest, /*.iov_len = */ dlen};
-
- lnet_copy_iov2iov(1, &diov, doffset,
- nsiov, siov, soffset, nob);
-}
-
-static inline void
-lnet_copy_kiov2flat(int dlen, void *dest, unsigned int doffset,
- unsigned int nsiov, const lnet_kiov_t *skiov,
- unsigned int soffset, unsigned int nob)
-{
- struct kvec diov = {/* .iov_base = */ dest, /* .iov_len = */ dlen};
-
- lnet_copy_kiov2iov(1, &diov, doffset,
- nsiov, skiov, soffset, nob);
-}
-
-static inline void
-lnet_copy_flat2iov(unsigned int ndiov, const struct kvec *diov, unsigned int doffset,
- int slen, void *src, unsigned int soffset, unsigned int nob)
-{
- struct kvec siov = {/*.iov_base = */ src, /*.iov_len = */slen};
-
- lnet_copy_iov2iov(ndiov, diov, doffset,
- 1, &siov, soffset, nob);
-}
-
-static inline void
-lnet_copy_flat2kiov(unsigned int ndiov, const lnet_kiov_t *dkiov,
- unsigned int doffset, int slen, void *src,
- unsigned int soffset, unsigned int nob)
-{
- struct kvec siov = {/* .iov_base = */ src, /* .iov_len = */ slen};
-
- lnet_copy_iov2kiov(ndiov, dkiov, doffset,
- 1, &siov, soffset, nob);
-}
void lnet_me_unlink(lnet_me_t *me);
lnet_kiov_t *payload_kiov = lntmsg->msg_kiov;
unsigned int payload_offset = lntmsg->msg_offset;
unsigned int payload_nob = lntmsg->msg_len;
+ struct iov_iter from;
struct kib_msg *ibmsg;
struct kib_rdma_desc *rd;
struct kib_tx *tx;
/* payload is either all vaddrs or all pages */
LASSERT(!(payload_kiov && payload_iov));
+ if (payload_kiov)
+ iov_iter_bvec(&from, ITER_BVEC | WRITE,
+ payload_kiov, payload_niov,
+ payload_nob + payload_offset);
+ else
+ iov_iter_kvec(&from, ITER_KVEC | WRITE,
+ payload_iov, payload_niov,
+ payload_nob + payload_offset);
+
+ iov_iter_advance(&from, payload_offset);
+
switch (type) {
default:
LBUG();
ibmsg = tx->tx_msg;
ibmsg->ibm_u.immediate.ibim_hdr = *hdr;
- if (payload_kiov)
- lnet_copy_kiov2flat(IBLND_MSG_SIZE, ibmsg,
- offsetof(struct kib_msg, ibm_u.immediate.ibim_payload),
- payload_niov, payload_kiov,
- payload_offset, payload_nob);
- else
- lnet_copy_iov2flat(IBLND_MSG_SIZE, ibmsg,
- offsetof(struct kib_msg, ibm_u.immediate.ibim_payload),
- payload_niov, payload_iov,
- payload_offset, payload_nob);
-
+ copy_from_iter(&ibmsg->ibm_u.immediate.ibim_payload, IBLND_MSG_SIZE,
+ &from);
nob = offsetof(struct kib_immediate_msg, ibim_payload[payload_nob]);
kiblnd_init_tx_msg(ni, tx, IBLND_MSG_IMMEDIATE, nob);
break;
}
- if (to->type & ITER_BVEC)
- lnet_copy_flat2kiov(to->nr_segs, to->bvec, to->iov_offset,
- IBLND_MSG_SIZE, rxmsg,
- offsetof(struct kib_msg, ibm_u.immediate.ibim_payload),
- iov_iter_count(to));
- else
- lnet_copy_flat2iov(to->nr_segs, to->kvec, to->iov_offset,
- IBLND_MSG_SIZE, rxmsg,
- offsetof(struct kib_msg, ibm_u.immediate.ibim_payload),
- iov_iter_count(to));
+ copy_to_iter(&rxmsg->ibm_u.immediate.ibim_payload,
+ IBLND_MSG_SIZE, to);
lnet_finalize(ni, lntmsg, 0);
break;
EXPORT_SYMBOL(lnet_iov_nob);
void
-lnet_copy_iov2iov(unsigned int ndiov, const struct kvec *diov, unsigned int doffset,
- unsigned int nsiov, const struct kvec *siov, unsigned int soffset,
- unsigned int nob)
+lnet_copy_iov2iter(struct iov_iter *to,
+ unsigned int nsiov, const struct kvec *siov,
+ unsigned int soffset, unsigned int nob)
{
/* NB diov, siov are READ-ONLY */
- unsigned int this_nob;
+ const char *s;
+ size_t left;
if (!nob)
return;
- /* skip complete frags before 'doffset' */
- LASSERT(ndiov > 0);
- while (doffset >= diov->iov_len) {
- doffset -= diov->iov_len;
- diov++;
- ndiov--;
- LASSERT(ndiov > 0);
- }
-
/* skip complete frags before 'soffset' */
LASSERT(nsiov > 0);
while (soffset >= siov->iov_len) {
LASSERT(nsiov > 0);
}
+ s = (char *)siov->iov_base + soffset;
+ left = siov->iov_len - soffset;
do {
- LASSERT(ndiov > 0);
+ size_t n, copy = left;
LASSERT(nsiov > 0);
- this_nob = min(diov->iov_len - doffset,
- siov->iov_len - soffset);
- this_nob = min(this_nob, nob);
- memcpy((char *)diov->iov_base + doffset,
- (char *)siov->iov_base + soffset, this_nob);
- nob -= this_nob;
+ if (copy > nob)
+ copy = nob;
+ n = copy_to_iter(s, copy, to);
+ if (n != copy)
+ return;
+ nob -= n;
- if (diov->iov_len > doffset + this_nob) {
- doffset += this_nob;
- } else {
- diov++;
- ndiov--;
- doffset = 0;
- }
+ siov++;
+ s = (char *)siov->iov_base;
+ left = siov->iov_len;
+ nsiov--;
+ } while (nob > 0);
+}
+EXPORT_SYMBOL(lnet_copy_iov2iter);
- if (siov->iov_len > soffset + this_nob) {
- soffset += this_nob;
- } else {
- siov++;
- nsiov--;
- soffset = 0;
- }
+void
+lnet_copy_kiov2iter(struct iov_iter *to,
+ unsigned int nsiov, const lnet_kiov_t *siov,
+ unsigned int soffset, unsigned int nob)
+{
+ if (!nob)
+ return;
+
+ LASSERT(!in_interrupt());
+
+ LASSERT(nsiov > 0);
+ while (soffset >= siov->bv_len) {
+ soffset -= siov->bv_len;
+ siov++;
+ nsiov--;
+ LASSERT(nsiov > 0);
+ }
+
+ do {
+ size_t copy = siov->bv_len - soffset, n;
+
+ LASSERT(nsiov > 0);
+
+ if (copy > nob)
+ copy = nob;
+ n = copy_page_to_iter(siov->bv_page,
+ siov->bv_offset + soffset,
+ copy, to);
+ if (n != copy)
+ return;
+ nob -= n;
+ siov++;
+ nsiov--;
+ soffset = 0;
} while (nob > 0);
}
-EXPORT_SYMBOL(lnet_copy_iov2iov);
+EXPORT_SYMBOL(lnet_copy_kiov2iter);
int
lnet_extract_iov(int dst_niov, struct kvec *dst,
}
EXPORT_SYMBOL(lnet_kiov_nob);
-void
-lnet_copy_kiov2kiov(unsigned int ndiov, const lnet_kiov_t *diov, unsigned int doffset,
- unsigned int nsiov, const lnet_kiov_t *siov, unsigned int soffset,
- unsigned int nob)
-{
- /* NB diov, siov are READ-ONLY */
- unsigned int this_nob;
- char *daddr = NULL;
- char *saddr = NULL;
-
- if (!nob)
- return;
-
- LASSERT(!in_interrupt());
-
- LASSERT(ndiov > 0);
- while (doffset >= diov->bv_len) {
- doffset -= diov->bv_len;
- diov++;
- ndiov--;
- LASSERT(ndiov > 0);
- }
-
- LASSERT(nsiov > 0);
- while (soffset >= siov->bv_len) {
- soffset -= siov->bv_len;
- siov++;
- nsiov--;
- LASSERT(nsiov > 0);
- }
-
- do {
- LASSERT(ndiov > 0);
- LASSERT(nsiov > 0);
- this_nob = min(diov->bv_len - doffset,
- siov->bv_len - soffset);
- this_nob = min(this_nob, nob);
-
- if (!daddr)
- daddr = ((char *)kmap(diov->bv_page)) +
- diov->bv_offset + doffset;
- if (!saddr)
- saddr = ((char *)kmap(siov->bv_page)) +
- siov->bv_offset + soffset;
-
- /*
- * Vanishing risk of kmap deadlock when mapping 2 pages.
- * However in practice at least one of the kiovs will be mapped
- * kernel pages and the map/unmap will be NOOPs
- */
- memcpy(daddr, saddr, this_nob);
- nob -= this_nob;
-
- if (diov->bv_len > doffset + this_nob) {
- daddr += this_nob;
- doffset += this_nob;
- } else {
- kunmap(diov->bv_page);
- daddr = NULL;
- diov++;
- ndiov--;
- doffset = 0;
- }
-
- if (siov->bv_len > soffset + this_nob) {
- saddr += this_nob;
- soffset += this_nob;
- } else {
- kunmap(siov->bv_page);
- saddr = NULL;
- siov++;
- nsiov--;
- soffset = 0;
- }
- } while (nob > 0);
-
- if (daddr)
- kunmap(diov->bv_page);
- if (saddr)
- kunmap(siov->bv_page);
-}
-EXPORT_SYMBOL(lnet_copy_kiov2kiov);
-
-void
-lnet_copy_kiov2iov(unsigned int niov, const struct kvec *iov, unsigned int iovoffset,
- unsigned int nkiov, const lnet_kiov_t *kiov,
- unsigned int kiovoffset, unsigned int nob)
-{
- /* NB iov, kiov are READ-ONLY */
- unsigned int this_nob;
- char *addr = NULL;
-
- if (!nob)
- return;
-
- LASSERT(!in_interrupt());
-
- LASSERT(niov > 0);
- while (iovoffset >= iov->iov_len) {
- iovoffset -= iov->iov_len;
- iov++;
- niov--;
- LASSERT(niov > 0);
- }
-
- LASSERT(nkiov > 0);
- while (kiovoffset >= kiov->bv_len) {
- kiovoffset -= kiov->bv_len;
- kiov++;
- nkiov--;
- LASSERT(nkiov > 0);
- }
-
- do {
- LASSERT(niov > 0);
- LASSERT(nkiov > 0);
- this_nob = min(iov->iov_len - iovoffset,
- (__kernel_size_t)kiov->bv_len - kiovoffset);
- this_nob = min(this_nob, nob);
-
- if (!addr)
- addr = ((char *)kmap(kiov->bv_page)) +
- kiov->bv_offset + kiovoffset;
-
- memcpy((char *)iov->iov_base + iovoffset, addr, this_nob);
- nob -= this_nob;
-
- if (iov->iov_len > iovoffset + this_nob) {
- iovoffset += this_nob;
- } else {
- iov++;
- niov--;
- iovoffset = 0;
- }
-
- if (kiov->bv_len > kiovoffset + this_nob) {
- addr += this_nob;
- kiovoffset += this_nob;
- } else {
- kunmap(kiov->bv_page);
- addr = NULL;
- kiov++;
- nkiov--;
- kiovoffset = 0;
- }
-
- } while (nob > 0);
-
- if (addr)
- kunmap(kiov->bv_page);
-}
-EXPORT_SYMBOL(lnet_copy_kiov2iov);
-
-void
-lnet_copy_iov2kiov(unsigned int nkiov, const lnet_kiov_t *kiov,
- unsigned int kiovoffset, unsigned int niov,
- const struct kvec *iov, unsigned int iovoffset,
- unsigned int nob)
-{
- /* NB kiov, iov are READ-ONLY */
- unsigned int this_nob;
- char *addr = NULL;
-
- if (!nob)
- return;
-
- LASSERT(!in_interrupt());
-
- LASSERT(nkiov > 0);
- while (kiovoffset >= kiov->bv_len) {
- kiovoffset -= kiov->bv_len;
- kiov++;
- nkiov--;
- LASSERT(nkiov > 0);
- }
-
- LASSERT(niov > 0);
- while (iovoffset >= iov->iov_len) {
- iovoffset -= iov->iov_len;
- iov++;
- niov--;
- LASSERT(niov > 0);
- }
-
- do {
- LASSERT(nkiov > 0);
- LASSERT(niov > 0);
- this_nob = min((__kernel_size_t)kiov->bv_len - kiovoffset,
- iov->iov_len - iovoffset);
- this_nob = min(this_nob, nob);
-
- if (!addr)
- addr = ((char *)kmap(kiov->bv_page)) +
- kiov->bv_offset + kiovoffset;
-
- memcpy(addr, (char *)iov->iov_base + iovoffset, this_nob);
- nob -= this_nob;
-
- if (kiov->bv_len > kiovoffset + this_nob) {
- addr += this_nob;
- kiovoffset += this_nob;
- } else {
- kunmap(kiov->bv_page);
- addr = NULL;
- kiov++;
- nkiov--;
- kiovoffset = 0;
- }
-
- if (iov->iov_len > iovoffset + this_nob) {
- iovoffset += this_nob;
- } else {
- iov++;
- niov--;
- iovoffset = 0;
- }
- } while (nob > 0);
-
- if (addr)
- kunmap(kiov->bv_page);
-}
-EXPORT_SYMBOL(lnet_copy_iov2kiov);
-
int
lnet_extract_kiov(int dst_niov, lnet_kiov_t *dst,
int src_niov, const lnet_kiov_t *src,
lnet_msg_t *sendmsg = private;
if (lntmsg) { /* not discarding */
- if (sendmsg->msg_iov) {
- if (to->type & ITER_KVEC)
- lnet_copy_iov2iov(to->nr_segs, to->kvec, to->iov_offset,
- sendmsg->msg_niov,
- sendmsg->msg_iov,
- sendmsg->msg_offset, iov_iter_count(to));
- else
- lnet_copy_iov2kiov(to->nr_segs, to->bvec, to->iov_offset,
- sendmsg->msg_niov,
- sendmsg->msg_iov,
- sendmsg->msg_offset, iov_iter_count(to));
- } else {
- if (to->type & ITER_KVEC)
- lnet_copy_kiov2iov(to->nr_segs, to->kvec, to->iov_offset,
- sendmsg->msg_niov,
- sendmsg->msg_kiov,
- sendmsg->msg_offset, iov_iter_count(to));
- else
- lnet_copy_kiov2kiov(to->nr_segs, to->bvec, to->iov_offset,
- sendmsg->msg_niov,
- sendmsg->msg_kiov,
- sendmsg->msg_offset, iov_iter_count(to));
- }
+ if (sendmsg->msg_iov)
+ lnet_copy_iov2iter(to,
+ sendmsg->msg_niov,
+ sendmsg->msg_iov,
+ sendmsg->msg_offset,
+ iov_iter_count(to));
+ else
+ lnet_copy_kiov2iter(to,
+ sendmsg->msg_niov,
+ sendmsg->msg_kiov,
+ sendmsg->msg_offset,
+ iov_iter_count(to));
lnet_finalize(ni, lntmsg, 0);
}
projects / cascardo / linux.git / commitdiff