2 * Copyright (c) 2013, Mellanox Technologies inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/kref.h>
35 #include <linux/random.h>
36 #include <linux/debugfs.h>
37 #include <linux/export.h>
38 #include <linux/delay.h>
39 #include <rdma/ib_umem.h>
43 MAX_PENDING_REG_MR = 8,
50 static __be64 *mr_align(__be64 *ptr, int align)
52 unsigned long mask = align - 1;
54 return (__be64 *)(((unsigned long)ptr + mask) & ~mask);
57 static int order2idx(struct mlx5_ib_dev *dev, int order)
59 struct mlx5_mr_cache *cache = &dev->cache;
61 if (order < cache->ent[0].order)
64 return order - cache->ent[0].order;
67 static void reg_mr_callback(int status, void *context)
69 struct mlx5_ib_mr *mr = context;
70 struct mlx5_ib_dev *dev = mr->dev;
71 struct mlx5_mr_cache *cache = &dev->cache;
72 int c = order2idx(dev, mr->order);
73 struct mlx5_cache_ent *ent = &cache->ent[c];
76 struct mlx5_mr_table *table = &dev->mdev->priv.mr_table;
79 spin_lock_irqsave(&ent->lock, flags);
81 spin_unlock_irqrestore(&ent->lock, flags);
83 mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status);
86 mod_timer(&dev->delay_timer, jiffies + HZ);
90 if (mr->out.hdr.status) {
91 mlx5_ib_warn(dev, "failed - status %d, syndorme 0x%x\n",
93 be32_to_cpu(mr->out.hdr.syndrome));
96 mod_timer(&dev->delay_timer, jiffies + HZ);
100 spin_lock_irqsave(&dev->mdev->priv.mkey_lock, flags);
101 key = dev->mdev->priv.mkey_key++;
102 spin_unlock_irqrestore(&dev->mdev->priv.mkey_lock, flags);
103 mr->mmr.key = mlx5_idx_to_mkey(be32_to_cpu(mr->out.mkey) & 0xffffff) | key;
105 cache->last_add = jiffies;
107 spin_lock_irqsave(&ent->lock, flags);
108 list_add_tail(&mr->list, &ent->head);
111 spin_unlock_irqrestore(&ent->lock, flags);
113 write_lock_irqsave(&table->lock, flags);
114 err = radix_tree_insert(&table->tree, mlx5_base_mkey(mr->mmr.key),
117 pr_err("Error inserting to mr tree. 0x%x\n", -err);
118 write_unlock_irqrestore(&table->lock, flags);
121 static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
123 struct mlx5_mr_cache *cache = &dev->cache;
124 struct mlx5_cache_ent *ent = &cache->ent[c];
125 struct mlx5_create_mkey_mbox_in *in;
126 struct mlx5_ib_mr *mr;
127 int npages = 1 << ent->order;
131 in = kzalloc(sizeof(*in), GFP_KERNEL);
135 for (i = 0; i < num; i++) {
136 if (ent->pending >= MAX_PENDING_REG_MR) {
141 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
146 mr->order = ent->order;
149 in->seg.status = 1 << 6;
150 in->seg.xlt_oct_size = cpu_to_be32((npages + 1) / 2);
151 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
152 in->seg.flags = MLX5_ACCESS_MODE_MTT | MLX5_PERM_UMR_EN;
153 in->seg.log2_page_size = 12;
155 spin_lock_irq(&ent->lock);
157 spin_unlock_irq(&ent->lock);
158 err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in,
159 sizeof(*in), reg_mr_callback,
162 spin_lock_irq(&ent->lock);
164 spin_unlock_irq(&ent->lock);
165 mlx5_ib_warn(dev, "create mkey failed %d\n", err);
175 static void remove_keys(struct mlx5_ib_dev *dev, int c, int num)
177 struct mlx5_mr_cache *cache = &dev->cache;
178 struct mlx5_cache_ent *ent = &cache->ent[c];
179 struct mlx5_ib_mr *mr;
183 for (i = 0; i < num; i++) {
184 spin_lock_irq(&ent->lock);
185 if (list_empty(&ent->head)) {
186 spin_unlock_irq(&ent->lock);
189 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
193 spin_unlock_irq(&ent->lock);
194 err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmr);
196 mlx5_ib_warn(dev, "failed destroy mkey\n");
202 static ssize_t size_write(struct file *filp, const char __user *buf,
203 size_t count, loff_t *pos)
205 struct mlx5_cache_ent *ent = filp->private_data;
206 struct mlx5_ib_dev *dev = ent->dev;
212 if (copy_from_user(lbuf, buf, sizeof(lbuf)))
215 c = order2idx(dev, ent->order);
216 lbuf[sizeof(lbuf) - 1] = 0;
218 if (sscanf(lbuf, "%u", &var) != 1)
221 if (var < ent->limit)
224 if (var > ent->size) {
226 err = add_keys(dev, c, var - ent->size);
227 if (err && err != -EAGAIN)
230 usleep_range(3000, 5000);
232 } else if (var < ent->size) {
233 remove_keys(dev, c, ent->size - var);
239 static ssize_t size_read(struct file *filp, char __user *buf, size_t count,
242 struct mlx5_cache_ent *ent = filp->private_data;
249 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->size);
253 if (copy_to_user(buf, lbuf, err))
261 static const struct file_operations size_fops = {
262 .owner = THIS_MODULE,
268 static ssize_t limit_write(struct file *filp, const char __user *buf,
269 size_t count, loff_t *pos)
271 struct mlx5_cache_ent *ent = filp->private_data;
272 struct mlx5_ib_dev *dev = ent->dev;
278 if (copy_from_user(lbuf, buf, sizeof(lbuf)))
281 c = order2idx(dev, ent->order);
282 lbuf[sizeof(lbuf) - 1] = 0;
284 if (sscanf(lbuf, "%u", &var) != 1)
292 if (ent->cur < ent->limit) {
293 err = add_keys(dev, c, 2 * ent->limit - ent->cur);
301 static ssize_t limit_read(struct file *filp, char __user *buf, size_t count,
304 struct mlx5_cache_ent *ent = filp->private_data;
311 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit);
315 if (copy_to_user(buf, lbuf, err))
323 static const struct file_operations limit_fops = {
324 .owner = THIS_MODULE,
326 .write = limit_write,
330 static int someone_adding(struct mlx5_mr_cache *cache)
334 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
335 if (cache->ent[i].cur < cache->ent[i].limit)
342 static void __cache_work_func(struct mlx5_cache_ent *ent)
344 struct mlx5_ib_dev *dev = ent->dev;
345 struct mlx5_mr_cache *cache = &dev->cache;
346 int i = order2idx(dev, ent->order);
352 ent = &dev->cache.ent[i];
353 if (ent->cur < 2 * ent->limit && !dev->fill_delay) {
354 err = add_keys(dev, i, 1);
355 if (ent->cur < 2 * ent->limit) {
356 if (err == -EAGAIN) {
357 mlx5_ib_dbg(dev, "returned eagain, order %d\n",
359 queue_delayed_work(cache->wq, &ent->dwork,
360 msecs_to_jiffies(3));
362 mlx5_ib_warn(dev, "command failed order %d, err %d\n",
364 queue_delayed_work(cache->wq, &ent->dwork,
365 msecs_to_jiffies(1000));
367 queue_work(cache->wq, &ent->work);
370 } else if (ent->cur > 2 * ent->limit) {
371 if (!someone_adding(cache) &&
372 time_after(jiffies, cache->last_add + 300 * HZ)) {
373 remove_keys(dev, i, 1);
374 if (ent->cur > ent->limit)
375 queue_work(cache->wq, &ent->work);
377 queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ);
382 static void delayed_cache_work_func(struct work_struct *work)
384 struct mlx5_cache_ent *ent;
386 ent = container_of(work, struct mlx5_cache_ent, dwork.work);
387 __cache_work_func(ent);
390 static void cache_work_func(struct work_struct *work)
392 struct mlx5_cache_ent *ent;
394 ent = container_of(work, struct mlx5_cache_ent, work);
395 __cache_work_func(ent);
398 static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order)
400 struct mlx5_mr_cache *cache = &dev->cache;
401 struct mlx5_ib_mr *mr = NULL;
402 struct mlx5_cache_ent *ent;
406 c = order2idx(dev, order);
407 if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) {
408 mlx5_ib_warn(dev, "order %d, cache index %d\n", order, c);
412 for (i = c; i < MAX_MR_CACHE_ENTRIES; i++) {
413 ent = &cache->ent[i];
415 mlx5_ib_dbg(dev, "order %d, cache index %d\n", ent->order, i);
417 spin_lock_irq(&ent->lock);
418 if (!list_empty(&ent->head)) {
419 mr = list_first_entry(&ent->head, struct mlx5_ib_mr,
423 spin_unlock_irq(&ent->lock);
424 if (ent->cur < ent->limit)
425 queue_work(cache->wq, &ent->work);
428 spin_unlock_irq(&ent->lock);
430 queue_work(cache->wq, &ent->work);
437 cache->ent[c].miss++;
442 static void free_cached_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
444 struct mlx5_mr_cache *cache = &dev->cache;
445 struct mlx5_cache_ent *ent;
449 c = order2idx(dev, mr->order);
450 if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) {
451 mlx5_ib_warn(dev, "order %d, cache index %d\n", mr->order, c);
454 ent = &cache->ent[c];
455 spin_lock_irq(&ent->lock);
456 list_add_tail(&mr->list, &ent->head);
458 if (ent->cur > 2 * ent->limit)
460 spin_unlock_irq(&ent->lock);
463 queue_work(cache->wq, &ent->work);
466 static void clean_keys(struct mlx5_ib_dev *dev, int c)
468 struct mlx5_mr_cache *cache = &dev->cache;
469 struct mlx5_cache_ent *ent = &cache->ent[c];
470 struct mlx5_ib_mr *mr;
473 cancel_delayed_work(&ent->dwork);
475 spin_lock_irq(&ent->lock);
476 if (list_empty(&ent->head)) {
477 spin_unlock_irq(&ent->lock);
480 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
484 spin_unlock_irq(&ent->lock);
485 err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmr);
487 mlx5_ib_warn(dev, "failed destroy mkey\n");
493 static int mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev)
495 struct mlx5_mr_cache *cache = &dev->cache;
496 struct mlx5_cache_ent *ent;
499 if (!mlx5_debugfs_root)
502 cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root);
506 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
507 ent = &cache->ent[i];
508 sprintf(ent->name, "%d", ent->order);
509 ent->dir = debugfs_create_dir(ent->name, cache->root);
513 ent->fsize = debugfs_create_file("size", 0600, ent->dir, ent,
518 ent->flimit = debugfs_create_file("limit", 0600, ent->dir, ent,
523 ent->fcur = debugfs_create_u32("cur", 0400, ent->dir,
528 ent->fmiss = debugfs_create_u32("miss", 0600, ent->dir,
537 static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)
539 if (!mlx5_debugfs_root)
542 debugfs_remove_recursive(dev->cache.root);
545 static void delay_time_func(unsigned long ctx)
547 struct mlx5_ib_dev *dev = (struct mlx5_ib_dev *)ctx;
552 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
554 struct mlx5_mr_cache *cache = &dev->cache;
555 struct mlx5_cache_ent *ent;
560 cache->wq = create_singlethread_workqueue("mkey_cache");
562 mlx5_ib_warn(dev, "failed to create work queue\n");
566 setup_timer(&dev->delay_timer, delay_time_func, (unsigned long)dev);
567 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
568 INIT_LIST_HEAD(&cache->ent[i].head);
569 spin_lock_init(&cache->ent[i].lock);
571 ent = &cache->ent[i];
572 INIT_LIST_HEAD(&ent->head);
573 spin_lock_init(&ent->lock);
577 if (dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE)
578 limit = dev->mdev->profile->mr_cache[i].limit;
582 INIT_WORK(&ent->work, cache_work_func);
583 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
585 queue_work(cache->wq, &ent->work);
588 err = mlx5_mr_cache_debugfs_init(dev);
590 mlx5_ib_warn(dev, "cache debugfs failure\n");
595 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
599 dev->cache.stopped = 1;
600 flush_workqueue(dev->cache.wq);
602 mlx5_mr_cache_debugfs_cleanup(dev);
604 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++)
607 destroy_workqueue(dev->cache.wq);
608 del_timer_sync(&dev->delay_timer);
613 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)
615 struct mlx5_ib_dev *dev = to_mdev(pd->device);
616 struct mlx5_core_dev *mdev = dev->mdev;
617 struct mlx5_create_mkey_mbox_in *in;
618 struct mlx5_mkey_seg *seg;
619 struct mlx5_ib_mr *mr;
622 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
624 return ERR_PTR(-ENOMEM);
626 in = kzalloc(sizeof(*in), GFP_KERNEL);
633 seg->flags = convert_access(acc) | MLX5_ACCESS_MODE_PA;
634 seg->flags_pd = cpu_to_be32(to_mpd(pd)->pdn | MLX5_MKEY_LEN64);
635 seg->qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
638 err = mlx5_core_create_mkey(mdev, &mr->mmr, in, sizeof(*in), NULL, NULL,
644 mr->ibmr.lkey = mr->mmr.key;
645 mr->ibmr.rkey = mr->mmr.key;
659 static int get_octo_len(u64 addr, u64 len, int page_size)
664 offset = addr & (page_size - 1);
665 npages = ALIGN(len + offset, page_size) >> ilog2(page_size);
666 return (npages + 1) / 2;
669 static int use_umr(int order)
674 static void prep_umr_reg_wqe(struct ib_pd *pd, struct ib_send_wr *wr,
675 struct ib_sge *sg, u64 dma, int n, u32 key,
676 int page_shift, u64 virt_addr, u64 len,
679 struct mlx5_ib_dev *dev = to_mdev(pd->device);
680 struct ib_mr *mr = dev->umrc.mr;
683 sg->length = ALIGN(sizeof(u64) * n, 64);
694 wr->opcode = MLX5_IB_WR_UMR;
695 wr->wr.fast_reg.page_list_len = n;
696 wr->wr.fast_reg.page_shift = page_shift;
697 wr->wr.fast_reg.rkey = key;
698 wr->wr.fast_reg.iova_start = virt_addr;
699 wr->wr.fast_reg.length = len;
700 wr->wr.fast_reg.access_flags = access_flags;
701 wr->wr.fast_reg.page_list = (struct ib_fast_reg_page_list *)pd;
704 static void prep_umr_unreg_wqe(struct mlx5_ib_dev *dev,
705 struct ib_send_wr *wr, u32 key)
707 wr->send_flags = MLX5_IB_SEND_UMR_UNREG;
708 wr->opcode = MLX5_IB_WR_UMR;
709 wr->wr.fast_reg.rkey = key;
712 void mlx5_umr_cq_handler(struct ib_cq *cq, void *cq_context)
714 struct mlx5_ib_umr_context *context;
719 err = ib_poll_cq(cq, 1, &wc);
721 pr_warn("poll cq error %d\n", err);
727 context = (struct mlx5_ib_umr_context *) (unsigned long) wc.wr_id;
728 context->status = wc.status;
729 complete(&context->done);
731 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
734 static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem,
735 u64 virt_addr, u64 len, int npages,
736 int page_shift, int order, int access_flags)
738 struct mlx5_ib_dev *dev = to_mdev(pd->device);
739 struct device *ddev = dev->ib_dev.dma_device;
740 struct umr_common *umrc = &dev->umrc;
741 struct mlx5_ib_umr_context umr_context;
742 struct ib_send_wr wr, *bad;
743 struct mlx5_ib_mr *mr;
745 int size = sizeof(u64) * npages;
749 for (i = 0; i < 1; i++) {
750 mr = alloc_cached_mr(dev, order);
754 err = add_keys(dev, order2idx(dev, order), 1);
755 if (err && err != -EAGAIN) {
756 mlx5_ib_warn(dev, "add_keys failed, err %d\n", err);
762 return ERR_PTR(-EAGAIN);
764 mr->pas = kmalloc(size + MLX5_UMR_ALIGN - 1, GFP_KERNEL);
770 mlx5_ib_populate_pas(dev, umem, page_shift,
771 mr_align(mr->pas, MLX5_UMR_ALIGN), 1);
773 mr->dma = dma_map_single(ddev, mr_align(mr->pas, MLX5_UMR_ALIGN), size,
775 if (dma_mapping_error(ddev, mr->dma)) {
780 memset(&wr, 0, sizeof(wr));
781 wr.wr_id = (u64)(unsigned long)&umr_context;
782 prep_umr_reg_wqe(pd, &wr, &sg, mr->dma, npages, mr->mmr.key, page_shift, virt_addr, len, access_flags);
784 mlx5_ib_init_umr_context(&umr_context);
786 err = ib_post_send(umrc->qp, &wr, &bad);
788 mlx5_ib_warn(dev, "post send failed, err %d\n", err);
791 wait_for_completion(&umr_context.done);
792 if (umr_context.status != IB_WC_SUCCESS) {
793 mlx5_ib_warn(dev, "reg umr failed\n");
798 mr->mmr.iova = virt_addr;
800 mr->mmr.pd = to_mpd(pd)->pdn;
804 dma_unmap_single(ddev, mr->dma, size, DMA_TO_DEVICE);
811 free_cached_mr(dev, mr);
818 static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, u64 virt_addr,
819 u64 length, struct ib_umem *umem,
820 int npages, int page_shift,
823 struct mlx5_ib_dev *dev = to_mdev(pd->device);
824 struct mlx5_create_mkey_mbox_in *in;
825 struct mlx5_ib_mr *mr;
829 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
831 return ERR_PTR(-ENOMEM);
833 inlen = sizeof(*in) + sizeof(*in->pas) * ((npages + 1) / 2) * 2;
834 in = mlx5_vzalloc(inlen);
839 mlx5_ib_populate_pas(dev, umem, page_shift, in->pas, 0);
841 in->seg.flags = convert_access(access_flags) |
842 MLX5_ACCESS_MODE_MTT;
843 in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn);
844 in->seg.start_addr = cpu_to_be64(virt_addr);
845 in->seg.len = cpu_to_be64(length);
846 in->seg.bsfs_octo_size = 0;
847 in->seg.xlt_oct_size = cpu_to_be32(get_octo_len(virt_addr, length, 1 << page_shift));
848 in->seg.log2_page_size = page_shift;
849 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
850 in->xlat_oct_act_size = cpu_to_be32(get_octo_len(virt_addr, length,
852 err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in, inlen, NULL,
855 mlx5_ib_warn(dev, "create mkey failed\n");
861 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmr.key);
874 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
875 u64 virt_addr, int access_flags,
876 struct ib_udata *udata)
878 struct mlx5_ib_dev *dev = to_mdev(pd->device);
879 struct mlx5_ib_mr *mr = NULL;
880 struct ib_umem *umem;
887 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
888 start, virt_addr, length, access_flags);
889 umem = ib_umem_get(pd->uobject->context, start, length, access_flags,
892 mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(umem));
896 mlx5_ib_cont_pages(umem, start, &npages, &page_shift, &ncont, &order);
898 mlx5_ib_warn(dev, "avoid zero region\n");
903 mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",
904 npages, ncont, order, page_shift);
906 if (use_umr(order)) {
907 mr = reg_umr(pd, umem, virt_addr, length, ncont, page_shift,
908 order, access_flags);
909 if (PTR_ERR(mr) == -EAGAIN) {
910 mlx5_ib_dbg(dev, "cache empty for order %d", order);
916 mr = reg_create(pd, virt_addr, length, umem, ncont, page_shift,
924 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmr.key);
928 spin_lock(&dev->mr_lock);
929 dev->mdev->priv.reg_pages += npages;
930 spin_unlock(&dev->mr_lock);
931 mr->ibmr.lkey = mr->mmr.key;
932 mr->ibmr.rkey = mr->mmr.key;
937 ib_umem_release(umem);
941 static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
943 struct umr_common *umrc = &dev->umrc;
944 struct mlx5_ib_umr_context umr_context;
945 struct ib_send_wr wr, *bad;
948 memset(&wr, 0, sizeof(wr));
949 wr.wr_id = (u64)(unsigned long)&umr_context;
950 prep_umr_unreg_wqe(dev, &wr, mr->mmr.key);
952 mlx5_ib_init_umr_context(&umr_context);
954 err = ib_post_send(umrc->qp, &wr, &bad);
957 mlx5_ib_dbg(dev, "err %d\n", err);
960 wait_for_completion(&umr_context.done);
963 if (umr_context.status != IB_WC_SUCCESS) {
964 mlx5_ib_warn(dev, "unreg umr failed\n");
974 int mlx5_ib_dereg_mr(struct ib_mr *ibmr)
976 struct mlx5_ib_dev *dev = to_mdev(ibmr->device);
977 struct mlx5_ib_mr *mr = to_mmr(ibmr);
978 struct ib_umem *umem = mr->umem;
979 int npages = mr->npages;
980 int umred = mr->umred;
984 err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmr);
986 mlx5_ib_warn(dev, "failed to destroy mkey 0x%x (%d)\n",
991 err = unreg_umr(dev, mr);
993 mlx5_ib_warn(dev, "failed unregister\n");
996 free_cached_mr(dev, mr);
1000 ib_umem_release(umem);
1001 spin_lock(&dev->mr_lock);
1002 dev->mdev->priv.reg_pages -= npages;
1003 spin_unlock(&dev->mr_lock);
1012 struct ib_mr *mlx5_ib_create_mr(struct ib_pd *pd,
1013 struct ib_mr_init_attr *mr_init_attr)
1015 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1016 struct mlx5_create_mkey_mbox_in *in;
1017 struct mlx5_ib_mr *mr;
1018 int access_mode, err;
1019 int ndescs = roundup(mr_init_attr->max_reg_descriptors, 4);
1021 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1023 return ERR_PTR(-ENOMEM);
1025 in = kzalloc(sizeof(*in), GFP_KERNEL);
1031 in->seg.status = 1 << 6; /* free */
1032 in->seg.xlt_oct_size = cpu_to_be32(ndescs);
1033 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
1034 in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn);
1035 access_mode = MLX5_ACCESS_MODE_MTT;
1037 if (mr_init_attr->flags & IB_MR_SIGNATURE_EN) {
1040 in->seg.flags_pd = cpu_to_be32(be32_to_cpu(in->seg.flags_pd) |
1042 in->seg.bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
1043 mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL);
1049 /* create mem & wire PSVs */
1050 err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn,
1055 access_mode = MLX5_ACCESS_MODE_KLM;
1056 mr->sig->psv_memory.psv_idx = psv_index[0];
1057 mr->sig->psv_wire.psv_idx = psv_index[1];
1059 mr->sig->sig_status_checked = true;
1060 mr->sig->sig_err_exists = false;
1061 /* Next UMR, Arm SIGERR */
1062 ++mr->sig->sigerr_count;
1065 in->seg.flags = MLX5_PERM_UMR_EN | access_mode;
1066 err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in, sizeof(*in),
1069 goto err_destroy_psv;
1071 mr->ibmr.lkey = mr->mmr.key;
1072 mr->ibmr.rkey = mr->mmr.key;
1080 if (mlx5_core_destroy_psv(dev->mdev,
1081 mr->sig->psv_memory.psv_idx))
1082 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1083 mr->sig->psv_memory.psv_idx);
1084 if (mlx5_core_destroy_psv(dev->mdev,
1085 mr->sig->psv_wire.psv_idx))
1086 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1087 mr->sig->psv_wire.psv_idx);
1095 return ERR_PTR(err);
1098 int mlx5_ib_destroy_mr(struct ib_mr *ibmr)
1100 struct mlx5_ib_dev *dev = to_mdev(ibmr->device);
1101 struct mlx5_ib_mr *mr = to_mmr(ibmr);
1105 if (mlx5_core_destroy_psv(dev->mdev,
1106 mr->sig->psv_memory.psv_idx))
1107 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1108 mr->sig->psv_memory.psv_idx);
1109 if (mlx5_core_destroy_psv(dev->mdev,
1110 mr->sig->psv_wire.psv_idx))
1111 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1112 mr->sig->psv_wire.psv_idx);
1116 err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmr);
1118 mlx5_ib_warn(dev, "failed to destroy mkey 0x%x (%d)\n",
1128 struct ib_mr *mlx5_ib_alloc_fast_reg_mr(struct ib_pd *pd,
1129 int max_page_list_len)
1131 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1132 struct mlx5_create_mkey_mbox_in *in;
1133 struct mlx5_ib_mr *mr;
1136 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1138 return ERR_PTR(-ENOMEM);
1140 in = kzalloc(sizeof(*in), GFP_KERNEL);
1146 in->seg.status = 1 << 6; /* free */
1147 in->seg.xlt_oct_size = cpu_to_be32((max_page_list_len + 1) / 2);
1148 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
1149 in->seg.flags = MLX5_PERM_UMR_EN | MLX5_ACCESS_MODE_MTT;
1150 in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn);
1152 * TBD not needed - issue 197292 */
1153 in->seg.log2_page_size = PAGE_SHIFT;
1155 err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in, sizeof(*in), NULL,
1161 mr->ibmr.lkey = mr->mmr.key;
1162 mr->ibmr.rkey = mr->mmr.key;
1169 return ERR_PTR(err);
1172 struct ib_fast_reg_page_list *mlx5_ib_alloc_fast_reg_page_list(struct ib_device *ibdev,
1175 struct mlx5_ib_fast_reg_page_list *mfrpl;
1176 int size = page_list_len * sizeof(u64);
1178 mfrpl = kmalloc(sizeof(*mfrpl), GFP_KERNEL);
1180 return ERR_PTR(-ENOMEM);
1182 mfrpl->ibfrpl.page_list = kmalloc(size, GFP_KERNEL);
1183 if (!mfrpl->ibfrpl.page_list)
1186 mfrpl->mapped_page_list = dma_alloc_coherent(ibdev->dma_device,
1189 if (!mfrpl->mapped_page_list)
1192 WARN_ON(mfrpl->map & 0x3f);
1194 return &mfrpl->ibfrpl;
1197 kfree(mfrpl->ibfrpl.page_list);
1199 return ERR_PTR(-ENOMEM);
1202 void mlx5_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
1204 struct mlx5_ib_fast_reg_page_list *mfrpl = to_mfrpl(page_list);
1205 struct mlx5_ib_dev *dev = to_mdev(page_list->device);
1206 int size = page_list->max_page_list_len * sizeof(u64);
1208 dma_free_coherent(&dev->mdev->pdev->dev, size, mfrpl->mapped_page_list,
1210 kfree(mfrpl->ibfrpl.page_list);
1214 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
1215 struct ib_mr_status *mr_status)
1217 struct mlx5_ib_mr *mmr = to_mmr(ibmr);
1220 if (check_mask & ~IB_MR_CHECK_SIG_STATUS) {
1221 pr_err("Invalid status check mask\n");
1226 mr_status->fail_status = 0;
1227 if (check_mask & IB_MR_CHECK_SIG_STATUS) {
1230 pr_err("signature status check requested on a non-signature enabled MR\n");
1234 mmr->sig->sig_status_checked = true;
1235 if (!mmr->sig->sig_err_exists)
1238 if (ibmr->lkey == mmr->sig->err_item.key)
1239 memcpy(&mr_status->sig_err, &mmr->sig->err_item,
1240 sizeof(mr_status->sig_err));
1242 mr_status->sig_err.err_type = IB_SIG_BAD_GUARD;
1243 mr_status->sig_err.sig_err_offset = 0;
1244 mr_status->sig_err.key = mmr->sig->err_item.key;
1247 mmr->sig->sig_err_exists = false;
1248 mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS;