drivers/infiniband/hw/hns/hns_roce_hem.c

   1 /*
   2  * Copyright (c) 2016 Hisilicon Limited.
   3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/platform_device.h>
  35 #include "hns_roce_device.h"
  36 #include "hns_roce_hem.h"
  37 #include "hns_roce_common.h"
  38
  39 #define HNS_ROCE_HEM_ALLOC_SIZE         (1 << 17)
  40 #define HNS_ROCE_TABLE_CHUNK_SIZE       (1 << 17)
  41
  42 #define DMA_ADDR_T_SHIFT                12
  43 #define BT_BA_SHIFT                     32
  44
  45 struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, int npages,
  46                                         gfp_t gfp_mask)
  47 {
  48         struct hns_roce_hem_chunk *chunk = NULL;
  49         struct hns_roce_hem *hem;
  50         struct scatterlist *mem;
  51         int order;
  52         void *buf;
  53
  54         WARN_ON(gfp_mask & __GFP_HIGHMEM);
  55
  56         hem = kmalloc(sizeof(*hem),
  57                       gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
  58         if (!hem)
  59                 return NULL;
  60
  61         hem->refcount = 0;
  62         INIT_LIST_HEAD(&hem->chunk_list);
  63
  64         order = get_order(HNS_ROCE_HEM_ALLOC_SIZE);
  65
  66         while (npages > 0) {
  67                 if (!chunk) {
  68                         chunk = kmalloc(sizeof(*chunk),
  69                                 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
  70                         if (!chunk)
  71                                 goto fail;
  72
  73                         sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
  74                         chunk->npages = 0;
  75                         chunk->nsg = 0;
  76                         list_add_tail(&chunk->list, &hem->chunk_list);
  77                 }
  78
  79                 while (1 << order > npages)
  80                         --order;
  81
  82                 /*
  83                 * Alloc memory one time. If failed, don't alloc small block
  84                 * memory, directly return fail.
  85                 */
  86                 mem = &chunk->mem[chunk->npages];
  87                 buf = dma_alloc_coherent(&hr_dev->pdev->dev, PAGE_SIZE << order,
  88                                 &sg_dma_address(mem), gfp_mask);
  89                 if (!buf)
  90                         goto fail;
  91
  92                 sg_set_buf(mem, buf, PAGE_SIZE << order);
  93                 WARN_ON(mem->offset);
  94                 sg_dma_len(mem) = PAGE_SIZE << order;
  95
  96                 ++chunk->npages;
  97                 ++chunk->nsg;
  98                 npages -= 1 << order;
  99         }
 100
 101         return hem;
 102
 103 fail:
 104         hns_roce_free_hem(hr_dev, hem);
 105         return NULL;
 106 }
 107
 108 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
 109 {
 110         struct hns_roce_hem_chunk *chunk, *tmp;
 111         int i;
 112
 113         if (!hem)
 114                 return;
 115
 116         list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
 117                 for (i = 0; i < chunk->npages; ++i)
 118                         dma_free_coherent(&hr_dev->pdev->dev,
 119                                    chunk->mem[i].length,
 120                                    lowmem_page_address(sg_page(&chunk->mem[i])),
 121                                    sg_dma_address(&chunk->mem[i]));
 122                 kfree(chunk);
 123         }
 124
 125         kfree(hem);
 126 }
 127
 128 static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
 129                             struct hns_roce_hem_table *table, unsigned long obj)
 130 {
 131         struct device *dev = &hr_dev->pdev->dev;
 132         spinlock_t *lock = &hr_dev->bt_cmd_lock;
 133         unsigned long end = 0;
 134         unsigned long flags;
 135         struct hns_roce_hem_iter iter;
 136         void __iomem *bt_cmd;
 137         u32 bt_cmd_h_val = 0;
 138         u32 bt_cmd_val[2];
 139         u32 bt_cmd_l = 0;
 140         u64 bt_ba = 0;
 141         int ret = 0;
 142
 143         /* Find the HEM(Hardware Entry Memory) entry */
 144         unsigned long i = (obj & (table->num_obj - 1)) /
 145                           (HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size);
 146
 147         switch (table->type) {
 148         case HEM_TYPE_QPC:
 149                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 150                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
 151                 break;
 152         case HEM_TYPE_MTPT:
 153                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 154                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
 155                                HEM_TYPE_MTPT);
 156                 break;
 157         case HEM_TYPE_CQC:
 158                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 159                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
 160                 break;
 161         case HEM_TYPE_SRQC:
 162                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 163                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
 164                                HEM_TYPE_SRQC);
 165                 break;
 166         default:
 167                 return ret;
 168         }
 169         roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
 170                        ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
 171         roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
 172         roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
 173
 174         /* Currently iter only a chunk */
 175         for (hns_roce_hem_first(table->hem[i], &iter);
 176              !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
 177                 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT;
 178
 179                 spin_lock_irqsave(lock, flags);
 180
 181                 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
 182
 183                 end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
 184                 while (1) {
 185                         if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
 186                                 if (!(time_before(jiffies, end))) {
 187                                         dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
 188                                         spin_unlock_irqrestore(lock, flags);
 189                                         return -EBUSY;
 190                                 }
 191                         } else {
 192                                 break;
 193                         }
 194                         msleep(HW_SYNC_SLEEP_TIME_INTERVAL);
 195                 }
 196
 197                 bt_cmd_l = (u32)bt_ba;
 198                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
 199                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
 200                                bt_ba >> BT_BA_SHIFT);
 201
 202                 bt_cmd_val[0] = bt_cmd_l;
 203                 bt_cmd_val[1] = bt_cmd_h_val;
 204                 hns_roce_write64_k(bt_cmd_val,
 205                                    hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
 206                 spin_unlock_irqrestore(lock, flags);
 207         }
 208
 209         return ret;
 210 }
 211
 212 int hns_roce_table_get(struct hns_roce_dev *hr_dev,
 213                        struct hns_roce_hem_table *table, unsigned long obj)
 214 {
 215         struct device *dev = &hr_dev->pdev->dev;
 216         int ret = 0;
 217         unsigned long i;
 218
 219         i = (obj & (table->num_obj - 1)) / (HNS_ROCE_TABLE_CHUNK_SIZE /
 220              table->obj_size);
 221
 222         mutex_lock(&table->mutex);
 223
 224         if (table->hem[i]) {
 225                 ++table->hem[i]->refcount;
 226                 goto out;
 227         }
 228
 229         table->hem[i] = hns_roce_alloc_hem(hr_dev,
 230                                        HNS_ROCE_TABLE_CHUNK_SIZE >> PAGE_SHIFT,
 231                                        (table->lowmem ? GFP_KERNEL :
 232                                         GFP_HIGHUSER) | __GFP_NOWARN);
 233         if (!table->hem[i]) {
 234                 ret = -ENOMEM;
 235                 goto out;
 236         }
 237
 238         /* Set HEM base address(128K/page, pa) to Hardware */
 239         if (hns_roce_set_hem(hr_dev, table, obj)) {
 240                 ret = -ENODEV;
 241                 dev_err(dev, "set HEM base address to HW failed.\n");
 242                 goto out;
 243         }
 244
 245         ++table->hem[i]->refcount;
 246 out:
 247         mutex_unlock(&table->mutex);
 248         return ret;
 249 }
 250
 251 void hns_roce_table_put(struct hns_roce_dev *hr_dev,
 252                         struct hns_roce_hem_table *table, unsigned long obj)
 253 {
 254         struct device *dev = &hr_dev->pdev->dev;
 255         unsigned long i;
 256
 257         i = (obj & (table->num_obj - 1)) /
 258             (HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size);
 259
 260         mutex_lock(&table->mutex);
 261
 262         if (--table->hem[i]->refcount == 0) {
 263                 /* Clear HEM base address */
 264                 if (hr_dev->hw->clear_hem(hr_dev, table, obj))
 265                         dev_warn(dev, "Clear HEM base address failed.\n");
 266
 267                 hns_roce_free_hem(hr_dev, table->hem[i]);
 268                 table->hem[i] = NULL;
 269         }
 270
 271         mutex_unlock(&table->mutex);
 272 }
 273
 274 void *hns_roce_table_find(struct hns_roce_hem_table *table, unsigned long obj,
 275                           dma_addr_t *dma_handle)
 276 {
 277         struct hns_roce_hem_chunk *chunk;
 278         unsigned long idx;
 279         int i;
 280         int offset, dma_offset;
 281         struct hns_roce_hem *hem;
 282         struct page *page = NULL;
 283
 284         if (!table->lowmem)
 285                 return NULL;
 286
 287         mutex_lock(&table->mutex);
 288         idx = (obj & (table->num_obj - 1)) * table->obj_size;
 289         hem = table->hem[idx / HNS_ROCE_TABLE_CHUNK_SIZE];
 290         dma_offset = offset = idx % HNS_ROCE_TABLE_CHUNK_SIZE;
 291
 292         if (!hem)
 293                 goto out;
 294
 295         list_for_each_entry(chunk, &hem->chunk_list, list) {
 296                 for (i = 0; i < chunk->npages; ++i) {
 297                         if (dma_handle && dma_offset >= 0) {
 298                                 if (sg_dma_len(&chunk->mem[i]) >
 299                                     (u32)dma_offset)
 300                                         *dma_handle = sg_dma_address(
 301                                                 &chunk->mem[i]) + dma_offset;
 302                                 dma_offset -= sg_dma_len(&chunk->mem[i]);
 303                         }
 304
 305                         if (chunk->mem[i].length > (u32)offset) {
 306                                 page = sg_page(&chunk->mem[i]);
 307                                 goto out;
 308                         }
 309                         offset -= chunk->mem[i].length;
 310                 }
 311         }
 312
 313 out:
 314         mutex_unlock(&table->mutex);
 315         return page ? lowmem_page_address(page) + offset : NULL;
 316 }
 317
 318 int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
 319                              struct hns_roce_hem_table *table,
 320                              unsigned long start, unsigned long end)
 321 {
 322         unsigned long inc = HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size;
 323         unsigned long i = 0;
 324         int ret = 0;
 325
 326         /* Allocate MTT entry memory according to chunk(128K) */
 327         for (i = start; i <= end; i += inc) {
 328                 ret = hns_roce_table_get(hr_dev, table, i);
 329                 if (ret)
 330                         goto fail;
 331         }
 332
 333         return 0;
 334
 335 fail:
 336         while (i > start) {
 337                 i -= inc;
 338                 hns_roce_table_put(hr_dev, table, i);
 339         }
 340         return ret;
 341 }
 342
 343 void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
 344                               struct hns_roce_hem_table *table,
 345                               unsigned long start, unsigned long end)
 346 {
 347         unsigned long i;
 348
 349         for (i = start; i <= end;
 350                 i += HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size)
 351                 hns_roce_table_put(hr_dev, table, i);
 352 }
 353
 354 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
 355                             struct hns_roce_hem_table *table, u32 type,
 356                             unsigned long obj_size, unsigned long nobj,
 357                             int use_lowmem)
 358 {
 359         unsigned long obj_per_chunk;
 360         unsigned long num_hem;
 361
 362         obj_per_chunk = HNS_ROCE_TABLE_CHUNK_SIZE / obj_size;
 363         num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
 364
 365         table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
 366         if (!table->hem)
 367                 return -ENOMEM;
 368
 369         table->type = type;
 370         table->num_hem = num_hem;
 371         table->num_obj = nobj;
 372         table->obj_size = obj_size;
 373         table->lowmem = use_lowmem;
 374         mutex_init(&table->mutex);
 375
 376         return 0;
 377 }
 378
 379 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
 380                                 struct hns_roce_hem_table *table)
 381 {
 382         struct device *dev = &hr_dev->pdev->dev;
 383         unsigned long i;
 384
 385         for (i = 0; i < table->num_hem; ++i)
 386                 if (table->hem[i]) {
 387                         if (hr_dev->hw->clear_hem(hr_dev, table,
 388                             i * HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size))
 389                                 dev_err(dev, "Clear HEM base address failed.\n");
 390
 391                         hns_roce_free_hem(hr_dev, table->hem[i]);
 392                 }
 393
 394         kfree(table->hem);
 395 }
 396
 397 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
 398 {
 399         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
 400         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
 401         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
 402         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
 403         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
 404 }