drivers/crypto/qat/qat_common/qat_algs.c

   1 /*
   2   This file is provided under a dual BSD/GPLv2 license.  When using or
   3   redistributing this file, you may do so under either license.
   4
   5   GPL LICENSE SUMMARY
   6   Copyright(c) 2014 Intel Corporation.
   7   This program is free software; you can redistribute it and/or modify
   8   it under the terms of version 2 of the GNU General Public License as
   9   published by the Free Software Foundation.
  10
  11   This program is distributed in the hope that it will be useful, but
  12   WITHOUT ANY WARRANTY; without even the implied warranty of
  13   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14   General Public License for more details.
  15
  16   Contact Information:
  17   qat-linux@intel.com
  18
  19   BSD LICENSE
  20   Copyright(c) 2014 Intel Corporation.
  21   Redistribution and use in source and binary forms, with or without
  22   modification, are permitted provided that the following conditions
  23   are met:
  24
  25     * Redistributions of source code must retain the above copyright
  26       notice, this list of conditions and the following disclaimer.
  27     * Redistributions in binary form must reproduce the above copyright
  28       notice, this list of conditions and the following disclaimer in
  29       the documentation and/or other materials provided with the
  30       distribution.
  31     * Neither the name of Intel Corporation nor the names of its
  32       contributors may be used to endorse or promote products derived
  33       from this software without specific prior written permission.
  34
  35   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  36   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  37   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  38   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  39   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  41   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  42   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  43   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  44   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  45   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  46 */
  47 #include <linux/module.h>
  48 #include <linux/slab.h>
  49 #include <linux/crypto.h>
  50 #include <crypto/aead.h>
  51 #include <crypto/aes.h>
  52 #include <crypto/sha.h>
  53 #include <crypto/hash.h>
  54 #include <crypto/algapi.h>
  55 #include <crypto/authenc.h>
  56 #include <crypto/rng.h>
  57 #include <linux/dma-mapping.h>
  58 #include "adf_accel_devices.h"
  59 #include "adf_transport.h"
  60 #include "adf_common_drv.h"
  61 #include "qat_crypto.h"
  62 #include "icp_qat_hw.h"
  63 #include "icp_qat_fw.h"
  64 #include "icp_qat_fw_la.h"
  65
  66 #define QAT_AES_HW_CONFIG_ENC(alg) \
  67         ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
  68                         ICP_QAT_HW_CIPHER_NO_CONVERT, \
  69                         ICP_QAT_HW_CIPHER_ENCRYPT)
  70
  71 #define QAT_AES_HW_CONFIG_DEC(alg) \
  72         ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
  73                         ICP_QAT_HW_CIPHER_KEY_CONVERT, \
  74                         ICP_QAT_HW_CIPHER_DECRYPT)
  75
  76 static atomic_t active_dev;
  77
  78 struct qat_alg_buf {
  79         uint32_t len;
  80         uint32_t resrvd;
  81         uint64_t addr;
  82 } __packed;
  83
  84 struct qat_alg_buf_list {
  85         uint64_t resrvd;
  86         uint32_t num_bufs;
  87         uint32_t num_mapped_bufs;
  88         struct qat_alg_buf bufers[];
  89 } __packed __aligned(64);
  90
  91 /* Common content descriptor */
  92 struct qat_alg_cd {
  93         union {
  94                 struct qat_enc { /* Encrypt content desc */
  95                         struct icp_qat_hw_cipher_algo_blk cipher;
  96                         struct icp_qat_hw_auth_algo_blk hash;
  97                 } qat_enc_cd;
  98                 struct qat_dec { /* Decrytp content desc */
  99                         struct icp_qat_hw_auth_algo_blk hash;
 100                         struct icp_qat_hw_cipher_algo_blk cipher;
 101                 } qat_dec_cd;
 102         };
 103 } __aligned(64);
 104
 105 #define MAX_AUTH_STATE_SIZE sizeof(struct icp_qat_hw_auth_algo_blk)
 106
 107 struct qat_auth_state {
 108         uint8_t data[MAX_AUTH_STATE_SIZE + 64];
 109 } __aligned(64);
 110
 111 struct qat_alg_session_ctx {
 112         struct qat_alg_cd *enc_cd;
 113         dma_addr_t enc_cd_paddr;
 114         struct qat_alg_cd *dec_cd;
 115         dma_addr_t dec_cd_paddr;
 116         struct icp_qat_fw_la_bulk_req enc_fw_req_tmpl;
 117         struct icp_qat_fw_la_bulk_req dec_fw_req_tmpl;
 118         struct qat_crypto_instance *inst;
 119         struct crypto_tfm *tfm;
 120         struct crypto_shash *hash_tfm;
 121         enum icp_qat_hw_auth_algo qat_hash_alg;
 122         uint8_t salt[AES_BLOCK_SIZE];
 123         spinlock_t lock;        /* protects qat_alg_session_ctx struct */
 124 };
 125
 126 static int get_current_node(void)
 127 {
 128         return cpu_data(current_thread_info()->cpu).phys_proc_id;
 129 }
 130
 131 static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)
 132 {
 133         switch (qat_hash_alg) {
 134         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 135                 return ICP_QAT_HW_SHA1_STATE1_SZ;
 136         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 137                 return ICP_QAT_HW_SHA256_STATE1_SZ;
 138         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 139                 return ICP_QAT_HW_SHA512_STATE1_SZ;
 140         default:
 141                 return -EFAULT;
 142         };
 143         return -EFAULT;
 144 }
 145
 146 static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
 147                                   struct qat_alg_session_ctx *ctx,
 148                                   const uint8_t *auth_key,
 149                                   unsigned int auth_keylen)
 150 {
 151         struct qat_auth_state auth_state;
 152         SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);
 153         struct sha1_state sha1;
 154         struct sha256_state sha256;
 155         struct sha512_state sha512;
 156         int block_size = crypto_shash_blocksize(ctx->hash_tfm);
 157         int digest_size = crypto_shash_digestsize(ctx->hash_tfm);
 158         uint8_t *ipad = auth_state.data;
 159         uint8_t *opad = ipad + block_size;
 160         __be32 *hash_state_out;
 161         __be64 *hash512_state_out;
 162         int i, offset;
 163
 164         memset(auth_state.data, '\0', MAX_AUTH_STATE_SIZE + 64);
 165         shash->tfm = ctx->hash_tfm;
 166         shash->flags = 0x0;
 167
 168         if (auth_keylen > block_size) {
 169                 char buff[SHA512_BLOCK_SIZE];
 170                 int ret = crypto_shash_digest(shash, auth_key,
 171                                               auth_keylen, buff);
 172                 if (ret)
 173                         return ret;
 174
 175                 memcpy(ipad, buff, digest_size);
 176                 memcpy(opad, buff, digest_size);
 177                 memset(ipad + digest_size, 0, block_size - digest_size);
 178                 memset(opad + digest_size, 0, block_size - digest_size);
 179         } else {
 180                 memcpy(ipad, auth_key, auth_keylen);
 181                 memcpy(opad, auth_key, auth_keylen);
 182                 memset(ipad + auth_keylen, 0, block_size - auth_keylen);
 183                 memset(opad + auth_keylen, 0, block_size - auth_keylen);
 184         }
 185
 186         for (i = 0; i < block_size; i++) {
 187                 char *ipad_ptr = ipad + i;
 188                 char *opad_ptr = opad + i;
 189                 *ipad_ptr ^= 0x36;
 190                 *opad_ptr ^= 0x5C;
 191         }
 192
 193         if (crypto_shash_init(shash))
 194                 return -EFAULT;
 195
 196         if (crypto_shash_update(shash, ipad, block_size))
 197                 return -EFAULT;
 198
 199         hash_state_out = (__be32 *)hash->sha.state1;
 200         hash512_state_out = (__be64 *)hash_state_out;
 201
 202         switch (ctx->qat_hash_alg) {
 203         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 204                 if (crypto_shash_export(shash, &sha1))
 205                         return -EFAULT;
 206                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 207                         *hash_state_out = cpu_to_be32(*(sha1.state + i));
 208                 break;
 209         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 210                 if (crypto_shash_export(shash, &sha256))
 211                         return -EFAULT;
 212                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 213                         *hash_state_out = cpu_to_be32(*(sha256.state + i));
 214                 break;
 215         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 216                 if (crypto_shash_export(shash, &sha512))
 217                         return -EFAULT;
 218                 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
 219                         *hash512_state_out = cpu_to_be64(*(sha512.state + i));
 220                 break;
 221         default:
 222                 return -EFAULT;
 223         }
 224
 225         if (crypto_shash_init(shash))
 226                 return -EFAULT;
 227
 228         if (crypto_shash_update(shash, opad, block_size))
 229                 return -EFAULT;
 230
 231         offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);
 232         hash_state_out = (__be32 *)(hash->sha.state1 + offset);
 233         hash512_state_out = (__be64 *)hash_state_out;
 234
 235         switch (ctx->qat_hash_alg) {
 236         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 237                 if (crypto_shash_export(shash, &sha1))
 238                         return -EFAULT;
 239                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 240                         *hash_state_out = cpu_to_be32(*(sha1.state + i));
 241                 break;
 242         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 243                 if (crypto_shash_export(shash, &sha256))
 244                         return -EFAULT;
 245                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 246                         *hash_state_out = cpu_to_be32(*(sha256.state + i));
 247                 break;
 248         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 249                 if (crypto_shash_export(shash, &sha512))
 250                         return -EFAULT;
 251                 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
 252                         *hash512_state_out = cpu_to_be64(*(sha512.state + i));
 253                 break;
 254         default:
 255                 return -EFAULT;
 256         }
 257         return 0;
 258 }
 259
 260 static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
 261 {
 262         header->hdr_flags =
 263                 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
 264         header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA;
 265         header->comn_req_flags =
 266                 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
 267                                             QAT_COMN_PTR_TYPE_SGL);
 268         ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
 269                                            ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
 270         ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
 271                                   ICP_QAT_FW_LA_PARTIAL_NONE);
 272         ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
 273                                            ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
 274         ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
 275                                 ICP_QAT_FW_LA_NO_PROTO);
 276         ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
 277                                        ICP_QAT_FW_LA_NO_UPDATE_STATE);
 278 }
 279
 280 static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx,
 281                                     int alg, struct crypto_authenc_keys *keys)
 282 {
 283         struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm);
 284         unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize;
 285         struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd;
 286         struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher;
 287         struct icp_qat_hw_auth_algo_blk *hash =
 288                 (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx +
 289                 sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen);
 290         struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req_tmpl;
 291         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
 292         struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
 293         void *ptr = &req_tmpl->cd_ctrl;
 294         struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
 295         struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
 296
 297         /* CD setup */
 298         cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg);
 299         memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
 300         hash->sha.inner_setup.auth_config.config =
 301                 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
 302                                              ctx->qat_hash_alg, digestsize);
 303         hash->sha.inner_setup.auth_counter.counter =
 304                 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
 305
 306         if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
 307                 return -EFAULT;
 308
 309         /* Request setup */
 310         qat_alg_init_common_hdr(header);
 311         header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 312         ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
 313                                    ICP_QAT_FW_LA_RET_AUTH_RES);
 314         ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
 315                                    ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
 316         cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
 317         cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
 318
 319         /* Cipher CD config setup */
 320         cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
 321         cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
 322         cipher_cd_ctrl->cipher_cfg_offset = 0;
 323         ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 324         ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
 325         /* Auth CD config setup */
 326         hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3;
 327         hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
 328         hash_cd_ctrl->inner_res_sz = digestsize;
 329         hash_cd_ctrl->final_sz = digestsize;
 330
 331         switch (ctx->qat_hash_alg) {
 332         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 333                 hash_cd_ctrl->inner_state1_sz =
 334                         round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
 335                 hash_cd_ctrl->inner_state2_sz =
 336                         round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
 337                 break;
 338         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 339                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
 340                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
 341                 break;
 342         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 343                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
 344                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
 345                 break;
 346         default:
 347                 break;
 348         }
 349         hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
 350                         ((sizeof(struct icp_qat_hw_auth_setup) +
 351                          round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
 352         ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
 353         ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
 354         return 0;
 355 }
 356
 357 static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx,
 358                                     int alg, struct crypto_authenc_keys *keys)
 359 {
 360         struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm);
 361         unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize;
 362         struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd;
 363         struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash;
 364         struct icp_qat_hw_cipher_algo_blk *cipher =
 365                 (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx +
 366                 sizeof(struct icp_qat_hw_auth_setup) +
 367                 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);
 368         struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req_tmpl;
 369         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
 370         struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
 371         void *ptr = &req_tmpl->cd_ctrl;
 372         struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
 373         struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
 374         struct icp_qat_fw_la_auth_req_params *auth_param =
 375                 (struct icp_qat_fw_la_auth_req_params *)
 376                 ((char *)&req_tmpl->serv_specif_rqpars +
 377                 sizeof(struct icp_qat_fw_la_cipher_req_params));
 378
 379         /* CD setup */
 380         cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg);
 381         memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
 382         hash->sha.inner_setup.auth_config.config =
 383                 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
 384                                              ctx->qat_hash_alg,
 385                                              digestsize);
 386         hash->sha.inner_setup.auth_counter.counter =
 387                 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
 388
 389         if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
 390                 return -EFAULT;
 391
 392         /* Request setup */
 393         qat_alg_init_common_hdr(header);
 394         header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 395         ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
 396                                    ICP_QAT_FW_LA_NO_RET_AUTH_RES);
 397         ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
 398                                    ICP_QAT_FW_LA_CMP_AUTH_RES);
 399         cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
 400         cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
 401
 402         /* Cipher CD config setup */
 403         cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
 404         cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
 405         cipher_cd_ctrl->cipher_cfg_offset =
 406                 (sizeof(struct icp_qat_hw_auth_setup) +
 407                  roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3;
 408         ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 409         ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
 410
 411         /* Auth CD config setup */
 412         hash_cd_ctrl->hash_cfg_offset = 0;
 413         hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
 414         hash_cd_ctrl->inner_res_sz = digestsize;
 415         hash_cd_ctrl->final_sz = digestsize;
 416
 417         switch (ctx->qat_hash_alg) {
 418         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 419                 hash_cd_ctrl->inner_state1_sz =
 420                         round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
 421                 hash_cd_ctrl->inner_state2_sz =
 422                         round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
 423                 break;
 424         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 425                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
 426                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
 427                 break;
 428         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 429                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
 430                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
 431                 break;
 432         default:
 433                 break;
 434         }
 435
 436         hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
 437                         ((sizeof(struct icp_qat_hw_auth_setup) +
 438                          round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
 439         auth_param->auth_res_sz = digestsize;
 440         ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
 441         ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 442         return 0;
 443 }
 444
 445 static int qat_alg_init_sessions(struct qat_alg_session_ctx *ctx,
 446                                  const uint8_t *key, unsigned int keylen)
 447 {
 448         struct crypto_authenc_keys keys;
 449         int alg;
 450
 451         if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE))
 452                 return -EFAULT;
 453
 454         if (crypto_authenc_extractkeys(&keys, key, keylen))
 455                 goto bad_key;
 456
 457         switch (keys.enckeylen) {
 458         case AES_KEYSIZE_128:
 459                 alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
 460                 break;
 461         case AES_KEYSIZE_192:
 462                 alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
 463                 break;
 464         case AES_KEYSIZE_256:
 465                 alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
 466                 break;
 467         default:
 468                 goto bad_key;
 469                 break;
 470         }
 471
 472         if (qat_alg_init_enc_session(ctx, alg, &keys))
 473                 goto error;
 474
 475         if (qat_alg_init_dec_session(ctx, alg, &keys))
 476                 goto error;
 477
 478         return 0;
 479 bad_key:
 480         crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 481         return -EINVAL;
 482 error:
 483         return -EFAULT;
 484 }
 485
 486 static int qat_alg_setkey(struct crypto_aead *tfm, const uint8_t *key,
 487                           unsigned int keylen)
 488 {
 489         struct qat_alg_session_ctx *ctx = crypto_aead_ctx(tfm);
 490         struct device *dev;
 491
 492         spin_lock(&ctx->lock);
 493         if (ctx->enc_cd) {
 494                 /* rekeying */
 495                 dev = &GET_DEV(ctx->inst->accel_dev);
 496                 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
 497                 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
 498                 memset(&ctx->enc_fw_req_tmpl, 0,
 499                        sizeof(struct icp_qat_fw_la_bulk_req));
 500                 memset(&ctx->dec_fw_req_tmpl, 0,
 501                        sizeof(struct icp_qat_fw_la_bulk_req));
 502         } else {
 503                 /* new key */
 504                 int node = get_current_node();
 505                 struct qat_crypto_instance *inst =
 506                                 qat_crypto_get_instance_node(node);
 507                 if (!inst) {
 508                         spin_unlock(&ctx->lock);
 509                         return -EINVAL;
 510                 }
 511
 512                 dev = &GET_DEV(inst->accel_dev);
 513                 ctx->inst = inst;
 514                 ctx->enc_cd = dma_zalloc_coherent(dev,
 515                                                   sizeof(struct qat_alg_cd),
 516                                                   &ctx->enc_cd_paddr,
 517                                                   GFP_ATOMIC);
 518                 if (!ctx->enc_cd) {
 519                         spin_unlock(&ctx->lock);
 520                         return -ENOMEM;
 521                 }
 522                 ctx->dec_cd = dma_zalloc_coherent(dev,
 523                                                   sizeof(struct qat_alg_cd),
 524                                                   &ctx->dec_cd_paddr,
 525                                                   GFP_ATOMIC);
 526                 if (!ctx->dec_cd) {
 527                         spin_unlock(&ctx->lock);
 528                         goto out_free_enc;
 529                 }
 530         }
 531         spin_unlock(&ctx->lock);
 532         if (qat_alg_init_sessions(ctx, key, keylen))
 533                 goto out_free_all;
 534
 535         return 0;
 536
 537 out_free_all:
 538         dma_free_coherent(dev, sizeof(struct qat_alg_cd),
 539                           ctx->dec_cd, ctx->dec_cd_paddr);
 540         ctx->dec_cd = NULL;
 541 out_free_enc:
 542         dma_free_coherent(dev, sizeof(struct qat_alg_cd),
 543                           ctx->enc_cd, ctx->enc_cd_paddr);
 544         ctx->enc_cd = NULL;
 545         return -ENOMEM;
 546 }
 547
 548 static void qat_alg_free_bufl(struct qat_crypto_instance *inst,
 549                               struct qat_crypto_request *qat_req)
 550 {
 551         struct device *dev = &GET_DEV(inst->accel_dev);
 552         struct qat_alg_buf_list *bl = qat_req->buf.bl;
 553         struct qat_alg_buf_list *blout = qat_req->buf.blout;
 554         dma_addr_t blp = qat_req->buf.blp;
 555         dma_addr_t blpout = qat_req->buf.bloutp;
 556         size_t sz = qat_req->buf.sz;
 557         int i, bufs = bl->num_bufs;
 558
 559         for (i = 0; i < bl->num_bufs; i++)
 560                 dma_unmap_single(dev, bl->bufers[i].addr,
 561                                  bl->bufers[i].len, DMA_BIDIRECTIONAL);
 562
 563         dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
 564         kfree(bl);
 565         if (blp != blpout) {
 566                 /* If out of place operation dma unmap only data */
 567                 int bufless = bufs - blout->num_mapped_bufs;
 568
 569                 for (i = bufless; i < bufs; i++) {
 570                         dma_unmap_single(dev, blout->bufers[i].addr,
 571                                          blout->bufers[i].len,
 572                                          DMA_BIDIRECTIONAL);
 573                 }
 574                 dma_unmap_single(dev, blpout, sz, DMA_TO_DEVICE);
 575                 kfree(blout);
 576         }
 577 }
 578
 579 static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
 580                                struct scatterlist *assoc,
 581                                struct scatterlist *sgl,
 582                                struct scatterlist *sglout, uint8_t *iv,
 583                                uint8_t ivlen,
 584                                struct qat_crypto_request *qat_req)
 585 {
 586         struct device *dev = &GET_DEV(inst->accel_dev);
 587         int i, bufs = 0, n = sg_nents(sgl), assoc_n = sg_nents(assoc);
 588         struct qat_alg_buf_list *bufl;
 589         struct qat_alg_buf_list *buflout = NULL;
 590         dma_addr_t blp;
 591         dma_addr_t bloutp = 0;
 592         struct scatterlist *sg;
 593         size_t sz = sizeof(struct qat_alg_buf_list) +
 594                         ((1 + n + assoc_n) * sizeof(struct qat_alg_buf));
 595
 596         if (unlikely(!n))
 597                 return -EINVAL;
 598
 599         bufl = kmalloc_node(sz, GFP_ATOMIC, inst->accel_dev->numa_node);
 600         if (unlikely(!bufl))
 601                 return -ENOMEM;
 602
 603         blp = dma_map_single(dev, bufl, sz, DMA_TO_DEVICE);
 604         if (unlikely(dma_mapping_error(dev, blp)))
 605                 goto err;
 606
 607         for_each_sg(assoc, sg, assoc_n, i) {
 608                 bufl->bufers[bufs].addr = dma_map_single(dev,
 609                                                          sg_virt(sg),
 610                                                          sg->length,
 611                                                          DMA_BIDIRECTIONAL);
 612                 bufl->bufers[bufs].len = sg->length;
 613                 if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr)))
 614                         goto err;
 615                 bufs++;
 616         }
 617         bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen,
 618                                                  DMA_BIDIRECTIONAL);
 619         bufl->bufers[bufs].len = ivlen;
 620         if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr)))
 621                 goto err;
 622         bufs++;
 623
 624         for_each_sg(sgl, sg, n, i) {
 625                 int y = i + bufs;
 626
 627                 bufl->bufers[y].addr = dma_map_single(dev, sg_virt(sg),
 628                                                       sg->length,
 629                                                       DMA_BIDIRECTIONAL);
 630                 bufl->bufers[y].len = sg->length;
 631                 if (unlikely(dma_mapping_error(dev, bufl->bufers[y].addr)))
 632                         goto err;
 633         }
 634         bufl->num_bufs = n + bufs;
 635         qat_req->buf.bl = bufl;
 636         qat_req->buf.blp = blp;
 637         qat_req->buf.sz = sz;
 638         /* Handle out of place operation */
 639         if (sgl != sglout) {
 640                 struct qat_alg_buf *bufers;
 641
 642                 buflout = kmalloc_node(sz, GFP_ATOMIC,
 643                                        inst->accel_dev->numa_node);
 644                 if (unlikely(!buflout))
 645                         goto err;
 646                 bloutp = dma_map_single(dev, buflout, sz, DMA_TO_DEVICE);
 647                 if (unlikely(dma_mapping_error(dev, bloutp)))
 648                         goto err;
 649                 bufers = buflout->bufers;
 650                 /* For out of place operation dma map only data and
 651                  * reuse assoc mapping and iv */
 652                 for (i = 0; i < bufs; i++) {
 653                         bufers[i].len = bufl->bufers[i].len;
 654                         bufers[i].addr = bufl->bufers[i].addr;
 655                 }
 656                 for_each_sg(sglout, sg, n, i) {
 657                         int y = i + bufs;
 658
 659                         bufers[y].addr = dma_map_single(dev, sg_virt(sg),
 660                                                         sg->length,
 661                                                         DMA_BIDIRECTIONAL);
 662                         buflout->bufers[y].len = sg->length;
 663                         if (unlikely(dma_mapping_error(dev, bufers[y].addr)))
 664                                 goto err;
 665                 }
 666                 buflout->num_bufs = n + bufs;
 667                 buflout->num_mapped_bufs = n;
 668                 qat_req->buf.blout = buflout;
 669                 qat_req->buf.bloutp = bloutp;
 670         } else {
 671                 /* Otherwise set the src and dst to the same address */
 672                 qat_req->buf.bloutp = qat_req->buf.blp;
 673         }
 674         return 0;
 675 err:
 676         dev_err(dev, "Failed to map buf for dma\n");
 677         for_each_sg(sgl, sg, n + bufs, i) {
 678                 if (!dma_mapping_error(dev, bufl->bufers[i].addr)) {
 679                         dma_unmap_single(dev, bufl->bufers[i].addr,
 680                                          bufl->bufers[i].len,
 681                                          DMA_BIDIRECTIONAL);
 682                 }
 683         }
 684         if (!dma_mapping_error(dev, blp))
 685                 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
 686         kfree(bufl);
 687         if (sgl != sglout && buflout) {
 688                 for_each_sg(sglout, sg, n, i) {
 689                         int y = i + bufs;
 690
 691                         if (!dma_mapping_error(dev, buflout->bufers[y].addr))
 692                                 dma_unmap_single(dev, buflout->bufers[y].addr,
 693                                                  buflout->bufers[y].len,
 694                                                  DMA_BIDIRECTIONAL);
 695                 }
 696                 if (!dma_mapping_error(dev, bloutp))
 697                         dma_unmap_single(dev, bloutp, sz, DMA_TO_DEVICE);
 698                 kfree(buflout);
 699         }
 700         return -ENOMEM;
 701 }
 702
 703 void qat_alg_callback(void *resp)
 704 {
 705         struct icp_qat_fw_la_resp *qat_resp = resp;
 706         struct qat_crypto_request *qat_req =
 707                                 (void *)(__force long)qat_resp->opaque_data;
 708         struct qat_alg_session_ctx *ctx = qat_req->ctx;
 709         struct qat_crypto_instance *inst = ctx->inst;
 710         struct aead_request *areq = qat_req->areq;
 711         uint8_t stat_filed = qat_resp->comn_resp.comn_status;
 712         int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
 713
 714         qat_alg_free_bufl(inst, qat_req);
 715         if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
 716                 res = -EBADMSG;
 717         areq->base.complete(&areq->base, res);
 718 }
 719
 720 static int qat_alg_dec(struct aead_request *areq)
 721 {
 722         struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
 723         struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
 724         struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
 725         struct qat_crypto_request *qat_req = aead_request_ctx(areq);
 726         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 727         struct icp_qat_fw_la_auth_req_params *auth_param;
 728         struct icp_qat_fw_la_bulk_req *msg;
 729         int digst_size = crypto_aead_crt(aead_tfm)->authsize;
 730         int ret, ctr = 0;
 731
 732         ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->src, areq->dst,
 733                                   areq->iv, AES_BLOCK_SIZE, qat_req);
 734         if (unlikely(ret))
 735                 return ret;
 736
 737         msg = &qat_req->req;
 738         *msg = ctx->dec_fw_req_tmpl;
 739         qat_req->ctx = ctx;
 740         qat_req->areq = areq;
 741         qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
 742         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
 743         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
 744         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
 745         cipher_param->cipher_length = areq->cryptlen - digst_size;
 746         cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE;
 747         memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE);
 748         auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
 749         auth_param->auth_off = 0;
 750         auth_param->auth_len = areq->assoclen +
 751                                 cipher_param->cipher_length + AES_BLOCK_SIZE;
 752         do {
 753                 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
 754         } while (ret == -EAGAIN && ctr++ < 10);
 755
 756         if (ret == -EAGAIN) {
 757                 qat_alg_free_bufl(ctx->inst, qat_req);
 758                 return -EBUSY;
 759         }
 760         return -EINPROGRESS;
 761 }
 762
 763 static int qat_alg_enc_internal(struct aead_request *areq, uint8_t *iv,
 764                                 int enc_iv)
 765 {
 766         struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
 767         struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
 768         struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
 769         struct qat_crypto_request *qat_req = aead_request_ctx(areq);
 770         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 771         struct icp_qat_fw_la_auth_req_params *auth_param;
 772         struct icp_qat_fw_la_bulk_req *msg;
 773         int ret, ctr = 0;
 774
 775         ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->src, areq->dst,
 776                                   iv, AES_BLOCK_SIZE, qat_req);
 777         if (unlikely(ret))
 778                 return ret;
 779
 780         msg = &qat_req->req;
 781         *msg = ctx->enc_fw_req_tmpl;
 782         qat_req->ctx = ctx;
 783         qat_req->areq = areq;
 784         qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
 785         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
 786         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
 787         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
 788         auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
 789
 790         if (enc_iv) {
 791                 cipher_param->cipher_length = areq->cryptlen + AES_BLOCK_SIZE;
 792                 cipher_param->cipher_offset = areq->assoclen;
 793         } else {
 794                 memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
 795                 cipher_param->cipher_length = areq->cryptlen;
 796                 cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE;
 797         }
 798         auth_param->auth_off = 0;
 799         auth_param->auth_len = areq->assoclen + areq->cryptlen + AES_BLOCK_SIZE;
 800
 801         do {
 802                 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
 803         } while (ret == -EAGAIN && ctr++ < 10);
 804
 805         if (ret == -EAGAIN) {
 806                 qat_alg_free_bufl(ctx->inst, qat_req);
 807                 return -EBUSY;
 808         }
 809         return -EINPROGRESS;
 810 }
 811
 812 static int qat_alg_enc(struct aead_request *areq)
 813 {
 814         return qat_alg_enc_internal(areq, areq->iv, 0);
 815 }
 816
 817 static int qat_alg_genivenc(struct aead_givcrypt_request *req)
 818 {
 819         struct crypto_aead *aead_tfm = crypto_aead_reqtfm(&req->areq);
 820         struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
 821         struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
 822         __be64 seq;
 823
 824         memcpy(req->giv, ctx->salt, AES_BLOCK_SIZE);
 825         seq = cpu_to_be64(req->seq);
 826         memcpy(req->giv + AES_BLOCK_SIZE - sizeof(uint64_t),
 827                &seq, sizeof(uint64_t));
 828         return qat_alg_enc_internal(&req->areq, req->giv, 1);
 829 }
 830
 831 static int qat_alg_init(struct crypto_tfm *tfm,
 832                         enum icp_qat_hw_auth_algo hash, const char *hash_name)
 833 {
 834         struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
 835
 836         memset(ctx, '\0', sizeof(*ctx));
 837         ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
 838         if (IS_ERR(ctx->hash_tfm))
 839                 return -EFAULT;
 840         spin_lock_init(&ctx->lock);
 841         ctx->qat_hash_alg = hash;
 842         tfm->crt_aead.reqsize = sizeof(struct aead_request) +
 843                                 sizeof(struct qat_crypto_request);
 844         ctx->tfm = tfm;
 845         return 0;
 846 }
 847
 848 static int qat_alg_sha1_init(struct crypto_tfm *tfm)
 849 {
 850         return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1");
 851 }
 852
 853 static int qat_alg_sha256_init(struct crypto_tfm *tfm)
 854 {
 855         return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256");
 856 }
 857
 858 static int qat_alg_sha512_init(struct crypto_tfm *tfm)
 859 {
 860         return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512");
 861 }
 862
 863 static void qat_alg_exit(struct crypto_tfm *tfm)
 864 {
 865         struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
 866         struct qat_crypto_instance *inst = ctx->inst;
 867         struct device *dev;
 868
 869         if (!IS_ERR(ctx->hash_tfm))
 870                 crypto_free_shash(ctx->hash_tfm);
 871
 872         if (!inst)
 873                 return;
 874
 875         dev = &GET_DEV(inst->accel_dev);
 876         if (ctx->enc_cd)
 877                 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
 878                                   ctx->enc_cd, ctx->enc_cd_paddr);
 879         if (ctx->dec_cd)
 880                 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
 881                                   ctx->dec_cd, ctx->dec_cd_paddr);
 882         qat_crypto_put_instance(inst);
 883 }
 884
 885 static struct crypto_alg qat_algs[] = { {
 886         .cra_name = "authenc(hmac(sha1),cbc(aes))",
 887         .cra_driver_name = "qat_aes_cbc_hmac_sha1",
 888         .cra_priority = 4001,
 889         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
 890         .cra_blocksize = AES_BLOCK_SIZE,
 891         .cra_ctxsize = sizeof(struct qat_alg_session_ctx),
 892         .cra_alignmask = 0,
 893         .cra_type = &crypto_aead_type,
 894         .cra_module = THIS_MODULE,
 895         .cra_init = qat_alg_sha1_init,
 896         .cra_exit = qat_alg_exit,
 897         .cra_u = {
 898                 .aead = {
 899                         .setkey = qat_alg_setkey,
 900                         .decrypt = qat_alg_dec,
 901                         .encrypt = qat_alg_enc,
 902                         .givencrypt = qat_alg_genivenc,
 903                         .ivsize = AES_BLOCK_SIZE,
 904                         .maxauthsize = SHA1_DIGEST_SIZE,
 905                 },
 906         },
 907 }, {
 908         .cra_name = "authenc(hmac(sha256),cbc(aes))",
 909         .cra_driver_name = "qat_aes_cbc_hmac_sha256",
 910         .cra_priority = 4001,
 911         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
 912         .cra_blocksize = AES_BLOCK_SIZE,
 913         .cra_ctxsize = sizeof(struct qat_alg_session_ctx),
 914         .cra_alignmask = 0,
 915         .cra_type = &crypto_aead_type,
 916         .cra_module = THIS_MODULE,
 917         .cra_init = qat_alg_sha256_init,
 918         .cra_exit = qat_alg_exit,
 919         .cra_u = {
 920                 .aead = {
 921                         .setkey = qat_alg_setkey,
 922                         .decrypt = qat_alg_dec,
 923                         .encrypt = qat_alg_enc,
 924                         .givencrypt = qat_alg_genivenc,
 925                         .ivsize = AES_BLOCK_SIZE,
 926                         .maxauthsize = SHA256_DIGEST_SIZE,
 927                 },
 928         },
 929 }, {
 930         .cra_name = "authenc(hmac(sha512),cbc(aes))",
 931         .cra_driver_name = "qat_aes_cbc_hmac_sha512",
 932         .cra_priority = 4001,
 933         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
 934         .cra_blocksize = AES_BLOCK_SIZE,
 935         .cra_ctxsize = sizeof(struct qat_alg_session_ctx),
 936         .cra_alignmask = 0,
 937         .cra_type = &crypto_aead_type,
 938         .cra_module = THIS_MODULE,
 939         .cra_init = qat_alg_sha512_init,
 940         .cra_exit = qat_alg_exit,
 941         .cra_u = {
 942                 .aead = {
 943                         .setkey = qat_alg_setkey,
 944                         .decrypt = qat_alg_dec,
 945                         .encrypt = qat_alg_enc,
 946                         .givencrypt = qat_alg_genivenc,
 947                         .ivsize = AES_BLOCK_SIZE,
 948                         .maxauthsize = SHA512_DIGEST_SIZE,
 949                 },
 950         },
 951 } };
 952
 953 int qat_algs_register(void)
 954 {
 955         if (atomic_add_return(1, &active_dev) == 1) {
 956                 int i;
 957
 958                 for (i = 0; i < ARRAY_SIZE(qat_algs); i++)
 959                         qat_algs[i].cra_flags = CRYPTO_ALG_TYPE_AEAD |
 960                                                 CRYPTO_ALG_ASYNC;
 961                 return crypto_register_algs(qat_algs, ARRAY_SIZE(qat_algs));
 962         }
 963         return 0;
 964 }
 965
 966 int qat_algs_unregister(void)
 967 {
 968         if (atomic_sub_return(1, &active_dev) == 0)
 969                 return crypto_unregister_algs(qat_algs, ARRAY_SIZE(qat_algs));
 970         return 0;
 971 }
 972
 973 int qat_algs_init(void)
 974 {
 975         atomic_set(&active_dev, 0);
 976         crypto_get_default_rng();
 977         return 0;
 978 }
 979
 980 void qat_algs_exit(void)
 981 {
 982         crypto_put_default_rng();
 983 }