crypto/crypto_engine.c

   1 /*
   2  * Handle async block request by crypto hardware engine.
   3  *
   4  * Copyright (C) 2016 Linaro, Inc.
   5  *
   6  * Author: Baolin Wang <baolin.wang@linaro.org>
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms of the GNU General Public License as published by the Free
  10  * Software Foundation; either version 2 of the License, or (at your option)
  11  * any later version.
  12  *
  13  */
  14
  15 #include <linux/err.h>
  16 #include <linux/delay.h>
  17 #include <crypto/engine.h>
  18 #include "internal.h"
  19
  20 #define CRYPTO_ENGINE_MAX_QLEN 10
  21
  22 void crypto_finalize_request(struct crypto_engine *engine,
  23                              struct ablkcipher_request *req, int err);
  24
  25 /**
  26  * crypto_pump_requests - dequeue one request from engine queue to process
  27  * @engine: the hardware engine
  28  * @in_kthread: true if we are in the context of the request pump thread
  29  *
  30  * This function checks if there is any request in the engine queue that
  31  * needs processing and if so call out to the driver to initialize hardware
  32  * and handle each request.
  33  */
  34 static void crypto_pump_requests(struct crypto_engine *engine,
  35                                  bool in_kthread)
  36 {
  37         struct crypto_async_request *async_req, *backlog;
  38         struct ablkcipher_request *req;
  39         unsigned long flags;
  40         bool was_busy = false;
  41         int ret;
  42
  43         spin_lock_irqsave(&engine->queue_lock, flags);
  44
  45         /* Make sure we are not already running a request */
  46         if (engine->cur_req)
  47                 goto out;
  48
  49         /* If another context is idling then defer */
  50         if (engine->idling) {
  51                 queue_kthread_work(&engine->kworker, &engine->pump_requests);
  52                 goto out;
  53         }
  54
  55         /* Check if the engine queue is idle */
  56         if (!crypto_queue_len(&engine->queue) || !engine->running) {
  57                 if (!engine->busy)
  58                         goto out;
  59
  60                 /* Only do teardown in the thread */
  61                 if (!in_kthread) {
  62                         queue_kthread_work(&engine->kworker,
  63                                            &engine->pump_requests);
  64                         goto out;
  65                 }
  66
  67                 engine->busy = false;
  68                 engine->idling = true;
  69                 spin_unlock_irqrestore(&engine->queue_lock, flags);
  70
  71                 if (engine->unprepare_crypt_hardware &&
  72                     engine->unprepare_crypt_hardware(engine))
  73                         pr_err("failed to unprepare crypt hardware\n");
  74
  75                 spin_lock_irqsave(&engine->queue_lock, flags);
  76                 engine->idling = false;
  77                 goto out;
  78         }
  79
  80         /* Get the fist request from the engine queue to handle */
  81         backlog = crypto_get_backlog(&engine->queue);
  82         async_req = crypto_dequeue_request(&engine->queue);
  83         if (!async_req)
  84                 goto out;
  85
  86         req = ablkcipher_request_cast(async_req);
  87
  88         engine->cur_req = req;
  89         if (backlog)
  90                 backlog->complete(backlog, -EINPROGRESS);
  91
  92         if (engine->busy)
  93                 was_busy = true;
  94         else
  95                 engine->busy = true;
  96
  97         spin_unlock_irqrestore(&engine->queue_lock, flags);
  98
  99         /* Until here we get the request need to be encrypted successfully */
 100         if (!was_busy && engine->prepare_crypt_hardware) {
 101                 ret = engine->prepare_crypt_hardware(engine);
 102                 if (ret) {
 103                         pr_err("failed to prepare crypt hardware\n");
 104                         goto req_err;
 105                 }
 106         }
 107
 108         if (engine->prepare_request) {
 109                 ret = engine->prepare_request(engine, engine->cur_req);
 110                 if (ret) {
 111                         pr_err("failed to prepare request: %d\n", ret);
 112                         goto req_err;
 113                 }
 114                 engine->cur_req_prepared = true;
 115         }
 116
 117         ret = engine->crypt_one_request(engine, engine->cur_req);
 118         if (ret) {
 119                 pr_err("failed to crypt one request from queue\n");
 120                 goto req_err;
 121         }
 122         return;
 123
 124 req_err:
 125         crypto_finalize_request(engine, engine->cur_req, ret);
 126         return;
 127
 128 out:
 129         spin_unlock_irqrestore(&engine->queue_lock, flags);
 130 }
 131
 132 static void crypto_pump_work(struct kthread_work *work)
 133 {
 134         struct crypto_engine *engine =
 135                 container_of(work, struct crypto_engine, pump_requests);
 136
 137         crypto_pump_requests(engine, true);
 138 }
 139
 140 /**
 141  * crypto_transfer_request - transfer the new request into the engine queue
 142  * @engine: the hardware engine
 143  * @req: the request need to be listed into the engine queue
 144  */
 145 int crypto_transfer_request(struct crypto_engine *engine,
 146                             struct ablkcipher_request *req, bool need_pump)
 147 {
 148         unsigned long flags;
 149         int ret;
 150
 151         spin_lock_irqsave(&engine->queue_lock, flags);
 152
 153         if (!engine->running) {
 154                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 155                 return -ESHUTDOWN;
 156         }
 157
 158         ret = ablkcipher_enqueue_request(&engine->queue, req);
 159
 160         if (!engine->busy && need_pump)
 161                 queue_kthread_work(&engine->kworker, &engine->pump_requests);
 162
 163         spin_unlock_irqrestore(&engine->queue_lock, flags);
 164         return ret;
 165 }
 166 EXPORT_SYMBOL_GPL(crypto_transfer_request);
 167
 168 /**
 169  * crypto_transfer_request_to_engine - transfer one request to list into the
 170  * engine queue
 171  * @engine: the hardware engine
 172  * @req: the request need to be listed into the engine queue
 173  */
 174 int crypto_transfer_request_to_engine(struct crypto_engine *engine,
 175                                       struct ablkcipher_request *req)
 176 {
 177         return crypto_transfer_request(engine, req, true);
 178 }
 179 EXPORT_SYMBOL_GPL(crypto_transfer_request_to_engine);
 180
 181 /**
 182  * crypto_finalize_request - finalize one request if the request is done
 183  * @engine: the hardware engine
 184  * @req: the request need to be finalized
 185  * @err: error number
 186  */
 187 void crypto_finalize_request(struct crypto_engine *engine,
 188                              struct ablkcipher_request *req, int err)
 189 {
 190         unsigned long flags;
 191         bool finalize_cur_req = false;
 192         int ret;
 193
 194         spin_lock_irqsave(&engine->queue_lock, flags);
 195         if (engine->cur_req == req)
 196                 finalize_cur_req = true;
 197         spin_unlock_irqrestore(&engine->queue_lock, flags);
 198
 199         if (finalize_cur_req) {
 200                 if (engine->cur_req_prepared && engine->unprepare_request) {
 201                         ret = engine->unprepare_request(engine, req);
 202                         if (ret)
 203                                 pr_err("failed to unprepare request\n");
 204                 }
 205
 206                 spin_lock_irqsave(&engine->queue_lock, flags);
 207                 engine->cur_req = NULL;
 208                 engine->cur_req_prepared = false;
 209                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 210         }
 211
 212         req->base.complete(&req->base, err);
 213
 214         queue_kthread_work(&engine->kworker, &engine->pump_requests);
 215 }
 216 EXPORT_SYMBOL_GPL(crypto_finalize_request);
 217
 218 /**
 219  * crypto_engine_start - start the hardware engine
 220  * @engine: the hardware engine need to be started
 221  *
 222  * Return 0 on success, else on fail.
 223  */
 224 int crypto_engine_start(struct crypto_engine *engine)
 225 {
 226         unsigned long flags;
 227
 228         spin_lock_irqsave(&engine->queue_lock, flags);
 229
 230         if (engine->running || engine->busy) {
 231                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 232                 return -EBUSY;
 233         }
 234
 235         engine->running = true;
 236         spin_unlock_irqrestore(&engine->queue_lock, flags);
 237
 238         queue_kthread_work(&engine->kworker, &engine->pump_requests);
 239
 240         return 0;
 241 }
 242 EXPORT_SYMBOL_GPL(crypto_engine_start);
 243
 244 /**
 245  * crypto_engine_stop - stop the hardware engine
 246  * @engine: the hardware engine need to be stopped
 247  *
 248  * Return 0 on success, else on fail.
 249  */
 250 int crypto_engine_stop(struct crypto_engine *engine)
 251 {
 252         unsigned long flags;
 253         unsigned limit = 500;
 254         int ret = 0;
 255
 256         spin_lock_irqsave(&engine->queue_lock, flags);
 257
 258         /*
 259          * If the engine queue is not empty or the engine is on busy state,
 260          * we need to wait for a while to pump the requests of engine queue.
 261          */
 262         while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {
 263                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 264                 msleep(20);
 265                 spin_lock_irqsave(&engine->queue_lock, flags);
 266         }
 267
 268         if (crypto_queue_len(&engine->queue) || engine->busy)
 269                 ret = -EBUSY;
 270         else
 271                 engine->running = false;
 272
 273         spin_unlock_irqrestore(&engine->queue_lock, flags);
 274
 275         if (ret)
 276                 pr_warn("could not stop engine\n");
 277
 278         return ret;
 279 }
 280 EXPORT_SYMBOL_GPL(crypto_engine_stop);
 281
 282 /**
 283  * crypto_engine_alloc_init - allocate crypto hardware engine structure and
 284  * initialize it.
 285  * @dev: the device attached with one hardware engine
 286  * @rt: whether this queue is set to run as a realtime task
 287  *
 288  * This must be called from context that can sleep.
 289  * Return: the crypto engine structure on success, else NULL.
 290  */
 291 struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
 292 {
 293         struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
 294         struct crypto_engine *engine;
 295
 296         if (!dev)
 297                 return NULL;
 298
 299         engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
 300         if (!engine)
 301                 return NULL;
 302
 303         engine->rt = rt;
 304         engine->running = false;
 305         engine->busy = false;
 306         engine->idling = false;
 307         engine->cur_req_prepared = false;
 308         engine->priv_data = dev;
 309         snprintf(engine->name, sizeof(engine->name),
 310                  "%s-engine", dev_name(dev));
 311
 312         crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN);
 313         spin_lock_init(&engine->queue_lock);
 314
 315         init_kthread_worker(&engine->kworker);
 316         engine->kworker_task = kthread_run(kthread_worker_fn,
 317                                            &engine->kworker, "%s",
 318                                            engine->name);
 319         if (IS_ERR(engine->kworker_task)) {
 320                 dev_err(dev, "failed to create crypto request pump task\n");
 321                 return NULL;
 322         }
 323         init_kthread_work(&engine->pump_requests, crypto_pump_work);
 324
 325         if (engine->rt) {
 326                 dev_info(dev, "will run requests pump with realtime priority\n");
 327                 sched_setscheduler(engine->kworker_task, SCHED_FIFO, &param);
 328         }
 329
 330         return engine;
 331 }
 332 EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
 333
 334 /**
 335  * crypto_engine_exit - free the resources of hardware engine when exit
 336  * @engine: the hardware engine need to be freed
 337  *
 338  * Return 0 for success.
 339  */
 340 int crypto_engine_exit(struct crypto_engine *engine)
 341 {
 342         int ret;
 343
 344         ret = crypto_engine_stop(engine);
 345         if (ret)
 346                 return ret;
 347
 348         flush_kthread_worker(&engine->kworker);
 349         kthread_stop(engine->kworker_task);
 350
 351         return 0;
 352 }
 353 EXPORT_SYMBOL_GPL(crypto_engine_exit);
 354
 355 MODULE_LICENSE("GPL");
 356 MODULE_DESCRIPTION("Crypto hardware engine framework");