arch/cris/arch-v32/drivers/cryptocop.c

   1 /*
   2  * Stream co-processor driver for the ETRAX FS
   3  *
   4  *    Copyright (C) 2003-2007  Axis Communications AB
   5  */
   6
   7 #include <linux/init.h>
   8 #include <linux/sched.h>
   9 #include <linux/module.h>
  10 #include <linux/slab.h>
  11 #include <linux/string.h>
  12 #include <linux/fs.h>
  13 #include <linux/mm.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/stddef.h>
  16
  17 #include <asm/uaccess.h>
  18 #include <asm/io.h>
  19 #include <linux/atomic.h>
  20
  21 #include <linux/list.h>
  22 #include <linux/interrupt.h>
  23
  24 #include <asm/signal.h>
  25 #include <asm/irq.h>
  26
  27 #include <dma.h>
  28 #include <hwregs/dma.h>
  29 #include <hwregs/reg_map.h>
  30 #include <hwregs/reg_rdwr.h>
  31 #include <hwregs/intr_vect_defs.h>
  32
  33 #include <hwregs/strcop.h>
  34 #include <hwregs/strcop_defs.h>
  35 #include <cryptocop.h>
  36
  37 #ifdef CONFIG_ETRAXFS
  38 #define IN_DMA 9
  39 #define OUT_DMA 8
  40 #define IN_DMA_INST regi_dma9
  41 #define OUT_DMA_INST regi_dma8
  42 #define DMA_IRQ DMA9_INTR_VECT
  43 #else
  44 #define IN_DMA 3
  45 #define OUT_DMA 2
  46 #define IN_DMA_INST regi_dma3
  47 #define OUT_DMA_INST regi_dma2
  48 #define DMA_IRQ DMA3_INTR_VECT
  49 #endif
  50
  51 #define DESCR_ALLOC_PAD  (31)
  52
  53 struct cryptocop_dma_desc {
  54         char *free_buf; /* If non-null will be kfreed in free_cdesc() */
  55         dma_descr_data *dma_descr;
  56
  57         unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
  58
  59         unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
  60         struct cryptocop_dma_desc *next;
  61 };
  62
  63
  64 struct cryptocop_int_operation{
  65         void                        *alloc_ptr;
  66         cryptocop_session_id        sid;
  67
  68         dma_descr_context           ctx_out;
  69         dma_descr_context           ctx_in;
  70
  71         /* DMA descriptors allocated by driver. */
  72         struct cryptocop_dma_desc   *cdesc_out;
  73         struct cryptocop_dma_desc   *cdesc_in;
  74
  75         /* Strcop config to use. */
  76         cryptocop_3des_mode         tdes_mode;
  77         cryptocop_csum_type         csum_mode;
  78
  79         /* DMA descrs provided by consumer. */
  80         dma_descr_data              *ddesc_out;
  81         dma_descr_data              *ddesc_in;
  82 };
  83
  84
  85 struct cryptocop_tfrm_ctx {
  86         cryptocop_tfrm_id tid;
  87         unsigned int blocklength;
  88
  89         unsigned int start_ix;
  90
  91         struct cryptocop_tfrm_cfg *tcfg;
  92         struct cryptocop_transform_ctx *tctx;
  93
  94         unsigned char previous_src;
  95         unsigned char current_src;
  96
  97         /* Values to use in metadata out. */
  98         unsigned char hash_conf;
  99         unsigned char hash_mode;
 100         unsigned char ciph_conf;
 101         unsigned char cbcmode;
 102         unsigned char decrypt;
 103
 104         unsigned int requires_padding:1;
 105         unsigned int strict_block_length:1;
 106         unsigned int active:1;
 107         unsigned int done:1;
 108         size_t consumed;
 109         size_t produced;
 110
 111         /* Pad (input) descriptors to put in the DMA out list when the transform
 112          * output is put on the DMA in list. */
 113         struct cryptocop_dma_desc *pad_descs;
 114
 115         struct cryptocop_tfrm_ctx *prev_src;
 116         struct cryptocop_tfrm_ctx *curr_src;
 117
 118         /* Mapping to HW. */
 119         unsigned char unit_no;
 120 };
 121
 122
 123 struct cryptocop_private{
 124         cryptocop_session_id sid;
 125         struct cryptocop_private *next;
 126 };
 127
 128 /* Session list. */
 129
 130 struct cryptocop_transform_ctx{
 131         struct cryptocop_transform_init init;
 132         unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
 133         unsigned int dec_key_set:1;
 134
 135         struct cryptocop_transform_ctx *next;
 136 };
 137
 138
 139 struct cryptocop_session{
 140         cryptocop_session_id sid;
 141
 142         struct cryptocop_transform_ctx *tfrm_ctx;
 143
 144         struct cryptocop_session *next;
 145 };
 146
 147 /* Priority levels for jobs sent to the cryptocop.  Checksum operations from
 148    kernel have highest priority since TCPIP stack processing must not
 149    be a bottleneck. */
 150 typedef enum {
 151         cryptocop_prio_kernel_csum = 0,
 152         cryptocop_prio_kernel = 1,
 153         cryptocop_prio_user = 2,
 154         cryptocop_prio_no_prios = 3
 155 } cryptocop_queue_priority;
 156
 157 struct cryptocop_prio_queue{
 158         struct list_head jobs;
 159         cryptocop_queue_priority prio;
 160 };
 161
 162 struct cryptocop_prio_job{
 163         struct list_head node;
 164         cryptocop_queue_priority prio;
 165
 166         struct cryptocop_operation *oper;
 167         struct cryptocop_int_operation *iop;
 168 };
 169
 170 struct ioctl_job_cb_ctx {
 171         unsigned int processed:1;
 172 };
 173
 174
 175 static struct cryptocop_session *cryptocop_sessions = NULL;
 176 spinlock_t cryptocop_sessions_lock;
 177
 178 /* Next Session ID to assign. */
 179 static cryptocop_session_id next_sid = 1;
 180
 181 /* Pad for checksum. */
 182 static const char csum_zero_pad[1] = {0x00};
 183
 184 /* Trash buffer for mem2mem operations. */
 185 #define MEM2MEM_DISCARD_BUF_LENGTH  (512)
 186 static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
 187
 188 /* Descriptor pool. */
 189 /* FIXME Tweak this value. */
 190 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE   (100)
 191 static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
 192 static struct cryptocop_dma_desc *descr_pool_free_list;
 193 static int descr_pool_no_free;
 194 static spinlock_t descr_pool_lock;
 195
 196 /* Lock to stop cryptocop to start processing of a new operation. The holder
 197    of this lock MUST call cryptocop_start_job() after it is unlocked. */
 198 spinlock_t cryptocop_process_lock;
 199
 200 static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
 201 static spinlock_t cryptocop_job_queue_lock;
 202 static struct cryptocop_prio_job *cryptocop_running_job = NULL;
 203 static spinlock_t running_job_lock;
 204
 205 /* The interrupt handler appends completed jobs to this list. The scehduled
 206  * tasklet removes them upon sending the response to the crypto consumer. */
 207 static struct list_head cryptocop_completed_jobs;
 208 static spinlock_t cryptocop_completed_jobs_lock;
 209
 210 DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
 211
 212
 213 /** Local functions. **/
 214
 215 static int cryptocop_open(struct inode *, struct file *);
 216
 217 static int cryptocop_release(struct inode *, struct file *);
 218
 219 static long cryptocop_ioctl(struct file *file,
 220                            unsigned int cmd, unsigned long arg);
 221
 222 static void cryptocop_start_job(void);
 223
 224 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
 225 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
 226
 227 static int cryptocop_job_queue_init(void);
 228 static void cryptocop_job_queue_close(void);
 229
 230 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
 231
 232 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
 233
 234 static int transform_ok(struct cryptocop_transform_init *tinit);
 235
 236 static struct cryptocop_session *get_session(cryptocop_session_id sid);
 237
 238 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
 239
 240 static void delete_internal_operation(struct cryptocop_int_operation *iop);
 241
 242 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength);
 243
 244 static int init_stream_coprocessor(void);
 245
 246 static void __exit exit_stream_coprocessor(void);
 247
 248 /*#define LDEBUG*/
 249 #ifdef LDEBUG
 250 #define DEBUG(s) s
 251 #define DEBUG_API(s) s
 252 static void print_cryptocop_operation(struct cryptocop_operation *cop);
 253 static void print_dma_descriptors(struct cryptocop_int_operation *iop);
 254 static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
 255 static void print_lock_status(void);
 256 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
 257 #define assert(s) do{if (!(s)) panic(#s);} while(0);
 258 #else
 259 #define DEBUG(s)
 260 #define DEBUG_API(s)
 261 #define assert(s)
 262 #endif
 263
 264
 265 /* Transform constants. */
 266 #define DES_BLOCK_LENGTH   (8)
 267 #define AES_BLOCK_LENGTH   (16)
 268 #define MD5_BLOCK_LENGTH   (64)
 269 #define SHA1_BLOCK_LENGTH  (64)
 270 #define CSUM_BLOCK_LENGTH  (2)
 271 #define MD5_STATE_LENGTH   (16)
 272 #define SHA1_STATE_LENGTH  (20)
 273
 274 /* The device number. */
 275 #define CRYPTOCOP_MAJOR    (254)
 276 #define CRYPTOCOP_MINOR    (0)
 277
 278
 279
 280 const struct file_operations cryptocop_fops = {
 281         .owner          = THIS_MODULE,
 282         .open           = cryptocop_open,
 283         .release        = cryptocop_release,
 284         .unlocked_ioctl = cryptocop_ioctl,
 285         .llseek         = noop_llseek,
 286 };
 287
 288
 289 static void free_cdesc(struct cryptocop_dma_desc *cdesc)
 290 {
 291         DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
 292         kfree(cdesc->free_buf);
 293
 294         if (cdesc->from_pool) {
 295                 unsigned long int flags;
 296                 spin_lock_irqsave(&descr_pool_lock, flags);
 297                 cdesc->next = descr_pool_free_list;
 298                 descr_pool_free_list = cdesc;
 299                 ++descr_pool_no_free;
 300                 spin_unlock_irqrestore(&descr_pool_lock, flags);
 301         } else {
 302                 kfree(cdesc);
 303         }
 304 }
 305
 306
 307 static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
 308 {
 309         int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
 310         struct cryptocop_dma_desc *cdesc;
 311
 312         if (use_pool) {
 313                 unsigned long int flags;
 314                 spin_lock_irqsave(&descr_pool_lock, flags);
 315                 if (!descr_pool_free_list) {
 316                         spin_unlock_irqrestore(&descr_pool_lock, flags);
 317                         DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
 318                         return NULL;
 319                 }
 320                 cdesc = descr_pool_free_list;
 321                 descr_pool_free_list = descr_pool_free_list->next;
 322                 --descr_pool_no_free;
 323                 spin_unlock_irqrestore(&descr_pool_lock, flags);
 324                 cdesc->from_pool = 1;
 325         } else {
 326                 cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
 327                 if (!cdesc) {
 328                         DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
 329                         return NULL;
 330                 }
 331                 cdesc->from_pool = 0;
 332         }
 333         cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
 334
 335         cdesc->next = NULL;
 336
 337         cdesc->free_buf = NULL;
 338         cdesc->dma_descr->out_eop = 0;
 339         cdesc->dma_descr->in_eop = 0;
 340         cdesc->dma_descr->intr = 0;
 341         cdesc->dma_descr->eol = 0;
 342         cdesc->dma_descr->wait = 0;
 343         cdesc->dma_descr->buf = NULL;
 344         cdesc->dma_descr->after = NULL;
 345
 346         DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
 347         return cdesc;
 348 }
 349
 350
 351 static void setup_descr_chain(struct cryptocop_dma_desc *cd)
 352 {
 353         DEBUG(printk("setup_descr_chain: entering\n"));
 354         while (cd) {
 355                 if (cd->next) {
 356                         cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
 357                 } else {
 358                         cd->dma_descr->next = NULL;
 359                 }
 360                 cd = cd->next;
 361         }
 362         DEBUG(printk("setup_descr_chain: exit\n"));
 363 }
 364
 365
 366 /* Create a pad descriptor for the transform.
 367  * Return -1 for error, 0 if pad created. */
 368 static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
 369 {
 370         struct cryptocop_dma_desc        *cdesc = NULL;
 371         int                              error = 0;
 372         struct strcop_meta_out           mo = {
 373                 .ciphsel = src_none,
 374                 .hashsel = src_none,
 375                 .csumsel = src_none
 376         };
 377         char                             *pad;
 378         size_t                           plen;
 379
 380         DEBUG(printk("create_pad_descriptor: start.\n"));
 381         /* Setup pad descriptor. */
 382
 383         DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
 384         cdesc = alloc_cdesc(alloc_flag);
 385         if (!cdesc){
 386                 DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
 387                 goto error_cleanup;
 388         }
 389         switch (tc->unit_no) {
 390         case src_md5:
 391                 error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
 392                 if (error){
 393                         DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
 394                         goto error_cleanup;
 395                 }
 396                 cdesc->free_buf = pad;
 397                 mo.hashsel = src_dma;
 398                 mo.hashconf = tc->hash_conf;
 399                 mo.hashmode = tc->hash_mode;
 400                 break;
 401         case src_sha1:
 402                 error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
 403                 if (error){
 404                         DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
 405                         goto error_cleanup;
 406                 }
 407                 cdesc->free_buf = pad;
 408                 mo.hashsel = src_dma;
 409                 mo.hashconf = tc->hash_conf;
 410                 mo.hashmode = tc->hash_mode;
 411                 break;
 412         case src_csum:
 413                 if (tc->consumed % tc->blocklength){
 414                         pad = (char*)csum_zero_pad;
 415                         plen = 1;
 416                 } else {
 417                         pad = (char*)cdesc; /* Use any pointer. */
 418                         plen = 0;
 419                 }
 420                 mo.csumsel = src_dma;
 421                 break;
 422         }
 423         cdesc->dma_descr->wait = 1;
 424         cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed.  EOP is ok here since the padded unit is the only one active. */
 425         cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
 426         cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
 427
 428         cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 429         *pad_desc = cdesc;
 430
 431         return 0;
 432
 433  error_cleanup:
 434         if (cdesc) free_cdesc(cdesc);
 435         return -1;
 436 }
 437
 438
 439 static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
 440 {
 441         struct cryptocop_dma_desc  *key_desc = alloc_cdesc(alloc_flag);
 442         struct strcop_meta_out     mo = {0};
 443
 444         DEBUG(printk("setup_key_dl_desc\n"));
 445
 446         if (!key_desc) {
 447                 DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
 448                 return -ENOMEM;
 449         }
 450
 451         /* Download key. */
 452         if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
 453                 /* Precook the AES decrypt key. */
 454                 if (!tc->tctx->dec_key_set){
 455                         get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
 456                         tc->tctx->dec_key_set = 1;
 457                 }
 458                 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
 459                 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
 460         } else {
 461                 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
 462                 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
 463         }
 464         /* Setup metadata. */
 465         mo.dlkey = 1;
 466         switch (tc->tctx->init.keylen) {
 467         case 64:
 468                 mo.decrypt = 0;
 469                 mo.hashmode = 0;
 470                 break;
 471         case 128:
 472                 mo.decrypt = 0;
 473                 mo.hashmode = 1;
 474                 break;
 475         case 192:
 476                 mo.decrypt = 1;
 477                 mo.hashmode = 0;
 478                 break;
 479         case 256:
 480                 mo.decrypt = 1;
 481                 mo.hashmode = 1;
 482                 break;
 483         default:
 484                 break;
 485         }
 486         mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
 487         key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 488
 489         key_desc->dma_descr->out_eop = 1;
 490         key_desc->dma_descr->wait = 1;
 491         key_desc->dma_descr->intr = 0;
 492
 493         *kd = key_desc;
 494         return 0;
 495 }
 496
 497 static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
 498 {
 499         struct cryptocop_dma_desc  *iv_desc = alloc_cdesc(alloc_flag);
 500         struct strcop_meta_out     mo = {0};
 501
 502         DEBUG(printk("setup_cipher_iv_desc\n"));
 503
 504         if (!iv_desc) {
 505                 DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
 506                 return -ENOMEM;
 507         }
 508         /* Download IV. */
 509         iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
 510         iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
 511
 512         /* Setup metadata. */
 513         mo.hashsel = mo.csumsel = src_none;
 514         mo.ciphsel = src_dma;
 515         mo.ciphconf = tc->ciph_conf;
 516         mo.cbcmode = tc->cbcmode;
 517
 518         iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 519
 520         iv_desc->dma_descr->out_eop = 0;
 521         iv_desc->dma_descr->wait = 1;
 522         iv_desc->dma_descr->intr = 0;
 523
 524         *id = iv_desc;
 525         return 0;
 526 }
 527
 528 /* Map the output length of the transform to operation output starting on the inject index. */
 529 static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
 530 {
 531         int                        err = 0;
 532         struct cryptocop_dma_desc  head = {0};
 533         struct cryptocop_dma_desc  *outdesc = &head;
 534         size_t                     iov_offset = 0;
 535         size_t                     out_ix = 0;
 536         int                        outiov_ix = 0;
 537         struct strcop_meta_in      mi = {0};
 538
 539         size_t                     out_length = tc->produced;
 540         int                        rem_length;
 541         int                        dlength;
 542
 543         assert(out_length != 0);
 544         if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
 545                 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
 546                 return -EINVAL;
 547         }
 548         /* Traverse the out iovec until the result inject index is reached. */
 549         while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
 550                 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
 551                 outiov_ix++;
 552         }
 553         if (outiov_ix >= operation->tfrm_op.outcount){
 554                 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
 555                 return -EINVAL;
 556         }
 557         iov_offset = tc->tcfg->inject_ix - out_ix;
 558         mi.dmasel = tc->unit_no;
 559
 560         /* Setup the output descriptors. */
 561         while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
 562                 outdesc->next = alloc_cdesc(alloc_flag);
 563                 if (!outdesc->next) {
 564                         DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
 565                         err = -ENOMEM;
 566                         goto error_cleanup;
 567                 }
 568                 outdesc = outdesc->next;
 569                 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
 570                 dlength = (out_length < rem_length) ? out_length : rem_length;
 571
 572                 DEBUG(printk("create_input_descriptors:\n"
 573                              "outiov_ix=%d, rem_length=%d, dlength=%d\n"
 574                              "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
 575                              "outcount=%d, outiov_ix=%d\n",
 576                              outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
 577
 578                 outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
 579                 outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
 580                 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 581
 582                 out_length -= dlength;
 583                 iov_offset += dlength;
 584                 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
 585                         iov_offset = 0;
 586                         ++outiov_ix;
 587                 }
 588         }
 589         if (out_length > 0){
 590                 DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
 591                 err = -EINVAL;
 592                 goto error_cleanup;
 593         }
 594         /* Set sync in last descriptor. */
 595         mi.sync = 1;
 596         outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 597
 598         *id = head.next;
 599         return 0;
 600
 601  error_cleanup:
 602         while (head.next) {
 603                 outdesc = head.next->next;
 604                 free_cdesc(head.next);
 605                 head.next = outdesc;
 606         }
 607         return err;
 608 }
 609
 610
 611 static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
 612 {
 613         while (desc_len != 0) {
 614                 struct cryptocop_dma_desc  *cdesc;
 615                 int                        rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
 616                 int                        dlength = (desc_len < rem_length) ? desc_len : rem_length;
 617
 618                 cdesc = alloc_cdesc(alloc_flag);
 619                 if (!cdesc) {
 620                         DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
 621                         return -ENOMEM;
 622                 }
 623                 (*current_out_cdesc)->next = cdesc;
 624                 (*current_out_cdesc) = cdesc;
 625
 626                 cdesc->free_buf = NULL;
 627
 628                 cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
 629                 cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
 630
 631                 assert(desc_len >= dlength);
 632                 desc_len -= dlength;
 633                 *iniov_offset += dlength;
 634                 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
 635                         *iniov_offset = 0;
 636                         ++(*iniov_ix);
 637                         if (*iniov_ix > operation->tfrm_op.incount) {
 638                                 DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
 639                                 return  -EINVAL;
 640                         }
 641                 }
 642                 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
 643         } /* while (desc_len != 0) */
 644         /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
 645         (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
 646
 647         return 0;
 648 }
 649
 650
 651 static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
 652 {
 653         DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
 654         if (tc->tcfg) {
 655                 int                        failed = 0;
 656                 struct cryptocop_dma_desc  *idescs = NULL;
 657                 DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
 658                 if (tc->pad_descs) {
 659                         DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
 660                         while (tc->pad_descs) {
 661                                 DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
 662                                 (*current_out_cdesc)->next = tc->pad_descs;
 663                                 tc->pad_descs = tc->pad_descs->next;
 664                                 (*current_out_cdesc) = (*current_out_cdesc)->next;
 665                         }
 666                 }
 667
 668                 /* Setup and append output descriptors to DMA in list. */
 669                 if (tc->unit_no == src_dma){
 670                         /* mem2mem.  Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
 671                         struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
 672                         unsigned int start_ix = tc->start_ix;
 673                         while (start_ix){
 674                                 unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
 675                                 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
 676                                 if (!(*current_in_cdesc)->next){
 677                                         DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
 678                                         return -ENOMEM;
 679                                 }
 680                                 (*current_in_cdesc) = (*current_in_cdesc)->next;
 681                                 (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
 682                                 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
 683                                 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 684                                 start_ix -= desclen;
 685                         }
 686                         mi.sync = 1;
 687                         (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 688                 }
 689
 690                 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
 691                 if (failed){
 692                         DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
 693                         return failed;
 694                 }
 695                 DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
 696                 while (idescs) {
 697                         DEBUG(printk("append descriptor 0x%p\n", idescs));
 698                         (*current_in_cdesc)->next = idescs;
 699                         idescs = idescs->next;
 700                         (*current_in_cdesc) = (*current_in_cdesc)->next;
 701                 }
 702         }
 703         return 0;
 704 }
 705
 706
 707
 708 static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
 709 {
 710         struct cryptocop_session *sess;
 711         struct cryptocop_transform_ctx *tctx;
 712
 713         struct cryptocop_tfrm_ctx digest_ctx = {
 714                 .previous_src = src_none,
 715                 .current_src = src_none,
 716                 .start_ix = 0,
 717                 .requires_padding = 1,
 718                 .strict_block_length = 0,
 719                 .hash_conf = 0,
 720                 .hash_mode = 0,
 721                 .ciph_conf = 0,
 722                 .cbcmode = 0,
 723                 .decrypt = 0,
 724                 .consumed = 0,
 725                 .produced = 0,
 726                 .pad_descs = NULL,
 727                 .active = 0,
 728                 .done = 0,
 729                 .prev_src = NULL,
 730                 .curr_src = NULL,
 731                 .tcfg = NULL};
 732         struct cryptocop_tfrm_ctx cipher_ctx = {
 733                 .previous_src = src_none,
 734                 .current_src = src_none,
 735                 .start_ix = 0,
 736                 .requires_padding = 0,
 737                 .strict_block_length = 1,
 738                 .hash_conf = 0,
 739                 .hash_mode = 0,
 740                 .ciph_conf = 0,
 741                 .cbcmode = 0,
 742                 .decrypt = 0,
 743                 .consumed = 0,
 744                 .produced = 0,
 745                 .pad_descs = NULL,
 746                 .active = 0,
 747                 .done = 0,
 748                 .prev_src = NULL,
 749                 .curr_src = NULL,
 750                 .tcfg = NULL};
 751         struct cryptocop_tfrm_ctx csum_ctx = {
 752                 .previous_src = src_none,
 753                 .current_src = src_none,
 754                 .start_ix = 0,
 755                 .blocklength = 2,
 756                 .requires_padding = 1,
 757                 .strict_block_length = 0,
 758                 .hash_conf = 0,
 759                 .hash_mode = 0,
 760                 .ciph_conf = 0,
 761                 .cbcmode = 0,
 762                 .decrypt = 0,
 763                 .consumed = 0,
 764                 .produced = 0,
 765                 .pad_descs = NULL,
 766                 .active = 0,
 767                 .done = 0,
 768                 .tcfg = NULL,
 769                 .prev_src = NULL,
 770                 .curr_src = NULL,
 771                 .unit_no = src_csum};
 772         struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
 773
 774         unsigned int indata_ix = 0;
 775
 776         /* iovec accounting. */
 777         int iniov_ix = 0;
 778         int iniov_offset = 0;
 779
 780         /* Operation descriptor cfg traversal pointer. */
 781         struct cryptocop_desc *odsc;
 782
 783         int failed = 0;
 784         /* List heads for allocated descriptors. */
 785         struct cryptocop_dma_desc out_cdesc_head = {0};
 786         struct cryptocop_dma_desc in_cdesc_head = {0};
 787
 788         struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
 789         struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
 790
 791         struct cryptocop_tfrm_ctx *output_tc = NULL;
 792         void                      *iop_alloc_ptr;
 793
 794         assert(operation != NULL);
 795         assert(int_op != NULL);
 796
 797         DEBUG(printk("cryptocop_setup_dma_list: start\n"));
 798         DEBUG(print_cryptocop_operation(operation));
 799
 800         sess = get_session(operation->sid);
 801         if (!sess) {
 802                 DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
 803                 failed = -EINVAL;
 804                 goto error_cleanup;
 805         }
 806         iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
 807         if (!iop_alloc_ptr) {
 808                 DEBUG_API(printk("cryptocop_setup_dma_list:  kmalloc cryptocop_int_operation\n"));
 809                 failed = -ENOMEM;
 810                 goto error_cleanup;
 811         }
 812         (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
 813         DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
 814         (*int_op)->alloc_ptr = iop_alloc_ptr;
 815         DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
 816
 817         (*int_op)->sid = operation->sid;
 818         (*int_op)->cdesc_out = NULL;
 819         (*int_op)->cdesc_in = NULL;
 820         (*int_op)->tdes_mode = cryptocop_3des_ede;
 821         (*int_op)->csum_mode = cryptocop_csum_le;
 822         (*int_op)->ddesc_out = NULL;
 823         (*int_op)->ddesc_in = NULL;
 824
 825         /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
 826         if (!tcfg) {
 827                 DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
 828                 failed = -EINVAL;
 829                 goto error_cleanup;
 830         }
 831         while (tcfg) {
 832                 tctx = get_transform_ctx(sess, tcfg->tid);
 833                 if (!tctx) {
 834                         DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
 835                         failed = -EINVAL;
 836                         goto error_cleanup;
 837                 }
 838                 if (tcfg->inject_ix > operation->tfrm_op.outlen){
 839                         DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
 840                         failed = -EINVAL;
 841                         goto error_cleanup;
 842                 }
 843                 switch (tctx->init.alg){
 844                 case cryptocop_alg_mem2mem:
 845                         if (cipher_ctx.tcfg != NULL){
 846                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
 847                                 failed = -EINVAL;
 848                                 goto error_cleanup;
 849                         }
 850                         /* mem2mem is handled as a NULL cipher. */
 851                         cipher_ctx.cbcmode = 0;
 852                         cipher_ctx.decrypt = 0;
 853                         cipher_ctx.blocklength = 1;
 854                         cipher_ctx.ciph_conf = 0;
 855                         cipher_ctx.unit_no = src_dma;
 856                         cipher_ctx.tcfg = tcfg;
 857                         cipher_ctx.tctx = tctx;
 858                         break;
 859                 case cryptocop_alg_des:
 860                 case cryptocop_alg_3des:
 861                 case cryptocop_alg_aes:
 862                         /* cipher */
 863                         if (cipher_ctx.tcfg != NULL){
 864                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
 865                                 failed = -EINVAL;
 866                                 goto error_cleanup;
 867                         }
 868                         cipher_ctx.tcfg = tcfg;
 869                         cipher_ctx.tctx = tctx;
 870                         if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
 871                                 cipher_ctx.decrypt = 1;
 872                         }
 873                         switch (tctx->init.cipher_mode) {
 874                         case cryptocop_cipher_mode_ecb:
 875                                 cipher_ctx.cbcmode = 0;
 876                                 break;
 877                         case cryptocop_cipher_mode_cbc:
 878                                 cipher_ctx.cbcmode = 1;
 879                                 break;
 880                         default:
 881                                 DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
 882                                 failed = -EINVAL;
 883                                 goto error_cleanup;
 884                         }
 885                         DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
 886                         switch (tctx->init.alg){
 887                         case cryptocop_alg_des:
 888                                 cipher_ctx.ciph_conf = 0;
 889                                 cipher_ctx.unit_no = src_des;
 890                                 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
 891                                 break;
 892                         case cryptocop_alg_3des:
 893                                 cipher_ctx.ciph_conf = 1;
 894                                 cipher_ctx.unit_no = src_des;
 895                                 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
 896                                 break;
 897                         case cryptocop_alg_aes:
 898                                 cipher_ctx.ciph_conf = 2;
 899                                 cipher_ctx.unit_no = src_aes;
 900                                 cipher_ctx.blocklength = AES_BLOCK_LENGTH;
 901                                 break;
 902                         default:
 903                                 panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
 904                         }
 905                         (*int_op)->tdes_mode = tctx->init.tdes_mode;
 906                         break;
 907                 case cryptocop_alg_md5:
 908                 case cryptocop_alg_sha1:
 909                         /* digest */
 910                         if (digest_ctx.tcfg != NULL){
 911                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
 912                                 failed = -EINVAL;
 913                                 goto error_cleanup;
 914                         }
 915                         digest_ctx.tcfg = tcfg;
 916                         digest_ctx.tctx = tctx;
 917                         digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
 918                         switch (tctx->init.alg){
 919                         case cryptocop_alg_md5:
 920                                 digest_ctx.blocklength = MD5_BLOCK_LENGTH;
 921                                 digest_ctx.unit_no = src_md5;
 922                                 digest_ctx.hash_conf = 1; /* 1 => MD-5 */
 923                                 break;
 924                         case cryptocop_alg_sha1:
 925                                 digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
 926                                 digest_ctx.unit_no = src_sha1;
 927                                 digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
 928                                 break;
 929                         default:
 930                                 panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
 931                         }
 932                         break;
 933                 case cryptocop_alg_csum:
 934                         /* digest */
 935                         if (csum_ctx.tcfg != NULL){
 936                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
 937                                 failed = -EINVAL;
 938                                 goto error_cleanup;
 939                         }
 940                         (*int_op)->csum_mode = tctx->init.csum_mode;
 941                         csum_ctx.tcfg = tcfg;
 942                         csum_ctx.tctx = tctx;
 943                         break;
 944                 default:
 945                         /* no algorithm. */
 946                         DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
 947                         failed = -EINVAL;
 948                         goto error_cleanup;
 949                 }
 950                 tcfg = tcfg->next;
 951         }
 952         /* Download key if a cipher is used. */
 953         if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
 954                 struct cryptocop_dma_desc  *key_desc = NULL;
 955
 956                 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
 957                 if (failed) {
 958                         DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
 959                         goto error_cleanup;
 960                 }
 961                 current_out_cdesc->next = key_desc;
 962                 current_out_cdesc = key_desc;
 963                 indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
 964
 965                 /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
 966                 if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
 967                         struct cryptocop_dma_desc  *iv_desc = NULL;
 968
 969                         DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
 970
 971                         failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
 972                         if (failed) {
 973                                 DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
 974                                 goto error_cleanup;
 975                         }
 976                         current_out_cdesc->next = iv_desc;
 977                         current_out_cdesc = iv_desc;
 978                         indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
 979                 }
 980         }
 981
 982         /* Process descriptors. */
 983         odsc = operation->tfrm_op.desc;
 984         while (odsc) {
 985                 struct cryptocop_desc_cfg   *dcfg = odsc->cfg;
 986                 struct strcop_meta_out      meta_out = {0};
 987                 size_t                      desc_len = odsc->length;
 988                 int                         active_count, eop_needed_count;
 989
 990                 output_tc = NULL;
 991
 992                 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
 993
 994                 while (dcfg) {
 995                         struct cryptocop_tfrm_ctx  *tc = NULL;
 996
 997                         DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
 998                         /* Get the local context for the transform and mark it as the output unit if it produces output. */
 999                         if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
1000                                 tc = &digest_ctx;
1001                         } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1002                                 tc = &cipher_ctx;
1003                         } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1004                                 tc = &csum_ctx;
1005                         }
1006                         if (!tc) {
1007                                 DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1008                                 failed = -EINVAL;
1009                                 goto error_cleanup;
1010                         }
1011                         if (tc->done) {
1012                                 DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1013                                 failed = -EINVAL;
1014                                 goto error_cleanup;
1015                         }
1016                         if (!tc->active) {
1017                                 tc->start_ix = indata_ix;
1018                                 tc->active = 1;
1019                         }
1020
1021                         tc->previous_src = tc->current_src;
1022                         tc->prev_src = tc->curr_src;
1023                         /* Map source unit id to DMA source config. */
1024                         switch (dcfg->src){
1025                         case cryptocop_source_dma:
1026                                 tc->current_src = src_dma;
1027                                 break;
1028                         case cryptocop_source_des:
1029                                 tc->current_src = src_des;
1030                                 break;
1031                         case cryptocop_source_3des:
1032                                 tc->current_src = src_des;
1033                                 break;
1034                         case cryptocop_source_aes:
1035                                 tc->current_src = src_aes;
1036                                 break;
1037                         case cryptocop_source_md5:
1038                         case cryptocop_source_sha1:
1039                         case cryptocop_source_csum:
1040                         case cryptocop_source_none:
1041                         default:
1042                                 /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1043                                  */
1044                                 DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1045                                 failed = -EINVAL;
1046                                 goto error_cleanup;
1047                         }
1048                         if (tc->current_src != src_dma) {
1049                                 /* Find the unit we are sourcing from. */
1050                                 if (digest_ctx.unit_no == tc->current_src){
1051                                         tc->curr_src = &digest_ctx;
1052                                 } else if (cipher_ctx.unit_no == tc->current_src){
1053                                         tc->curr_src = &cipher_ctx;
1054                                 } else if (csum_ctx.unit_no == tc->current_src){
1055                                         tc->curr_src = &csum_ctx;
1056                                 }
1057                                 if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1058                                         DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1059                                         failed = -EINVAL;
1060                                         goto error_cleanup;
1061                                 }
1062                         } else {
1063                                 tc->curr_src = NULL;
1064                         }
1065
1066                         /* Detect source switch. */
1067                         DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1068                         if (tc->active && (tc->current_src != tc->previous_src)) {
1069                                 /* Only allow source switch when both the old source unit and the new one have
1070                                  * no pending data to process (i.e. the consumed length must be a multiple of the
1071                                  * transform blocklength). */
1072                                 /* Note: if the src == NULL we are actually sourcing from DMA out. */
1073                                 if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1074                                     ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1075                                 {
1076                                         DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1077                                         failed = -EINVAL;
1078                                         goto error_cleanup;
1079                                 }
1080                         }
1081                         /* Detect unit deactivation. */
1082                         if (dcfg->last) {
1083                                 /* Length check of this is handled below. */
1084                                 tc->done = 1;
1085                         }
1086                         dcfg = dcfg->next;
1087                 } /* while (dcfg) */
1088                 DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1089
1090                 if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1091                         DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1092                         failed = -EINVAL;
1093                         goto error_cleanup;
1094                 }
1095                 if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1096                         DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1097                         failed = -EINVAL;
1098                         goto error_cleanup;
1099                 }
1100                 if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1101                         DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1102                         failed = -EINVAL;
1103                         goto error_cleanup;
1104                 }
1105
1106                 /* Update consumed and produced lengths.
1107
1108                    The consumed length accounting here is actually cheating.  If a unit source from DMA (or any
1109                    other unit that process data in blocks of one octet) it is correct, but if it source from a
1110                    block processing unit, i.e. a cipher, it will be temporarily incorrect at some times.  However
1111                    since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1112                    unit has processed an exact multiple of its block length the end result will be correct.
1113                    Beware that if the source change restriction change this code will need to be (much) reworked.
1114                 */
1115                 DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1116
1117                 if (csum_ctx.active) {
1118                         csum_ctx.consumed += desc_len;
1119                         if (csum_ctx.done) {
1120                                 csum_ctx.produced = 2;
1121                         }
1122                         DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1123                 }
1124                 if (digest_ctx.active) {
1125                         digest_ctx.consumed += desc_len;
1126                         if (digest_ctx.done) {
1127                                 if (digest_ctx.unit_no == src_md5) {
1128                                         digest_ctx.produced = MD5_STATE_LENGTH;
1129                                 } else {
1130                                         digest_ctx.produced = SHA1_STATE_LENGTH;
1131                                 }
1132                         }
1133                         DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1134                 }
1135                 if (cipher_ctx.active) {
1136                         /* Ciphers are allowed only to source from DMA out.  That is filtered above. */
1137                         assert(cipher_ctx.current_src == src_dma);
1138                         cipher_ctx.consumed += desc_len;
1139                         cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1140                         if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1141                                 cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1142                         }
1143                         DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1144                 }
1145
1146                 /* Setup the DMA out descriptors. */
1147                 /* Configure the metadata. */
1148                 active_count = 0;
1149                 eop_needed_count = 0;
1150                 if (cipher_ctx.active) {
1151                         ++active_count;
1152                         if (cipher_ctx.unit_no == src_dma){
1153                                 /* mem2mem */
1154                                 meta_out.ciphsel = src_none;
1155                         } else {
1156                                 meta_out.ciphsel = cipher_ctx.current_src;
1157                         }
1158                         meta_out.ciphconf = cipher_ctx.ciph_conf;
1159                         meta_out.cbcmode = cipher_ctx.cbcmode;
1160                         meta_out.decrypt = cipher_ctx.decrypt;
1161                         DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1162                         if (cipher_ctx.done) ++eop_needed_count;
1163                 } else {
1164                         meta_out.ciphsel = src_none;
1165                 }
1166
1167                 if (digest_ctx.active) {
1168                         ++active_count;
1169                         meta_out.hashsel = digest_ctx.current_src;
1170                         meta_out.hashconf = digest_ctx.hash_conf;
1171                         meta_out.hashmode = 0; /* Explicit mode is not used here. */
1172                         DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1173                         if (digest_ctx.done) {
1174                                 assert(digest_ctx.pad_descs == NULL);
1175                                 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1176                                 if (failed) {
1177                                         DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1178                                         goto error_cleanup;
1179                                 }
1180                         }
1181                 } else {
1182                         meta_out.hashsel = src_none;
1183                 }
1184
1185                 if (csum_ctx.active) {
1186                         ++active_count;
1187                         meta_out.csumsel = csum_ctx.current_src;
1188                         if (csum_ctx.done) {
1189                                 assert(csum_ctx.pad_descs == NULL);
1190                                 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1191                                 if (failed) {
1192                                         DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1193                                         goto error_cleanup;
1194                                 }
1195                         }
1196                 } else {
1197                         meta_out.csumsel = src_none;
1198                 }
1199                 DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1200                 /* Setup DMA out descriptors for the indata. */
1201                 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1202                 if (failed) {
1203                         DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1204                         goto error_cleanup;
1205                 }
1206                 /* Setup out EOP.  If there are active units that are not done here they cannot get an EOP
1207                  * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1208                  * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1209                  */
1210                 assert(active_count >= eop_needed_count);
1211                 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1212                 if (eop_needed_count) {
1213                         /* This means that the bulk operation (cipeher/m2m) is terminated. */
1214                         if (active_count > 1) {
1215                                 /* Use zero length EOP descriptor. */
1216                                 struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1217                                 struct strcop_meta_out    ed_mo = {0};
1218                                 if (!ed) {
1219                                         DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1220                                         failed = -ENOMEM;
1221                                         goto error_cleanup;
1222                                 }
1223
1224                                 assert(cipher_ctx.active && cipher_ctx.done);
1225
1226                                 if (cipher_ctx.unit_no == src_dma){
1227                                         /* mem2mem */
1228                                         ed_mo.ciphsel = src_none;
1229                                 } else {
1230                                         ed_mo.ciphsel = cipher_ctx.current_src;
1231                                 }
1232                                 ed_mo.ciphconf = cipher_ctx.ciph_conf;
1233                                 ed_mo.cbcmode = cipher_ctx.cbcmode;
1234                                 ed_mo.decrypt = cipher_ctx.decrypt;
1235
1236                                 ed->free_buf = NULL;
1237                                 ed->dma_descr->wait = 1;
1238                                 ed->dma_descr->out_eop = 1;
1239
1240                                 ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1241                                 ed->dma_descr->after = ed->dma_descr->buf;
1242                                 ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1243                                 current_out_cdesc->next = ed;
1244                                 current_out_cdesc = ed;
1245                         } else {
1246                                 /* Set EOP in the current out descriptor since the only active module is
1247                                  * the one needing the EOP. */
1248
1249                                 current_out_cdesc->dma_descr->out_eop = 1;
1250                         }
1251                 }
1252
1253                 if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1254                 if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1255                 if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1256                 indata_ix += odsc->length;
1257                 odsc = odsc->next;
1258         } /* while (odsc) */ /* Process descriptors. */
1259         DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1260         if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1261                 DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1262                 failed = -EINVAL;
1263                 goto error_cleanup;
1264         }
1265         if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1266                 DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1267                 failed = -EINVAL;
1268                 goto error_cleanup;
1269         }
1270         if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1271                 DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1272                 failed = -EINVAL;
1273                 goto error_cleanup;
1274         }
1275
1276         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1277         if (failed){
1278                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1279                 goto error_cleanup;
1280         }
1281         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1282         if (failed){
1283                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1284                 goto error_cleanup;
1285         }
1286         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1287         if (failed){
1288                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1289                 goto error_cleanup;
1290         }
1291
1292         DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1293         (*int_op)->cdesc_out = out_cdesc_head.next;
1294         (*int_op)->cdesc_in = in_cdesc_head.next;
1295         DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1296
1297         setup_descr_chain(out_cdesc_head.next);
1298         setup_descr_chain(in_cdesc_head.next);
1299
1300         /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1301          * last DMA out descriptor for EOL.
1302          */
1303         current_in_cdesc->dma_descr->intr = 1;
1304         current_in_cdesc->dma_descr->eol = 1;
1305         current_out_cdesc->dma_descr->eol = 1;
1306
1307         /* Setup DMA contexts. */
1308         (*int_op)->ctx_out.next = NULL;
1309         (*int_op)->ctx_out.eol = 1;
1310         (*int_op)->ctx_out.intr = 0;
1311         (*int_op)->ctx_out.store_mode = 0;
1312         (*int_op)->ctx_out.en = 0;
1313         (*int_op)->ctx_out.dis = 0;
1314         (*int_op)->ctx_out.md0 = 0;
1315         (*int_op)->ctx_out.md1 = 0;
1316         (*int_op)->ctx_out.md2 = 0;
1317         (*int_op)->ctx_out.md3 = 0;
1318         (*int_op)->ctx_out.md4 = 0;
1319         (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1320         (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1321
1322         (*int_op)->ctx_in.next = NULL;
1323         (*int_op)->ctx_in.eol = 1;
1324         (*int_op)->ctx_in.intr = 0;
1325         (*int_op)->ctx_in.store_mode = 0;
1326         (*int_op)->ctx_in.en = 0;
1327         (*int_op)->ctx_in.dis = 0;
1328         (*int_op)->ctx_in.md0 = 0;
1329         (*int_op)->ctx_in.md1 = 0;
1330         (*int_op)->ctx_in.md2 = 0;
1331         (*int_op)->ctx_in.md3 = 0;
1332         (*int_op)->ctx_in.md4 = 0;
1333
1334         (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1335         (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1336
1337         DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1338         return 0;
1339
1340 error_cleanup:
1341         {
1342                 /* Free all allocated resources. */
1343                 struct cryptocop_dma_desc *tmp_cdesc;
1344                 while (digest_ctx.pad_descs){
1345                         tmp_cdesc = digest_ctx.pad_descs->next;
1346                         free_cdesc(digest_ctx.pad_descs);
1347                         digest_ctx.pad_descs = tmp_cdesc;
1348                 }
1349                 while (csum_ctx.pad_descs){
1350                         tmp_cdesc = csum_ctx.pad_descs->next;
1351                         free_cdesc(csum_ctx.pad_descs);
1352                         csum_ctx.pad_descs = tmp_cdesc;
1353                 }
1354                 assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1355
1356                 if (*int_op != NULL) delete_internal_operation(*int_op);
1357         }
1358         DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1359         return failed;
1360 }
1361
1362
1363 static void delete_internal_operation(struct cryptocop_int_operation *iop)
1364 {
1365         void                      *ptr = iop->alloc_ptr;
1366         struct cryptocop_dma_desc *cd = iop->cdesc_out;
1367         struct cryptocop_dma_desc *next;
1368
1369         DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1370
1371         while (cd) {
1372                 next = cd->next;
1373                 free_cdesc(cd);
1374                 cd = next;
1375         }
1376         cd = iop->cdesc_in;
1377         while (cd) {
1378                 next = cd->next;
1379                 free_cdesc(cd);
1380                 cd = next;
1381         }
1382         kfree(ptr);
1383 }
1384
1385 #define MD5_MIN_PAD_LENGTH (9)
1386 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1387
1388 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1389 {
1390         size_t                  padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1391         unsigned char           *p;
1392         int                     i;
1393         unsigned long long int  bit_length = hashed_length << 3;
1394
1395         if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1396
1397         p = kzalloc(padlen, alloc_flag);
1398         if (!p) return -ENOMEM;
1399
1400         *p = 0x80;
1401
1402         DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1403
1404         i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1405         while (bit_length != 0){
1406                 p[i++] = bit_length % 0x100;
1407                 bit_length >>= 8;
1408         }
1409
1410         *pad = (char*)p;
1411         *pad_length = padlen;
1412
1413         return 0;
1414 }
1415
1416 #define SHA1_MIN_PAD_LENGTH (9)
1417 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1418
1419 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1420 {
1421         size_t                  padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1422         unsigned char           *p;
1423         int                     i;
1424         unsigned long long int  bit_length = hashed_length << 3;
1425
1426         if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1427
1428         p = kzalloc(padlen, alloc_flag);
1429         if (!p) return -ENOMEM;
1430
1431         *p = 0x80;
1432
1433         DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1434
1435         i = padlen - 1;
1436         while (bit_length != 0){
1437                 p[i--] = bit_length % 0x100;
1438                 bit_length >>= 8;
1439         }
1440
1441         *pad = (char*)p;
1442         *pad_length = padlen;
1443
1444         return 0;
1445 }
1446
1447
1448 static int transform_ok(struct cryptocop_transform_init *tinit)
1449 {
1450         switch (tinit->alg){
1451         case cryptocop_alg_csum:
1452                 switch (tinit->csum_mode){
1453                 case cryptocop_csum_le:
1454                 case cryptocop_csum_be:
1455                         break;
1456                 default:
1457                         DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1458                         return -EINVAL;
1459                 }
1460         case cryptocop_alg_mem2mem:
1461         case cryptocop_alg_md5:
1462         case cryptocop_alg_sha1:
1463                 if (tinit->keylen != 0) {
1464                         DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1465                         return -EINVAL; /* This check is a bit strict. */
1466                 }
1467                 break;
1468         case cryptocop_alg_des:
1469                 if (tinit->keylen != 64) {
1470                         DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1471                         return -EINVAL;
1472                 }
1473                 break;
1474         case cryptocop_alg_3des:
1475                 if (tinit->keylen != 192) {
1476                         DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1477                         return -EINVAL;
1478                 }
1479                 break;
1480         case cryptocop_alg_aes:
1481                 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1482                         DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1483                         return -EINVAL;
1484                 }
1485                 break;
1486         case cryptocop_no_alg:
1487         default:
1488                 DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1489                 return -EINVAL;
1490         }
1491
1492         switch (tinit->alg){
1493         case cryptocop_alg_des:
1494         case cryptocop_alg_3des:
1495         case cryptocop_alg_aes:
1496                 if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1497         default:
1498                  break;
1499         }
1500         return 0;
1501 }
1502
1503
1504 int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1505 {
1506         struct cryptocop_session         *sess;
1507         struct cryptocop_transform_init  *tfrm_in = tinit;
1508         struct cryptocop_transform_init  *tmp_in;
1509         int                              no_tfrms = 0;
1510         int                              i;
1511         unsigned long int                flags;
1512
1513         init_stream_coprocessor(); /* For safety if we are called early */
1514
1515         while (tfrm_in){
1516                 int err;
1517                 ++no_tfrms;
1518                 if ((err = transform_ok(tfrm_in))) {
1519                         DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1520                         return err;
1521                 }
1522                 tfrm_in = tfrm_in->next;
1523         }
1524         if (0 == no_tfrms) {
1525                 DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1526                 return -EINVAL;
1527         }
1528
1529         sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1530         if (!sess){
1531                 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1532                 return -ENOMEM;
1533         }
1534
1535         sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1536         if (!sess->tfrm_ctx) {
1537                 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1538                 kfree(sess);
1539                 return -ENOMEM;
1540         }
1541
1542         tfrm_in = tinit;
1543         for (i = 0; i < no_tfrms; i++){
1544                 tmp_in = tfrm_in->next;
1545                 while (tmp_in){
1546                         if (tmp_in->tid == tfrm_in->tid) {
1547                                 DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1548                                 kfree(sess->tfrm_ctx);
1549                                 kfree(sess);
1550                                 return -EINVAL;
1551                         }
1552                         tmp_in = tmp_in->next;
1553                 }
1554                 memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1555                 sess->tfrm_ctx[i].dec_key_set = 0;
1556                 sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1557
1558                 tfrm_in = tfrm_in->next;
1559         }
1560         sess->tfrm_ctx[i-1].next = NULL;
1561
1562         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1563         sess->sid = next_sid;
1564         next_sid++;
1565         /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1566          *      OTOH 2^64 is a really large number of session. */
1567         if (next_sid == 0) next_sid = 1;
1568
1569         /* Prepend to session list. */
1570         sess->next = cryptocop_sessions;
1571         cryptocop_sessions = sess;
1572         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1573         *sid = sess->sid;
1574         return 0;
1575 }
1576
1577
1578 int cryptocop_free_session(cryptocop_session_id sid)
1579 {
1580         struct cryptocop_transform_ctx    *tc;
1581         struct cryptocop_session          *sess = NULL;
1582         struct cryptocop_session          *psess = NULL;
1583         unsigned long int                 flags;
1584         int                               i;
1585         LIST_HEAD(remove_list);
1586         struct list_head                  *node, *tmp;
1587         struct cryptocop_prio_job         *pj;
1588
1589         DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1590
1591         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1592         sess = cryptocop_sessions;
1593         while (sess && sess->sid != sid){
1594                 psess = sess;
1595                 sess = sess->next;
1596         }
1597         if (sess){
1598                 if (psess){
1599                         psess->next = sess->next;
1600                 } else {
1601                         cryptocop_sessions = sess->next;
1602                 }
1603         }
1604         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1605
1606         if (!sess) return -EINVAL;
1607
1608         /* Remove queued jobs. */
1609         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1610
1611         for (i = 0; i < cryptocop_prio_no_prios; i++){
1612                 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1613                         list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1614                                 pj = list_entry(node, struct cryptocop_prio_job, node);
1615                                 if (pj->oper->sid == sid) {
1616                                         list_move_tail(node, &remove_list);
1617                                 }
1618                         }
1619                 }
1620         }
1621         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1622
1623         list_for_each_safe(node, tmp, &remove_list) {
1624                 list_del(node);
1625                 pj = list_entry(node, struct cryptocop_prio_job, node);
1626                 pj->oper->operation_status = -EAGAIN;  /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1627                 DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1628                 pj->oper->cb(pj->oper, pj->oper->cb_data);
1629                 delete_internal_operation(pj->iop);
1630                 kfree(pj);
1631         }
1632
1633         tc = sess->tfrm_ctx;
1634         /* Erase keying data. */
1635         while (tc){
1636                 DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1637                 memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1638                 memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1639                 tc = tc->next;
1640         }
1641         kfree(sess->tfrm_ctx);
1642         kfree(sess);
1643
1644         return 0;
1645 }
1646
1647 static struct cryptocop_session *get_session(cryptocop_session_id sid)
1648 {
1649         struct cryptocop_session    *sess;
1650         unsigned long int           flags;
1651
1652         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1653         sess = cryptocop_sessions;
1654         while (sess && (sess->sid != sid)){
1655                 sess = sess->next;
1656         }
1657         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1658
1659         return sess;
1660 }
1661
1662 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1663 {
1664         struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1665
1666         DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1667         assert(sess != NULL);
1668         while (tc && tc->init.tid != tid){
1669                 DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1670                 tc = tc->next;
1671         }
1672         DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1673         return tc;
1674 }
1675
1676
1677
1678 /* The AES s-transform matrix (s-box). */
1679 static const u8 aes_sbox[256] = {
1680         99,  124, 119, 123, 242, 107, 111, 197, 48,  1,   103, 43,  254, 215, 171, 118,
1681         202, 130, 201, 125, 250, 89,  71,  240, 173, 212, 162, 175, 156, 164, 114, 192,
1682         183, 253, 147, 38,  54,  63,  247, 204, 52,  165, 229, 241, 113, 216, 49,  21,
1683         4,   199, 35,  195, 24,  150, 5,   154, 7,   18,  128, 226, 235, 39,  178, 117,
1684         9,   131, 44,  26,  27,  110, 90,  160, 82,  59,  214, 179, 41,  227, 47,  132,
1685         83,  209, 0,   237, 32,  252, 177, 91,  106, 203, 190, 57,  74,  76,  88,  207,
1686         208, 239, 170, 251, 67,  77,  51,  133, 69,  249, 2,   127, 80,  60,  159, 168,
1687         81,  163, 64,  143, 146, 157, 56,  245, 188, 182, 218, 33,  16,  255, 243, 210,
1688         205, 12,  19,  236, 95,  151, 68,  23,  196, 167, 126, 61,  100, 93,  25,  115,
1689         96,  129, 79,  220, 34,  42,  144, 136, 70,  238, 184, 20,  222, 94,  11,  219,
1690         224, 50,  58,  10,  73,  6,   36,  92,  194, 211, 172, 98,  145, 149, 228, 121,
1691         231, 200, 55,  109, 141, 213, 78,  169, 108, 86,  244, 234, 101, 122, 174, 8,
1692         186, 120, 37,  46,  28,  166, 180, 198, 232, 221, 116, 31,  75,  189, 139, 138,
1693         112, 62,  181, 102, 72,  3,   246, 14,  97,  53,  87,  185, 134, 193, 29,  158,
1694         225, 248, 152, 17,  105, 217, 142, 148, 155, 30,  135, 233, 206, 85,  40,  223,
1695         140, 161, 137, 13,  191, 230, 66,  104, 65,  153, 45,  15,  176, 84,  187, 22
1696 };
1697
1698 /* AES has a 32 bit word round constants for each round in the
1699  * key schedule.  round_constant[i] is really Rcon[i+1] in FIPS187.
1700  */
1701 static u32 round_constant[11] = {
1702         0x01000000, 0x02000000, 0x04000000, 0x08000000,
1703         0x10000000, 0x20000000, 0x40000000, 0x80000000,
1704         0x1B000000, 0x36000000, 0x6C000000
1705 };
1706
1707 /* Apply the s-box to each of the four occtets in w. */
1708 static u32 aes_ks_subword(const u32 w)
1709 {
1710         u8 bytes[4];
1711
1712         *(u32*)(&bytes[0]) = w;
1713         bytes[0] = aes_sbox[bytes[0]];
1714         bytes[1] = aes_sbox[bytes[1]];
1715         bytes[2] = aes_sbox[bytes[2]];
1716         bytes[3] = aes_sbox[bytes[3]];
1717         return *(u32*)(&bytes[0]);
1718 }
1719
1720 /* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1721  * (Note that AES words are 32 bit long)
1722  *
1723  * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1724  * word temp
1725  * i = 0
1726  * while (i < Nk) {
1727  *   w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1728  *   i = i + 1
1729  * }
1730  * i = Nk
1731  *
1732  * while (i < (Nb * (Nr + 1))) {
1733  *   temp = w[i - 1]
1734  *   if ((i mod Nk) == 0) {
1735  *     temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1736  *   }
1737  *   else if ((Nk > 6) && ((i mod Nk) == 4)) {
1738  *     temp = SubWord(temp)
1739  *   }
1740  *   w[i] = w[i - Nk] xor temp
1741  * }
1742  * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1743  * SubWord(t) applies the AES s-box individually to each octet
1744  * in a 32 bit word.
1745  *
1746  * For AES Nk can have the values 4, 6, and 8 (corresponding to
1747  * values for Nr of 10, 12, and 14).  Nb is always 4.
1748  *
1749  * To construct w[i], w[i - 1] and w[i - Nk] must be
1750  * available.  Consequently we must keep a state of the last Nk words
1751  * to be able to create the last round keys.
1752  */
1753 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength)
1754 {
1755         u32 temp;
1756         u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1757         u8  w_last_ix;
1758         int i;
1759         u8  nr, nk;
1760
1761         switch (keylength){
1762         case 128:
1763                 nk = 4;
1764                 nr = 10;
1765                 break;
1766         case 192:
1767                 nk = 6;
1768                 nr = 12;
1769                 break;
1770         case 256:
1771                 nk = 8;
1772                 nr = 14;
1773                 break;
1774         default:
1775                 panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1776         };
1777
1778         /* Need to do host byte order correction here since key is byte oriented and the
1779          * kx algorithm is word (u32) oriented. */
1780         for (i = 0; i < nk; i+=1) {
1781                 w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1782         }
1783
1784         i = (int)nk;
1785         w_last_ix = i - 1;
1786         while (i < (4 * (nr + 2))) {
1787                 temp = w_ring[w_last_ix];
1788                 if (!(i % nk)) {
1789                         /* RotWord(temp) */
1790                         temp = (temp << 8) | (temp >> 24);
1791                         temp = aes_ks_subword(temp);
1792                         temp ^= round_constant[i/nk - 1];
1793                 } else if ((nk > 6) && ((i % nk) == 4)) {
1794                         temp = aes_ks_subword(temp);
1795                 }
1796                 w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1797                 temp ^= w_ring[w_last_ix];
1798                 w_ring[w_last_ix] = temp;
1799
1800                 /* We need the round keys for round Nr+1 and Nr+2 (round key
1801                  * Nr+2 is the round key beyond the last one used when
1802                  * encrypting).  Rounds are numbered starting from 0, Nr=10
1803                  * implies 11 rounds are used in encryption/decryption.
1804                  */
1805                 if (i >= (4 * nr)) {
1806                         /* Need to do host byte order correction here, the key
1807                          * is byte oriented. */
1808                         *(u32*)dec_key = cpu_to_be32(temp);
1809                         dec_key += 4;
1810                 }
1811                 ++i;
1812         }
1813 }
1814
1815
1816 /**** Job/operation management. ****/
1817
1818 int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1819 {
1820         return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1821 }
1822
1823 int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1824 {
1825         return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1826 }
1827
1828 int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1829 {
1830         return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1831 }
1832
1833 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1834 {
1835         int                           ret;
1836         struct cryptocop_prio_job     *pj = NULL;
1837         unsigned long int             flags;
1838
1839         DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1840
1841         if (!operation || !operation->cb){
1842                 DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1843                 return -EINVAL;
1844         }
1845
1846         if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1847                 DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1848                 return ret;
1849         }
1850         assert(pj != NULL);
1851
1852         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1853         list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1854         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1855
1856         /* Make sure a job is running */
1857         cryptocop_start_job();
1858         return 0;
1859 }
1860
1861 static void cryptocop_do_tasklet(unsigned long unused);
1862 DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1863
1864 static void cryptocop_do_tasklet(unsigned long unused)
1865 {
1866         struct list_head             *node;
1867         struct cryptocop_prio_job    *pj = NULL;
1868         unsigned long                flags;
1869
1870         DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1871
1872         do {
1873                 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1874                 if (!list_empty(&cryptocop_completed_jobs)){
1875                         node = cryptocop_completed_jobs.next;
1876                         list_del(node);
1877                         pj = list_entry(node, struct cryptocop_prio_job, node);
1878                 } else {
1879                         pj = NULL;
1880                 }
1881                 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1882                 if (pj) {
1883                         assert(pj->oper != NULL);
1884
1885                         /* Notify consumer of operation completeness. */
1886                         DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1887
1888                         pj->oper->operation_status = 0; /* Job is completed. */
1889                         pj->oper->cb(pj->oper, pj->oper->cb_data);
1890                         delete_internal_operation(pj->iop);
1891                         kfree(pj);
1892                 }
1893         } while (pj != NULL);
1894
1895         DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1896 }
1897
1898 static irqreturn_t
1899 dma_done_interrupt(int irq, void *dev_id)
1900 {
1901         struct cryptocop_prio_job *done_job;
1902         reg_dma_rw_ack_intr ack_intr = {
1903                 .data = 1,
1904         };
1905
1906         REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1907
1908         DEBUG(printk("cryptocop DMA done\n"));
1909
1910         spin_lock(&running_job_lock);
1911         if (cryptocop_running_job == NULL){
1912                 printk("stream co-processor got interrupt when not busy\n");
1913                 spin_unlock(&running_job_lock);
1914                 return IRQ_HANDLED;
1915         }
1916         done_job = cryptocop_running_job;
1917         cryptocop_running_job = NULL;
1918         spin_unlock(&running_job_lock);
1919
1920         /* Start processing a job. */
1921         if (!spin_trylock(&cryptocop_process_lock)){
1922                 DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1923         } else {
1924                 cryptocop_start_job();
1925                 spin_unlock(&cryptocop_process_lock);
1926         }
1927
1928         done_job->oper->operation_status = 0; /* Job is completed. */
1929         if (done_job->oper->fast_callback){
1930                 /* This operation wants callback from interrupt. */
1931                 done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1932                 delete_internal_operation(done_job->iop);
1933                 kfree(done_job);
1934         } else {
1935                 spin_lock(&cryptocop_completed_jobs_lock);
1936                 list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1937                 spin_unlock(&cryptocop_completed_jobs_lock);
1938                 tasklet_schedule(&cryptocop_tasklet);
1939         }
1940
1941         DEBUG(printk("cryptocop leave irq handler\n"));
1942         return IRQ_HANDLED;
1943 }
1944
1945
1946 /* Setup interrupts and DMA channels. */
1947 static int init_cryptocop(void)
1948 {
1949         unsigned long          flags;
1950         reg_dma_rw_cfg         dma_cfg = {.en = 1};
1951         reg_dma_rw_intr_mask   intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1952         reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
1953         reg_strcop_rw_cfg      strcop_cfg = {
1954                 .ipend = regk_strcop_little,
1955                 .td1 = regk_strcop_e,
1956                 .td2 = regk_strcop_d,
1957                 .td3 = regk_strcop_e,
1958                 .ignore_sync = 0,
1959                 .en = 1
1960         };
1961
1962         if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1963                         "stream co-processor DMA", NULL))
1964                 panic("request_irq stream co-processor irq dma9");
1965
1966         (void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1967                 0, dma_strp);
1968         (void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1969                 0, dma_strp);
1970
1971         local_irq_save(flags);
1972
1973         /* Reset and enable the cryptocop. */
1974         strcop_cfg.en = 0;
1975         REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1976         strcop_cfg.en = 1;
1977         REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1978
1979         /* Enable DMAs. */
1980         REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1981         REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
1982
1983         /* Set up wordsize = 4 for DMAs. */
1984         DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1985         DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
1986
1987         /* Enable interrupts. */
1988         REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
1989
1990         /* Clear intr ack. */
1991         REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1992
1993         local_irq_restore(flags);
1994
1995         return 0;
1996 }
1997
1998 /* Free used cryptocop hw resources (interrupt and DMA channels). */
1999 static void release_cryptocop(void)
2000 {
2001         unsigned long          flags;
2002         reg_dma_rw_cfg         dma_cfg = {.en = 0};
2003         reg_dma_rw_intr_mask   intr_mask_in = {0};
2004         reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
2005
2006         local_irq_save(flags);
2007
2008         /* Clear intr ack. */
2009         REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
2010
2011         /* Disable DMAs. */
2012         REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2013         REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
2014
2015         /* Disable interrupts. */
2016         REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
2017
2018         local_irq_restore(flags);
2019
2020         free_irq(DMA_IRQ, NULL);
2021
2022         (void)crisv32_free_dma(OUT_DMA);
2023         (void)crisv32_free_dma(IN_DMA);
2024 }
2025
2026
2027 /* Init job queue. */
2028 static int cryptocop_job_queue_init(void)
2029 {
2030         int i;
2031
2032         INIT_LIST_HEAD(&cryptocop_completed_jobs);
2033
2034         for (i = 0; i < cryptocop_prio_no_prios; i++){
2035                 cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2036                 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2037         }
2038         return 0;
2039 }
2040
2041
2042 static void cryptocop_job_queue_close(void)
2043 {
2044         struct list_head               *node, *tmp;
2045         struct cryptocop_prio_job      *pj = NULL;
2046         unsigned long int              process_flags, flags;
2047         int                            i;
2048
2049         /* FIXME: This is as yet untested code. */
2050
2051         /* Stop strcop from getting an operation to process while we are closing the
2052            module. */
2053         spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2054
2055         /* Empty the job queue. */
2056         for (i = 0; i < cryptocop_prio_no_prios; i++){
2057                 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2058                         list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2059                                 pj = list_entry(node, struct cryptocop_prio_job, node);
2060                                 list_del(node);
2061
2062                                 /* Call callback to notify consumer of job removal. */
2063                                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2064                                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2065                                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2066
2067                                 delete_internal_operation(pj->iop);
2068                                 kfree(pj);
2069                         }
2070                 }
2071         }
2072         spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2073
2074         /* Remove the running job, if any. */
2075         spin_lock_irqsave(&running_job_lock, flags);
2076         if (cryptocop_running_job){
2077                 reg_strcop_rw_cfg rw_cfg;
2078                 reg_dma_rw_cfg    dma_out_cfg, dma_in_cfg;
2079
2080                 /* Stop DMA. */
2081                 dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
2082                 dma_out_cfg.en = regk_dma_no;
2083                 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
2084
2085                 dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
2086                 dma_in_cfg.en = regk_dma_no;
2087                 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
2088
2089                 /* Disble the cryptocop. */
2090                 rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2091                 rw_cfg.en = 0;
2092                 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2093
2094                 pj = cryptocop_running_job;
2095                 cryptocop_running_job = NULL;
2096
2097                 /* Call callback to notify consumer of job removal. */
2098                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2099                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2100                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2101
2102                 delete_internal_operation(pj->iop);
2103                 kfree(pj);
2104         }
2105         spin_unlock_irqrestore(&running_job_lock, flags);
2106
2107         /* Remove completed jobs, if any. */
2108         spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2109
2110         list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2111                 pj = list_entry(node, struct cryptocop_prio_job, node);
2112                 list_del(node);
2113                 /* Call callback to notify consumer of job removal. */
2114                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2115                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2116                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2117
2118                 delete_internal_operation(pj->iop);
2119                 kfree(pj);
2120         }
2121         spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2122 }
2123
2124
2125 static void cryptocop_start_job(void)
2126 {
2127         int                          i;
2128         struct cryptocop_prio_job    *pj;
2129         unsigned long int            flags;
2130         unsigned long int            running_job_flags;
2131         reg_strcop_rw_cfg            rw_cfg = {.en = 1, .ignore_sync = 0};
2132
2133         DEBUG(printk("cryptocop_start_job: entering\n"));
2134
2135         spin_lock_irqsave(&running_job_lock, running_job_flags);
2136         if (cryptocop_running_job != NULL){
2137                 /* Already running. */
2138                 DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2139                 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2140                 return;
2141         }
2142         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2143
2144         /* Check the queues in priority order. */
2145         for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2146         if (i == cryptocop_prio_no_prios) {
2147                 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2148                 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2149                 DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2150                 return; /* No jobs to run */
2151         }
2152         DEBUG(printk("starting job for prio %d\n", i));
2153
2154         /* TODO: Do not starve lower priority jobs.  Let in a lower
2155          * prio job for every N-th processed higher prio job or some
2156          * other scheduling policy.  This could reasonably be
2157          * tweakable since the optimal balance would depend on the
2158          * type of load on the system. */
2159
2160         /* Pull the DMA lists from the job and start the DMA client. */
2161         pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2162         list_del(&pj->node);
2163         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2164         cryptocop_running_job = pj;
2165
2166         /* Set config register (3DES and CSUM modes). */
2167         switch (pj->iop->tdes_mode){
2168         case cryptocop_3des_eee:
2169                 rw_cfg.td1 = regk_strcop_e;
2170                 rw_cfg.td2 = regk_strcop_e;
2171                 rw_cfg.td3 = regk_strcop_e;
2172                 break;
2173         case cryptocop_3des_eed:
2174                 rw_cfg.td1 = regk_strcop_e;
2175                 rw_cfg.td2 = regk_strcop_e;
2176                 rw_cfg.td3 = regk_strcop_d;
2177                 break;
2178         case cryptocop_3des_ede:
2179                 rw_cfg.td1 = regk_strcop_e;
2180                 rw_cfg.td2 = regk_strcop_d;
2181                 rw_cfg.td3 = regk_strcop_e;
2182                 break;
2183         case cryptocop_3des_edd:
2184                 rw_cfg.td1 = regk_strcop_e;
2185                 rw_cfg.td2 = regk_strcop_d;
2186                 rw_cfg.td3 = regk_strcop_d;
2187                 break;
2188         case cryptocop_3des_dee:
2189                 rw_cfg.td1 = regk_strcop_d;
2190                 rw_cfg.td2 = regk_strcop_e;
2191                 rw_cfg.td3 = regk_strcop_e;
2192                 break;
2193         case cryptocop_3des_ded:
2194                 rw_cfg.td1 = regk_strcop_d;
2195                 rw_cfg.td2 = regk_strcop_e;
2196                 rw_cfg.td3 = regk_strcop_d;
2197                 break;
2198         case cryptocop_3des_dde:
2199                 rw_cfg.td1 = regk_strcop_d;
2200                 rw_cfg.td2 = regk_strcop_d;
2201                 rw_cfg.td3 = regk_strcop_e;
2202                 break;
2203         case cryptocop_3des_ddd:
2204                 rw_cfg.td1 = regk_strcop_d;
2205                 rw_cfg.td2 = regk_strcop_d;
2206                 rw_cfg.td3 = regk_strcop_d;
2207                 break;
2208         default:
2209                 DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2210         }
2211         switch (pj->iop->csum_mode){
2212         case cryptocop_csum_le:
2213                 rw_cfg.ipend = regk_strcop_little;
2214                 break;
2215         case cryptocop_csum_be:
2216                 rw_cfg.ipend = regk_strcop_big;
2217                 break;
2218         default:
2219                 DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2220         }
2221         REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2222
2223         DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2224                      "ctx_in: 0x%p, phys: 0x%p\n"
2225                      "ctx_out: 0x%p, phys: 0x%p\n",
2226                      pj,
2227                      &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2228                      &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2229
2230         /* Start input DMA. */
2231         flush_dma_context(&pj->iop->ctx_in);
2232         DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
2233
2234         /* Start output DMA. */
2235         DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
2236
2237         spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2238         DEBUG(printk("cryptocop_start_job: exiting\n"));
2239 }
2240
2241
2242 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2243 {
2244         int  err;
2245         int  alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2246         void *iop_alloc_ptr = NULL;
2247
2248         *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2249         if (!*pj) return -ENOMEM;
2250
2251         DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2252
2253         (*pj)->oper = operation;
2254         DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n",  (*pj)->oper->cb, (*pj)->oper->cb_data));
2255
2256         if (operation->use_dmalists) {
2257                 DEBUG(print_user_dma_lists(&operation->list_op));
2258                 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2259                         DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2260                         kfree(*pj);
2261                         return -EINVAL;
2262                 }
2263                 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2264                 if (!iop_alloc_ptr) {
2265                         DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2266                         kfree(*pj);
2267                         return -ENOMEM;
2268                 }
2269                 (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2270                 DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2271                 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2272                 (*pj)->iop->sid = operation->sid;
2273                 (*pj)->iop->cdesc_out = NULL;
2274                 (*pj)->iop->cdesc_in = NULL;
2275                 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2276                 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2277                 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2278                 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2279
2280                 /* Setup DMA contexts. */
2281                 (*pj)->iop->ctx_out.next = NULL;
2282                 (*pj)->iop->ctx_out.eol = 1;
2283                 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2284                 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2285
2286                 (*pj)->iop->ctx_in.next = NULL;
2287                 (*pj)->iop->ctx_in.eol = 1;
2288                 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2289                 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2290         } else {
2291                 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2292                         DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2293                         kfree(*pj);
2294                         return err;
2295                 }
2296         }
2297         DEBUG(print_dma_descriptors((*pj)->iop));
2298
2299         DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2300
2301         return 0;
2302 }
2303
2304 static int cryptocop_open(struct inode *inode, struct file *filp)
2305 {
2306         int p = iminor(inode);
2307
2308         if (p != CRYPTOCOP_MINOR) return -EINVAL;
2309
2310         filp->private_data = NULL;
2311         return 0;
2312 }
2313
2314
2315 static int cryptocop_release(struct inode *inode, struct file *filp)
2316 {
2317         struct cryptocop_private *dev = filp->private_data;
2318         struct cryptocop_private *dev_next;
2319
2320         while (dev){
2321                 dev_next = dev->next;
2322                 if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2323                         (void)cryptocop_free_session(dev->sid);
2324                 }
2325                 kfree(dev);
2326                 dev = dev_next;
2327         }
2328
2329         return 0;
2330 }
2331
2332
2333 static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2334                                          unsigned int cmd, unsigned long arg)
2335 {
2336         struct cryptocop_private  *dev = filp->private_data;
2337         struct cryptocop_private  *prev_dev = NULL;
2338         struct strcop_session_op  *sess_op = (struct strcop_session_op *)arg;
2339         struct strcop_session_op  sop;
2340         int                       err;
2341
2342         DEBUG(printk("cryptocop_ioctl_close_session\n"));
2343
2344         if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2345                 return -EFAULT;
2346         err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2347         if (err) return -EFAULT;
2348
2349         while (dev && (dev->sid != sop.ses_id)) {
2350                 prev_dev = dev;
2351                 dev = dev->next;
2352         }
2353         if (dev){
2354                 if (prev_dev){
2355                         prev_dev->next = dev->next;
2356                 } else {
2357                         filp->private_data = dev->next;
2358                 }
2359                 err = cryptocop_free_session(dev->sid);
2360                 if (err) return -EFAULT;
2361         } else {
2362                 DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2363                 return -EINVAL;
2364         }
2365         return 0;
2366 }
2367
2368
2369 static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2370 {
2371         struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2372
2373         DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2374
2375         jc->processed = 1;
2376         wake_up(&cryptocop_ioc_process_wq);
2377 }
2378
2379
2380 #define CRYPTOCOP_IOCTL_CIPHER_TID  (1)
2381 #define CRYPTOCOP_IOCTL_DIGEST_TID  (2)
2382 #define CRYPTOCOP_IOCTL_CSUM_TID    (3)
2383
2384 static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2385 {
2386         size_t ch_ix = 0;
2387
2388         if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2389         if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2390         if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2391
2392         DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2393         return ch_ix;
2394 }
2395
2396
2397 static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2398 {
2399         size_t ch_ix = INT_MAX;
2400         size_t tmp_ix = 0;
2401
2402         if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2403                 if (crp_op->cipher_start > ix) {
2404                         ch_ix = crp_op->cipher_start;
2405                 } else {
2406                         ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2407                 }
2408         }
2409         if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2410                 if (crp_op->digest_start > ix) {
2411                         tmp_ix = crp_op->digest_start;
2412                 } else {
2413                         tmp_ix = crp_op->digest_start + crp_op->digest_len;
2414                 }
2415                 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2416         }
2417         if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2418                 if (crp_op->csum_start > ix) {
2419                         tmp_ix = crp_op->csum_start;
2420                 } else {
2421                         tmp_ix = crp_op->csum_start + crp_op->csum_len;
2422                 }
2423                 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2424         }
2425         if (ch_ix == INT_MAX) ch_ix = ix;
2426         DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2427         return ch_ix;
2428 }
2429
2430
2431 /* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2432  * Return -1 for ok, 0 for fail. */
2433 static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2434 {
2435         int tmplen;
2436
2437         assert(iov != NULL);
2438         assert(iovix != NULL);
2439         assert(pages != NULL);
2440         assert(pageix != NULL);
2441         assert(pageoffset != NULL);
2442
2443         DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2444
2445         while (map_length > 0){
2446                 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2447                 if (*iovix >= iovlen){
2448                         DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2449                         return 0;
2450                 }
2451                 if (*pageix >= nopages){
2452                         DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2453                         return 0;
2454                 }
2455                 iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2456                 tmplen = PAGE_SIZE - *pageoffset;
2457                 if (tmplen < map_length){
2458                         (*pageoffset) = 0;
2459                         (*pageix)++;
2460                 } else {
2461                         tmplen = map_length;
2462                         (*pageoffset) += map_length;
2463                 }
2464                 DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2465                 iov[*iovix].iov_len = tmplen;
2466                 map_length -= tmplen;
2467                 (*iovix)++;
2468         }
2469         DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2470         return -1;
2471 }
2472
2473
2474
2475 static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2476 {
2477         int                             i;
2478         struct cryptocop_private        *dev = filp->private_data;
2479         struct strcop_crypto_op         *crp_oper = (struct strcop_crypto_op *)arg;
2480         struct strcop_crypto_op         oper = {0};
2481         int                             err = 0;
2482         struct cryptocop_operation      *cop = NULL;
2483
2484         struct ioctl_job_cb_ctx         *jc = NULL;
2485
2486         struct page                     **inpages = NULL;
2487         struct page                     **outpages = NULL;
2488         int                             noinpages = 0;
2489         int                             nooutpages = 0;
2490
2491         struct cryptocop_desc           descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2492                                                    * can get connected/disconnected on different places in the indata. */
2493         struct cryptocop_desc_cfg       dcfgs[5*3];
2494         int                             desc_ix = 0;
2495         int                             dcfg_ix = 0;
2496         struct cryptocop_tfrm_cfg       ciph_tcfg = {0};
2497         struct cryptocop_tfrm_cfg       digest_tcfg = {0};
2498         struct cryptocop_tfrm_cfg       csum_tcfg = {0};
2499
2500         unsigned char                   *digest_result = NULL;
2501         int                             digest_length = 0;
2502         int                             cblocklen = 0;
2503         unsigned char                   csum_result[CSUM_BLOCK_LENGTH];
2504         struct cryptocop_session        *sess;
2505
2506         int    iovlen = 0;
2507         int    iovix = 0;
2508         int    pageix = 0;
2509         int    pageoffset = 0;
2510
2511         size_t prev_ix = 0;
2512         size_t next_ix;
2513
2514         int    cipher_active, digest_active, csum_active;
2515         int    end_digest, end_csum;
2516         int    digest_done = 0;
2517         int    cipher_done = 0;
2518         int    csum_done = 0;
2519
2520         DEBUG(printk("cryptocop_ioctl_process\n"));
2521
2522         if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2523                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2524                 return -EFAULT;
2525         }
2526         if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2527                 DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2528                 return -EFAULT;
2529         }
2530         DEBUG(print_strcop_crypto_op(&oper));
2531
2532         while (dev && dev->sid != oper.ses_id) dev = dev->next;
2533         if (!dev){
2534                 DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2535                 return -EINVAL;
2536         }
2537
2538         /* Check buffers. */
2539         if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2540                 DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2541                 return -EINVAL;
2542         }
2543
2544         if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2545                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2546                 return -EFAULT;
2547         }
2548         if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2549                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2550                 return -EFAULT;
2551         }
2552
2553         cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2554         if (!cop) {
2555                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2556                 return -ENOMEM;
2557         }
2558         jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2559         if (!jc) {
2560                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2561                 err = -ENOMEM;
2562                 goto error_cleanup;
2563         }
2564         jc->processed = 0;
2565
2566         cop->cb_data = jc;
2567         cop->cb = ioctl_process_job_callback;
2568         cop->operation_status = 0;
2569         cop->use_dmalists = 0;
2570         cop->in_interrupt = 0;
2571         cop->fast_callback = 0;
2572         cop->tfrm_op.tfrm_cfg = NULL;
2573         cop->tfrm_op.desc = NULL;
2574         cop->tfrm_op.indata = NULL;
2575         cop->tfrm_op.incount = 0;
2576         cop->tfrm_op.inlen = 0;
2577         cop->tfrm_op.outdata = NULL;
2578         cop->tfrm_op.outcount = 0;
2579         cop->tfrm_op.outlen = 0;
2580
2581         sess = get_session(oper.ses_id);
2582         if (!sess){
2583                 DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2584                 kfree(cop);
2585                 kfree(jc);
2586                 return -EINVAL;
2587         }
2588
2589         if (oper.do_cipher) {
2590                 unsigned int                    cipher_outlen = 0;
2591                 struct cryptocop_transform_ctx  *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2592                 if (!tc) {
2593                         DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2594                         err = -EINVAL;
2595                         goto error_cleanup;
2596                 }
2597                 ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2598                 ciph_tcfg.inject_ix = 0;
2599                 ciph_tcfg.flags = 0;
2600                 if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2601                         DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2602                         kfree(cop);
2603                         kfree(jc);
2604                         return -EINVAL;
2605                 }
2606                 cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2607                 if (oper.cipher_len % cblocklen) {
2608                         kfree(cop);
2609                         kfree(jc);
2610                         DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2611                         return -EINVAL;
2612                 }
2613                 cipher_outlen = oper.cipher_len;
2614                 if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2615                         if (oper.cipher_explicit) {
2616                                 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2617                                 memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2618                         } else {
2619                                 cipher_outlen = oper.cipher_len - cblocklen;
2620                         }
2621                 } else {
2622                         if (oper.cipher_explicit){
2623                                 kfree(cop);
2624                                 kfree(jc);
2625                                 DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2626                                 return -EINVAL;
2627                         }
2628                 }
2629                 if (oper.cipher_outlen != cipher_outlen) {
2630                         kfree(cop);
2631                         kfree(jc);
2632                         DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2633                         return -EINVAL;
2634                 }
2635
2636                 if (oper.decrypt){
2637                         ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2638                 } else {
2639                         ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2640                 }
2641                 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2642                 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2643         }
2644         if (oper.do_digest){
2645                 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2646                 if (!tc) {
2647                         DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2648                         err = -EINVAL;
2649                         goto error_cleanup;
2650                 }
2651                 digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2652                 digest_result = kmalloc(digest_length, GFP_KERNEL);
2653                 if (!digest_result) {
2654                         DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2655                         err = -EINVAL;
2656                         goto error_cleanup;
2657                 }
2658                 DEBUG(memset(digest_result, 0xff, digest_length));
2659
2660                 digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2661                 digest_tcfg.inject_ix = 0;
2662                 ciph_tcfg.inject_ix += digest_length;
2663                 if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2664                         DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2665                         err = -EINVAL;
2666                         goto error_cleanup;
2667                 }
2668
2669                 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2670                 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2671         }
2672         if (oper.do_csum){
2673                 csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2674                 csum_tcfg.inject_ix = digest_length;
2675                 ciph_tcfg.inject_ix += 2;
2676
2677                 if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2678                         DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2679                         kfree(cop);
2680                         kfree(jc);
2681                         return -EINVAL;
2682                 }
2683
2684                 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2685                 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2686         }
2687
2688         prev_ix = first_cfg_change_ix(&oper);
2689         if (prev_ix > oper.inlen) {
2690                 DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2691                 nooutpages = noinpages = 0;
2692                 err = -EINVAL;
2693                 goto error_cleanup;
2694         }
2695         DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2696
2697         /* Map user pages for in and out data of the operation. */
2698         noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2699         DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2700         inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2701         if (!inpages){
2702                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2703                 nooutpages = noinpages = 0;
2704                 err = -ENOMEM;
2705                 goto error_cleanup;
2706         }
2707         if (oper.do_cipher){
2708                 nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2709                 DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2710                 outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2711                 if (!outpages){
2712                         DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2713                         nooutpages = noinpages = 0;
2714                         err = -ENOMEM;
2715                         goto error_cleanup;
2716                 }
2717         }
2718
2719         /* Acquire the mm page semaphore. */
2720         down_read(&current->mm->mmap_sem);
2721
2722         err = get_user_pages((unsigned long int)(oper.indata + prev_ix),
2723                              noinpages,
2724                              0,  /* read access only for in data */
2725                              0, /* no force */
2726                              inpages,
2727                              NULL);
2728
2729         if (err < 0) {
2730                 up_read(&current->mm->mmap_sem);
2731                 nooutpages = noinpages = 0;
2732                 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2733                 goto error_cleanup;
2734         }
2735         noinpages = err;
2736         if (oper.do_cipher){
2737                 err = get_user_pages((unsigned long int)oper.cipher_outdata,
2738                                      nooutpages,
2739                                      1, /* write access for out data */
2740                                      0, /* no force */
2741                                      outpages,
2742                                      NULL);
2743                 up_read(&current->mm->mmap_sem);
2744                 if (err < 0) {
2745                         nooutpages = 0;
2746                         DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2747                         goto error_cleanup;
2748                 }
2749                 nooutpages = err;
2750         } else {
2751                 up_read(&current->mm->mmap_sem);
2752         }
2753
2754         /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2755          * csum output and splits when units are (dis-)connected. */
2756         cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2757         cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2758         if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2759                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2760                 err = -ENOMEM;
2761                 goto error_cleanup;
2762         }
2763
2764         cop->tfrm_op.inlen = oper.inlen - prev_ix;
2765         cop->tfrm_op.outlen = 0;
2766         if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2767         if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2768         if (oper.do_csum) cop->tfrm_op.outlen += 2;
2769
2770         /* Setup the in iovecs. */
2771         cop->tfrm_op.incount = noinpages;
2772         if (noinpages > 1){
2773                 size_t tmplen = cop->tfrm_op.inlen;
2774
2775                 cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2776                 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2777                 tmplen -= cop->tfrm_op.indata[0].iov_len;
2778                 for (i = 1; i<noinpages; i++){
2779                         cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2780                         cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2781                         tmplen -= PAGE_SIZE;
2782                 }
2783         } else {
2784                 cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2785                 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2786         }
2787
2788         iovlen = nooutpages + 6;
2789         pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2790
2791         next_ix = next_cfg_change_ix(&oper, prev_ix);
2792         if (prev_ix == next_ix){
2793                 DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2794                 err = -EINVAL;  /* This should be impossible barring bugs. */
2795                 goto error_cleanup;
2796         }
2797         while (prev_ix != next_ix){
2798                 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2799                 descs[desc_ix].cfg = NULL;
2800                 descs[desc_ix].length = next_ix - prev_ix;
2801
2802                 if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2803                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2804                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2805                         cipher_active = 1;
2806
2807                         if (next_ix == (oper.cipher_start + oper.cipher_len)){
2808                                 cipher_done = 1;
2809                                 dcfgs[dcfg_ix].last = 1;
2810                         } else {
2811                                 dcfgs[dcfg_ix].last = 0;
2812                         }
2813                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2814                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2815                         ++dcfg_ix;
2816                 }
2817                 if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2818                         digest_active = 1;
2819                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2820                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2821                         if (next_ix == (oper.digest_start + oper.digest_len)){
2822                                 assert(!digest_done);
2823                                 digest_done = 1;
2824                                 dcfgs[dcfg_ix].last = 1;
2825                         } else {
2826                                 dcfgs[dcfg_ix].last = 0;
2827                         }
2828                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2829                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2830                         ++dcfg_ix;
2831                 }
2832                 if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2833                         csum_active = 1;
2834                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2835                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2836                         if (next_ix == (oper.csum_start + oper.csum_len)){
2837                                 csum_done = 1;
2838                                 dcfgs[dcfg_ix].last = 1;
2839                         } else {
2840                                 dcfgs[dcfg_ix].last = 0;
2841                         }
2842                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2843                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2844                         ++dcfg_ix;
2845                 }
2846                 if (!descs[desc_ix].cfg){
2847                         DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2848                         err = -EINVAL;
2849                         goto error_cleanup;
2850                 }
2851                 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2852                 ++desc_ix;
2853                 prev_ix = next_ix;
2854                 next_ix = next_cfg_change_ix(&oper, prev_ix);
2855         }
2856         if (desc_ix > 0){
2857                 descs[desc_ix-1].next = NULL;
2858         } else {
2859                 descs[0].next = NULL;
2860         }
2861         if (oper.do_digest) {
2862                 DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2863                 /* Add outdata iovec, length == <length of type of digest> */
2864                 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2865                 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2866                 ++iovix;
2867         }
2868         if (oper.do_csum) {
2869                 /* Add outdata iovec, length == 2, the length of csum. */
2870                 DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2871                 /* Add outdata iovec, length == <length of type of digest> */
2872                 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2873                 cop->tfrm_op.outdata[iovix].iov_len = 2;
2874                 ++iovix;
2875         }
2876         if (oper.do_cipher) {
2877                 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2878                         DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2879                         err = -ENOSYS; /* This should be impossible barring bugs. */
2880                         goto error_cleanup;
2881                 }
2882         }
2883         DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2884         cop->tfrm_op.outcount = iovix;
2885         assert(iovix <= (nooutpages + 6));
2886
2887         cop->sid = oper.ses_id;
2888         cop->tfrm_op.desc = &descs[0];
2889
2890         DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2891
2892         if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2893                 DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2894                 err = -EINVAL;
2895                 goto error_cleanup;
2896         }
2897
2898         DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2899
2900         wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2901         DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2902         if (!jc->processed){
2903                 printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2904                 err = -EIO;
2905                 goto error_cleanup;
2906         }
2907
2908         /* Job process done.  Cipher output should already be correct in job so no post processing of outdata. */
2909         DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2910         if (cop->operation_status == 0){
2911                 if (oper.do_digest){
2912                         DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2913                         err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2914                         if (0 != err){
2915                                 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2916                                 err = -EFAULT;
2917                                 goto error_cleanup;
2918                         }
2919                 }
2920                 if (oper.do_csum){
2921                         DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2922                         err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2923                         if (0 != err){
2924                                 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2925                                 err = -EFAULT;
2926                                 goto error_cleanup;
2927                         }
2928                 }
2929                 err = 0;
2930         } else {
2931                 DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2932                 err = cop->operation_status;
2933         }
2934
2935  error_cleanup:
2936         /* Release page caches. */
2937         for (i = 0; i < noinpages; i++){
2938                 put_page(inpages[i]);
2939         }
2940         for (i = 0; i < nooutpages; i++){
2941                 int spdl_err;
2942                 /* Mark output pages dirty. */
2943                 spdl_err = set_page_dirty_lock(outpages[i]);
2944                 DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2945         }
2946         for (i = 0; i < nooutpages; i++){
2947                 put_page(outpages[i]);
2948         }
2949
2950         kfree(digest_result);
2951         kfree(inpages);
2952         kfree(outpages);
2953         if (cop){
2954                 kfree(cop->tfrm_op.indata);
2955                 kfree(cop->tfrm_op.outdata);
2956                 kfree(cop);
2957         }
2958         kfree(jc);
2959
2960         DEBUG(print_lock_status());
2961
2962         return err;
2963 }
2964
2965
2966 static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2967 {
2968         cryptocop_session_id             sid;
2969         int                              err;
2970         struct cryptocop_private         *dev;
2971         struct strcop_session_op         *sess_op = (struct strcop_session_op *)arg;
2972         struct strcop_session_op         sop;
2973         struct cryptocop_transform_init  *tis = NULL;
2974         struct cryptocop_transform_init  ti_cipher = {0};
2975         struct cryptocop_transform_init  ti_digest = {0};
2976         struct cryptocop_transform_init  ti_csum = {0};
2977
2978         if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2979                 return -EFAULT;
2980         err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2981         if (err) return -EFAULT;
2982         if (sop.cipher != cryptocop_cipher_none) {
2983                 if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2984         }
2985         DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2986
2987         DEBUG(printk("\tcipher:%d\n"
2988                      "\tcipher_mode:%d\n"
2989                      "\tdigest:%d\n"
2990                      "\tcsum:%d\n",
2991                      (int)sop.cipher,
2992                      (int)sop.cmode,
2993                      (int)sop.digest,
2994                      (int)sop.csum));
2995
2996         if (sop.cipher != cryptocop_cipher_none){
2997                 /* Init the cipher. */
2998                 switch (sop.cipher){
2999                 case cryptocop_cipher_des:
3000                         ti_cipher.alg = cryptocop_alg_des;
3001                         break;
3002                 case cryptocop_cipher_3des:
3003                         ti_cipher.alg = cryptocop_alg_3des;
3004                         break;
3005                 case cryptocop_cipher_aes:
3006                         ti_cipher.alg = cryptocop_alg_aes;
3007                         break;
3008                 default:
3009                         DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3010                         return -EINVAL;
3011                 };
3012                 DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3013                 copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3014                 ti_cipher.keylen = sop.keylen;
3015                 switch (sop.cmode){
3016                 case cryptocop_cipher_mode_cbc:
3017                 case cryptocop_cipher_mode_ecb:
3018                         ti_cipher.cipher_mode = sop.cmode;
3019                         break;
3020                 default:
3021                         DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3022                         return -EINVAL;
3023                 }
3024                 DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3025                 switch (sop.des3_mode){
3026                 case cryptocop_3des_eee:
3027                 case cryptocop_3des_eed:
3028                 case cryptocop_3des_ede:
3029                 case cryptocop_3des_edd:
3030                 case cryptocop_3des_dee:
3031                 case cryptocop_3des_ded:
3032                 case cryptocop_3des_dde:
3033                 case cryptocop_3des_ddd:
3034                         ti_cipher.tdes_mode = sop.des3_mode;
3035                         break;
3036                 default:
3037                         DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3038                         return -EINVAL;
3039                 }
3040                 ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3041                 ti_cipher.next = tis;
3042                 tis = &ti_cipher;
3043         } /* if (sop.cipher != cryptocop_cipher_none) */
3044         if (sop.digest != cryptocop_digest_none){
3045                 DEBUG(printk("setting digest transform\n"));
3046                 switch (sop.digest){
3047                 case cryptocop_digest_md5:
3048                         ti_digest.alg = cryptocop_alg_md5;
3049                         break;
3050                 case cryptocop_digest_sha1:
3051                         ti_digest.alg = cryptocop_alg_sha1;
3052                         break;
3053                 default:
3054                         DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3055                         return -EINVAL;
3056                 }
3057                 ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3058                 ti_digest.next = tis;
3059                 tis = &ti_digest;
3060         } /* if (sop.digest != cryptocop_digest_none) */
3061         if (sop.csum != cryptocop_csum_none){
3062                 DEBUG(printk("setting csum transform\n"));
3063                 switch (sop.csum){
3064                 case cryptocop_csum_le:
3065                 case cryptocop_csum_be:
3066                         ti_csum.csum_mode = sop.csum;
3067                         break;
3068                 default:
3069                         DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3070                         return -EINVAL;
3071                 }
3072                 ti_csum.alg = cryptocop_alg_csum;
3073                 ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3074                 ti_csum.next = tis;
3075                 tis = &ti_csum;
3076         } /* (sop.csum != cryptocop_csum_none) */
3077         dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3078         if (!dev){
3079                 DEBUG_API(printk("create session, alloc dev\n"));
3080                 return -ENOMEM;
3081         }
3082
3083         err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3084         DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3085
3086         if (err) {
3087                 kfree(dev);
3088                 return err;
3089         }
3090         sess_op->ses_id = sid;
3091         dev->sid = sid;
3092         dev->next = filp->private_data;
3093         filp->private_data = dev;
3094
3095         return 0;
3096 }
3097
3098 static long cryptocop_ioctl_unlocked(struct inode *inode,
3099         struct file *filp, unsigned int cmd, unsigned long arg)
3100 {
3101         int err = 0;
3102         if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3103                 DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3104                 return -ENOTTY;
3105         }
3106         if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3107                 return -ENOTTY;
3108         }
3109         /* Access check of the argument.  Some commands, e.g. create session and process op,
3110            needs additional checks.  Those are handled in the command handling functions. */
3111         if (_IOC_DIR(cmd) & _IOC_READ)
3112                 err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3113         else if (_IOC_DIR(cmd) & _IOC_WRITE)
3114                 err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3115         if (err) return -EFAULT;
3116
3117         switch (cmd) {
3118         case CRYPTOCOP_IO_CREATE_SESSION:
3119                 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3120         case CRYPTOCOP_IO_CLOSE_SESSION:
3121                 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3122         case CRYPTOCOP_IO_PROCESS_OP:
3123                 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3124         default:
3125                 DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3126                 return -ENOTTY;
3127         }
3128         return 0;
3129 }
3130
3131 static long
3132 cryptocop_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
3133 {
3134        long ret;
3135
3136        mutex_lock(&cryptocop_mutex);
3137        ret = cryptocop_ioctl_unlocked(file_inode(filp), filp, cmd, arg);
3138        mutex_unlock(&cryptocop_mutex);
3139
3140        return ret;
3141 }
3142
3143
3144 #ifdef LDEBUG
3145 static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3146 {
3147         struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3148         struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3149         int                       i;
3150
3151         printk("print_dma_descriptors start\n");
3152
3153         printk("iop:\n");
3154         printk("\tsid: 0x%lld\n", iop->sid);
3155
3156         printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3157         printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3158         printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3159         printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3160
3161         printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3162         printk("\tnext: 0x%p\n"
3163                "\tsaved_data: 0x%p\n"
3164                "\tsaved_data_buf: 0x%p\n",
3165                iop->ctx_out.next,
3166                iop->ctx_out.saved_data,
3167                iop->ctx_out.saved_data_buf);
3168
3169         printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3170         printk("\tnext: 0x%p\n"
3171                "\tsaved_data: 0x%p\n"
3172                "\tsaved_data_buf: 0x%p\n",
3173                iop->ctx_in.next,
3174                iop->ctx_in.saved_data,
3175                iop->ctx_in.saved_data_buf);
3176
3177         i = 0;
3178         while (cdesc_out) {
3179                 dma_descr_data *td;
3180                 printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3181                 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3182                 td = cdesc_out->dma_descr;
3183                 printk("\n\tbuf: 0x%p\n"
3184                        "\tafter: 0x%p\n"
3185                        "\tmd: 0x%04x\n"
3186                        "\tnext: 0x%p\n",
3187                        td->buf,
3188                        td->after,
3189                        td->md,
3190                        td->next);
3191                 printk("flags:\n"
3192                        "\twait:\t%d\n"
3193                        "\teol:\t%d\n"
3194                        "\touteop:\t%d\n"
3195                        "\tineop:\t%d\n"
3196                        "\tintr:\t%d\n",
3197                        td->wait,
3198                        td->eol,
3199                        td->out_eop,
3200                        td->in_eop,
3201                        td->intr);
3202                 cdesc_out = cdesc_out->next;
3203                 i++;
3204         }
3205         i = 0;
3206         while (cdesc_in) {
3207                 dma_descr_data *td;
3208                 printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3209                 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3210                 td = cdesc_in->dma_descr;
3211                 printk("\n\tbuf: 0x%p\n"
3212                        "\tafter: 0x%p\n"
3213                        "\tmd: 0x%04x\n"
3214                        "\tnext: 0x%p\n",
3215                        td->buf,
3216                        td->after,
3217                        td->md,
3218                        td->next);
3219                 printk("flags:\n"
3220                        "\twait:\t%d\n"
3221                        "\teol:\t%d\n"
3222                        "\touteop:\t%d\n"
3223                        "\tineop:\t%d\n"
3224                        "\tintr:\t%d\n",
3225                        td->wait,
3226                        td->eol,
3227                        td->out_eop,
3228                        td->in_eop,
3229                        td->intr);
3230                 cdesc_in = cdesc_in->next;
3231                 i++;
3232         }
3233
3234         printk("print_dma_descriptors end\n");
3235 }
3236
3237
3238 static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3239 {
3240         printk("print_strcop_crypto_op, 0x%p\n", cop);
3241
3242         /* Indata. */
3243         printk("indata=0x%p\n"
3244                "inlen=%d\n"
3245                "do_cipher=%d\n"
3246                "decrypt=%d\n"
3247                "cipher_explicit=%d\n"
3248                "cipher_start=%d\n"
3249                "cipher_len=%d\n"
3250                "outdata=0x%p\n"
3251                "outlen=%d\n",
3252                cop->indata,
3253                cop->inlen,
3254                cop->do_cipher,
3255                cop->decrypt,
3256                cop->cipher_explicit,
3257                cop->cipher_start,
3258                cop->cipher_len,
3259                cop->cipher_outdata,
3260                cop->cipher_outlen);
3261
3262         printk("do_digest=%d\n"
3263                "digest_start=%d\n"
3264                "digest_len=%d\n",
3265                cop->do_digest,
3266                cop->digest_start,
3267                cop->digest_len);
3268
3269         printk("do_csum=%d\n"
3270                "csum_start=%d\n"
3271                "csum_len=%d\n",
3272                cop->do_csum,
3273                cop->csum_start,
3274                cop->csum_len);
3275 }
3276
3277 static void print_cryptocop_operation(struct cryptocop_operation *cop)
3278 {
3279         struct cryptocop_desc      *d;
3280         struct cryptocop_tfrm_cfg  *tc;
3281         struct cryptocop_desc_cfg  *dc;
3282         int                        i;
3283
3284         printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3285         printk("sid: %lld\n", cop->sid);
3286         printk("operation_status=%d\n"
3287                "use_dmalists=%d\n"
3288                "in_interrupt=%d\n"
3289                "fast_callback=%d\n",
3290                cop->operation_status,
3291                cop->use_dmalists,
3292                cop->in_interrupt,
3293                cop->fast_callback);
3294
3295         if (cop->use_dmalists){
3296                 print_user_dma_lists(&cop->list_op);
3297         } else {
3298                 printk("cop->tfrm_op\n"
3299                        "tfrm_cfg=0x%p\n"
3300                        "desc=0x%p\n"
3301                        "indata=0x%p\n"
3302                        "incount=%d\n"
3303                        "inlen=%d\n"
3304                        "outdata=0x%p\n"
3305                        "outcount=%d\n"
3306                        "outlen=%d\n\n",
3307                        cop->tfrm_op.tfrm_cfg,
3308                        cop->tfrm_op.desc,
3309                        cop->tfrm_op.indata,
3310                        cop->tfrm_op.incount,
3311                        cop->tfrm_op.inlen,
3312                        cop->tfrm_op.outdata,
3313                        cop->tfrm_op.outcount,
3314                        cop->tfrm_op.outlen);
3315
3316                 tc = cop->tfrm_op.tfrm_cfg;
3317                 while (tc){
3318                         printk("tfrm_cfg, 0x%p\n"
3319                                "tid=%d\n"
3320                                "flags=%d\n"
3321                                "inject_ix=%d\n"
3322                                "next=0x%p\n",
3323                                tc,
3324                                tc->tid,
3325                                tc->flags,
3326                                tc->inject_ix,
3327                                tc->next);
3328                         tc = tc->next;
3329                 }
3330                 d = cop->tfrm_op.desc;
3331                 while (d){
3332                         printk("\n======================desc, 0x%p\n"
3333                                "length=%d\n"
3334                                "cfg=0x%p\n"
3335                                "next=0x%p\n",
3336                                d,
3337                                d->length,
3338                                d->cfg,
3339                                d->next);
3340                         dc = d->cfg;
3341                         while (dc){
3342                                 printk("=========desc_cfg, 0x%p\n"
3343                                        "tid=%d\n"
3344                                        "src=%d\n"
3345                                        "last=%d\n"
3346                                        "next=0x%p\n",
3347                                        dc,
3348                                        dc->tid,
3349                                        dc->src,
3350                                        dc->last,
3351                                        dc->next);
3352                                 dc = dc->next;
3353                         }
3354                         d = d->next;
3355                 }
3356                 printk("\n====iniov\n");
3357                 for (i = 0; i < cop->tfrm_op.incount; i++){
3358                         printk("indata[%d]\n"
3359                                "base=0x%p\n"
3360                                "len=%d\n",
3361                                i,
3362                                cop->tfrm_op.indata[i].iov_base,
3363                                cop->tfrm_op.indata[i].iov_len);
3364                 }
3365                 printk("\n====outiov\n");
3366                 for (i = 0; i < cop->tfrm_op.outcount; i++){
3367                         printk("outdata[%d]\n"
3368                                "base=0x%p\n"
3369                                "len=%d\n",
3370                                i,
3371                                cop->tfrm_op.outdata[i].iov_base,
3372                                cop->tfrm_op.outdata[i].iov_len);
3373                 }
3374         }
3375         printk("------------end print_cryptocop_operation\n");
3376 }
3377
3378
3379 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3380 {
3381         dma_descr_data *dd;
3382         int i;
3383
3384         printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3385
3386         printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3387         printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3388
3389         printk("##############outlist\n");
3390         dd = phys_to_virt((unsigned long int)dma_op->outlist);
3391         i = 0;
3392         while (dd != NULL) {
3393                 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3394                 printk("\n\tbuf: 0x%p\n"
3395                        "\tafter: 0x%p\n"
3396                        "\tmd: 0x%04x\n"
3397                        "\tnext: 0x%p\n",
3398                        dd->buf,
3399                        dd->after,
3400                        dd->md,
3401                        dd->next);
3402                 printk("flags:\n"
3403                        "\twait:\t%d\n"
3404                        "\teol:\t%d\n"
3405                        "\touteop:\t%d\n"
3406                        "\tineop:\t%d\n"
3407                        "\tintr:\t%d\n",
3408                        dd->wait,
3409                        dd->eol,
3410                        dd->out_eop,
3411                        dd->in_eop,
3412                        dd->intr);
3413                 if (dd->eol)
3414                         dd = NULL;
3415                 else
3416                         dd = phys_to_virt((unsigned long int)dd->next);
3417                 ++i;
3418         }
3419
3420         printk("##############inlist\n");
3421         dd = phys_to_virt((unsigned long int)dma_op->inlist);
3422         i = 0;
3423         while (dd != NULL) {
3424                 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3425                 printk("\n\tbuf: 0x%p\n"
3426                        "\tafter: 0x%p\n"
3427                        "\tmd: 0x%04x\n"
3428                        "\tnext: 0x%p\n",
3429                        dd->buf,
3430                        dd->after,
3431                        dd->md,
3432                        dd->next);
3433                 printk("flags:\n"
3434                        "\twait:\t%d\n"
3435                        "\teol:\t%d\n"
3436                        "\touteop:\t%d\n"
3437                        "\tineop:\t%d\n"
3438                        "\tintr:\t%d\n",
3439                        dd->wait,
3440                        dd->eol,
3441                        dd->out_eop,
3442                        dd->in_eop,
3443                        dd->intr);
3444                 if (dd->eol)
3445                         dd = NULL;
3446                 else
3447                         dd = phys_to_virt((unsigned long int)dd->next);
3448                 ++i;
3449         }
3450 }
3451
3452
3453 static void print_lock_status(void)
3454 {
3455         printk("**********************print_lock_status\n");
3456         printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3457         printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3458         printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3459         printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3460         printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3461         printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3462 }
3463 #endif /* LDEBUG */
3464
3465
3466 static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3467
3468 static int init_stream_coprocessor(void)
3469 {
3470         int err;
3471         int i;
3472         static int initialized = 0;
3473
3474         if (initialized)
3475                 return 0;
3476
3477         initialized = 1;
3478
3479         printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3480
3481         err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3482         if (err < 0) {
3483                 printk(KERN_ERR "stream co-processor: could not get major number.\n");
3484                 return err;
3485         }
3486
3487         err = init_cryptocop();
3488         if (err) {
3489                 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3490                 return err;
3491         }
3492         err = cryptocop_job_queue_init();
3493         if (err) {
3494                 release_cryptocop();
3495                 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3496                 return err;
3497         }
3498         /* Init the descriptor pool. */
3499         for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3500                 descr_pool[i].from_pool = 1;
3501                 descr_pool[i].next = &descr_pool[i + 1];
3502         }
3503         descr_pool[i].from_pool = 1;
3504         descr_pool[i].next = NULL;
3505         descr_pool_free_list = &descr_pool[0];
3506         descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3507
3508         spin_lock_init(&cryptocop_completed_jobs_lock);
3509         spin_lock_init(&cryptocop_job_queue_lock);
3510         spin_lock_init(&descr_pool_lock);
3511         spin_lock_init(&cryptocop_sessions_lock);
3512         spin_lock_init(&running_job_lock);
3513         spin_lock_init(&cryptocop_process_lock);
3514
3515         cryptocop_sessions = NULL;
3516         next_sid = 1;
3517
3518         cryptocop_running_job = NULL;
3519
3520         printk("stream co-processor: init done.\n");
3521         return 0;
3522 }
3523
3524 static void __exit exit_stream_coprocessor(void)
3525 {
3526         release_cryptocop();
3527         cryptocop_job_queue_close();
3528 }
3529
3530 module_init(init_stream_coprocessor);
3531 module_exit(exit_stream_coprocessor);
3532