arm64/crypto: AES using ARMv8 Crypto Extensions

This patch adds support for the AES symmetric encryption algorithm for CPUs
that have support for the AES part of the ARM v8 Crypto Extensions.

Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Acked-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 0c50859..9ba32c0 100644 (file)
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -18,10 +18,15 @@ config CRYPTO_SHA2_ARM64_CE
        depends on ARM64 && KERNEL_MODE_NEON
        select CRYPTO_HASH
 
-
 config CRYPTO_GHASH_ARM64_CE
        tristate "GHASH (for GCM chaining mode) using ARMv8 Crypto Extensions"
        depends on ARM64 && KERNEL_MODE_NEON
        select CRYPTO_HASH
 
+config CRYPTO_AES_ARM64_CE
+       tristate "AES core cipher using ARMv8 Crypto Extensions"
+       depends on ARM64 && KERNEL_MODE_NEON
+       select CRYPTO_ALGAPI
+       select CRYPTO_AES
+
 endif

diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index e8c81a0..908abd9 100644 (file)
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -16,3 +16,6 @@ sha2-ce-y := sha2-ce-glue.o sha2-ce-core.o
 
 obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o
 ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o
+
+obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o
+CFLAGS_aes-ce-cipher.o += -march=armv8-a+crypto

diff --git a/arch/arm64/crypto/aes-ce-cipher.c b/arch/arm64/crypto/aes-ce-cipher.c
new file mode 100644 (file)
index 0000000..2075e1a
--- /dev/null
+++ b/arch/arm64/crypto/aes-ce-cipher.c
@@ -0,0 +1,155 @@
+/*
+ * aes-ce-cipher.c - core AES cipher using ARMv8 Crypto Extensions
+ *
+ * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/neon.h>
+#include <crypto/aes.h>
+#include <linux/cpufeature.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+
+MODULE_DESCRIPTION("Synchronous AES cipher using ARMv8 Crypto Extensions");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+
+struct aes_block {
+       u8 b[AES_BLOCK_SIZE];
+};
+
+static int num_rounds(struct crypto_aes_ctx *ctx)
+{
+       /*
+        * # of rounds specified by AES:
+        * 128 bit key          10 rounds
+        * 192 bit key          12 rounds
+        * 256 bit key          14 rounds
+        * => n byte key        => 6 + (n/4) rounds
+        */
+       return 6 + ctx->key_length / 4;
+}
+
+static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
+{
+       struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+       struct aes_block *out = (struct aes_block *)dst;
+       struct aes_block const *in = (struct aes_block *)src;
+       void *dummy0;
+       int dummy1;
+
+       kernel_neon_begin_partial(4);
+
+       __asm__("       ld1     {v0.16b}, %[in]                 ;"
+               "       ld1     {v1.2d}, [%[key]], #16          ;"
+               "       cmp     %w[rounds], #10                 ;"
+               "       bmi     0f                              ;"
+               "       bne     3f                              ;"
+               "       mov     v3.16b, v1.16b                  ;"
+               "       b       2f                              ;"
+               "0:     mov     v2.16b, v1.16b                  ;"
+               "       ld1     {v3.2d}, [%[key]], #16          ;"
+               "1:     aese    v0.16b, v2.16b                  ;"
+               "       aesmc   v0.16b, v0.16b                  ;"
+               "2:     ld1     {v1.2d}, [%[key]], #16          ;"
+               "       aese    v0.16b, v3.16b                  ;"
+               "       aesmc   v0.16b, v0.16b                  ;"
+               "3:     ld1     {v2.2d}, [%[key]], #16          ;"
+               "       subs    %w[rounds], %w[rounds], #3      ;"
+               "       aese    v0.16b, v1.16b                  ;"
+               "       aesmc   v0.16b, v0.16b                  ;"
+               "       ld1     {v3.2d}, [%[key]], #16          ;"
+               "       bpl     1b                              ;"
+               "       aese    v0.16b, v2.16b                  ;"
+               "       eor     v0.16b, v0.16b, v3.16b          ;"
+               "       st1     {v0.16b}, %[out]                ;"
+
+       :       [out]           "=Q"(*out),
+               [key]           "=r"(dummy0),
+               [rounds]        "=r"(dummy1)
+       :       [in]            "Q"(*in),
+                               "1"(ctx->key_enc),
+                               "2"(num_rounds(ctx) - 2)
+       :       "cc");
+
+       kernel_neon_end();
+}
+
+static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
+{
+       struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+       struct aes_block *out = (struct aes_block *)dst;
+       struct aes_block const *in = (struct aes_block *)src;
+       void *dummy0;
+       int dummy1;
+
+       kernel_neon_begin_partial(4);
+
+       __asm__("       ld1     {v0.16b}, %[in]                 ;"
+               "       ld1     {v1.2d}, [%[key]], #16          ;"
+               "       cmp     %w[rounds], #10                 ;"
+               "       bmi     0f                              ;"
+               "       bne     3f                              ;"
+               "       mov     v3.16b, v1.16b                  ;"
+               "       b       2f                              ;"
+               "0:     mov     v2.16b, v1.16b                  ;"
+               "       ld1     {v3.2d}, [%[key]], #16          ;"
+               "1:     aesd    v0.16b, v2.16b                  ;"
+               "       aesimc  v0.16b, v0.16b                  ;"
+               "2:     ld1     {v1.2d}, [%[key]], #16          ;"
+               "       aesd    v0.16b, v3.16b                  ;"
+               "       aesimc  v0.16b, v0.16b                  ;"
+               "3:     ld1     {v2.2d}, [%[key]], #16          ;"
+               "       subs    %w[rounds], %w[rounds], #3      ;"
+               "       aesd    v0.16b, v1.16b                  ;"
+               "       aesimc  v0.16b, v0.16b                  ;"
+               "       ld1     {v3.2d}, [%[key]], #16          ;"
+               "       bpl     1b                              ;"
+               "       aesd    v0.16b, v2.16b                  ;"
+               "       eor     v0.16b, v0.16b, v3.16b          ;"
+               "       st1     {v0.16b}, %[out]                ;"
+
+       :       [out]           "=Q"(*out),
+               [key]           "=r"(dummy0),
+               [rounds]        "=r"(dummy1)
+       :       [in]            "Q"(*in),
+                               "1"(ctx->key_dec),
+                               "2"(num_rounds(ctx) - 2)
+       :       "cc");
+
+       kernel_neon_end();
+}
+
+static struct crypto_alg aes_alg = {
+       .cra_name               = "aes",
+       .cra_driver_name        = "aes-ce",
+       .cra_priority           = 300,
+       .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
+       .cra_blocksize          = AES_BLOCK_SIZE,
+       .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
+       .cra_module             = THIS_MODULE,
+       .cra_cipher = {
+               .cia_min_keysize        = AES_MIN_KEY_SIZE,
+               .cia_max_keysize        = AES_MAX_KEY_SIZE,
+               .cia_setkey             = crypto_aes_set_key,
+               .cia_encrypt            = aes_cipher_encrypt,
+               .cia_decrypt            = aes_cipher_decrypt
+       }
+};
+
+static int __init aes_mod_init(void)
+{
+       return crypto_register_alg(&aes_alg);
+}
+
+static void __exit aes_mod_exit(void)
+{
+       crypto_unregister_alg(&aes_alg);
+}
+
+module_cpu_feature_match(AES, aes_mod_init);
+module_exit(aes_mod_exit);