tristate "AES core cipher using ARMv8 Crypto Extensions"
depends on ARM64 && KERNEL_MODE_NEON
select CRYPTO_ALGAPI
- select CRYPTO_AES
config CRYPTO_AES_ARM64_CE_CCM
tristate "AES in CCM mode using ARMv8 Crypto Extensions"
depends on ARM64 && KERNEL_MODE_NEON
select CRYPTO_ALGAPI
- select CRYPTO_AES
+ select CRYPTO_AES_ARM64_CE
select CRYPTO_AEAD
config CRYPTO_AES_ARM64_CE_BLK
tristate "AES in ECB/CBC/CTR/XTS modes using ARMv8 Crypto Extensions"
depends on ARM64 && KERNEL_MODE_NEON
select CRYPTO_BLKCIPHER
- select CRYPTO_AES
+ select CRYPTO_AES_ARM64_CE
select CRYPTO_ABLK_HELPER
config CRYPTO_AES_ARM64_NEON_BLK
select CRYPTO_AES
select CRYPTO_ABLK_HELPER
+ config CRYPTO_CRC32_ARM64
+ tristate "CRC32 and CRC32C using optional ARMv8 instructions"
+ depends on ARM64
+ select CRYPTO_HASH
endif
#include <linux/crypto.h>
#include <linux/module.h>
+#include "aes-ce-setkey.h"
+
static int num_rounds(struct crypto_aes_ctx *ctx)
{
/*
struct crypto_aes_ctx *ctx = crypto_aead_ctx(tfm);
int ret;
- ret = crypto_aes_expand_key(ctx, in_key, key_len);
+ ret = ce_aes_expandkey(ctx, in_key, key_len);
if (!ret)
return 0;
MODULE_DESCRIPTION("Synchronous AES in CCM mode using ARMv8 Crypto Extensions");
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");
- MODULE_ALIAS("ccm(aes)");
+ MODULE_ALIAS_CRYPTO("ccm(aes)");
#include <linux/module.h>
#include <linux/cpufeature.h>
+#include "aes-ce-setkey.h"
+
#ifdef USE_V8_CRYPTO_EXTENSIONS
#define MODE "ce"
#define PRIO 300
+#define aes_setkey ce_aes_setkey
+#define aes_expandkey ce_aes_expandkey
#define aes_ecb_encrypt ce_aes_ecb_encrypt
#define aes_ecb_decrypt ce_aes_ecb_decrypt
#define aes_cbc_encrypt ce_aes_cbc_encrypt
#else
#define MODE "neon"
#define PRIO 200
+#define aes_setkey crypto_aes_set_key
+#define aes_expandkey crypto_aes_expand_key
#define aes_ecb_encrypt neon_aes_ecb_encrypt
#define aes_ecb_decrypt neon_aes_ecb_decrypt
#define aes_cbc_encrypt neon_aes_cbc_encrypt
#define aes_xts_encrypt neon_aes_xts_encrypt
#define aes_xts_decrypt neon_aes_xts_decrypt
MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON");
- MODULE_ALIAS("ecb(aes)");
- MODULE_ALIAS("cbc(aes)");
- MODULE_ALIAS("ctr(aes)");
- MODULE_ALIAS("xts(aes)");
+ MODULE_ALIAS_CRYPTO("ecb(aes)");
+ MODULE_ALIAS_CRYPTO("cbc(aes)");
+ MODULE_ALIAS_CRYPTO("ctr(aes)");
+ MODULE_ALIAS_CRYPTO("xts(aes)");
#endif
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
struct crypto_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
int ret;
- ret = crypto_aes_expand_key(&ctx->key1, in_key, key_len / 2);
+ ret = aes_expandkey(&ctx->key1, in_key, key_len / 2);
if (!ret)
- ret = crypto_aes_expand_key(&ctx->key2, &in_key[key_len / 2],
- key_len / 2);
+ ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2],
+ key_len / 2);
if (!ret)
return 0;
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
- .setkey = crypto_aes_set_key,
+ .setkey = aes_setkey,
.encrypt = ecb_encrypt,
.decrypt = ecb_decrypt,
},
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
- .setkey = crypto_aes_set_key,
+ .setkey = aes_setkey,
.encrypt = cbc_encrypt,
.decrypt = cbc_decrypt,
},
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
- .setkey = crypto_aes_set_key,
+ .setkey = aes_setkey,
.encrypt = ctr_encrypt,
.decrypt = ctr_encrypt,
},
{
struct cmsghdr *cmsg;
- for (cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
+ for_each_cmsghdr(cmsg, msg) {
if (!CMSG_OK(msg, cmsg))
return -EINVAL;
if (cmsg->cmsg_level != SOL_ALG)
con->op = *(u32 *)CMSG_DATA(cmsg);
break;
+ case ALG_SET_AEAD_ASSOCLEN:
+ if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32)))
+ return -EINVAL;
+ con->aead_assoclen = *(u32 *)CMSG_DATA(cmsg);
+ break;
+
default:
return -EINVAL;
}
struct alg_sock *ask = alg_sk(sk);
struct hash_ctx *ctx = ask->private;
unsigned long iovlen;
- struct iovec *iov;
+ const struct iovec *iov;
long copied = 0;
int err;
ctx->more = 0;
- for (iov = msg->msg_iov, iovlen = msg->msg_iovlen; iovlen > 0;
+ for (iov = msg->msg_iter.iov, iovlen = msg->msg_iter.nr_segs; iovlen > 0;
iovlen--, iov++) {
unsigned long seglen = iov->iov_len;
char __user *from = iov->iov_base;
goto unlock;
}
- err = memcpy_toiovec(msg->msg_iov, ctx->result, len);
+ err = memcpy_to_msg(msg, ctx->result, len);
unlock:
release_sock(sk);
struct alg_sock *ask = alg_sk(sk);
struct hash_ctx *ctx = ask->private;
- sock_kfree_s(sk, ctx->result,
- crypto_ahash_digestsize(crypto_ahash_reqtfm(&ctx->req)));
+ sock_kzfree_s(sk, ctx->result,
+ crypto_ahash_digestsize(crypto_ahash_reqtfm(&ctx->req)));
sock_kfree_s(sk, ctx, ctx->len);
af_alg_release_parent(sk);
}
struct af_alg_control con = {};
long copied = 0;
bool enc = 0;
+ bool init = 0;
int err;
int i;
if (err)
return err;
+ init = 1;
switch (con.op) {
case ALG_OP_ENCRYPT:
enc = 1;
if (!ctx->more && ctx->used)
goto unlock;
- if (!ctx->used) {
+ if (init) {
ctx->enc = enc;
if (con.iv)
memcpy(ctx->iv, con.iv->iv, ivsize);
len = min_t(unsigned long, len,
PAGE_SIZE - sg->offset - sg->length);
- err = memcpy_fromiovec(page_address(sg_page(sg)) +
- sg->offset + sg->length,
- msg->msg_iov, len);
+ err = memcpy_from_msg(page_address(sg_page(sg)) +
+ sg->offset + sg->length,
+ msg, len);
if (err)
goto unlock;
if (!sg_page(sg + i))
goto unlock;
- err = memcpy_fromiovec(page_address(sg_page(sg + i)),
- msg->msg_iov, plen);
+ err = memcpy_from_msg(page_address(sg_page(sg + i)),
+ msg, plen);
if (err) {
__free_page(sg_page(sg + i));
sg_assign_page(sg + i, NULL);
err = 0;
ctx->more = msg->msg_flags & MSG_MORE;
- if (!ctx->more && !list_empty(&ctx->tsgl))
- sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
unlock:
skcipher_data_wakeup(sk);
done:
ctx->more = flags & MSG_MORE;
- if (!ctx->more && !list_empty(&ctx->tsgl))
- sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
unlock:
skcipher_data_wakeup(sk);
struct skcipher_sg_list *sgl;
struct scatterlist *sg;
unsigned long iovlen;
- struct iovec *iov;
+ const struct iovec *iov;
int err = -EAGAIN;
int used;
long copied = 0;
lock_sock(sk);
- for (iov = msg->msg_iov, iovlen = msg->msg_iovlen; iovlen > 0;
+ for (iov = msg->msg_iter.iov, iovlen = msg->msg_iter.nr_segs; iovlen > 0;
iovlen--, iov++) {
unsigned long seglen = iov->iov_len;
char __user *from = iov->iov_base;
while (!sg->length)
sg++;
- used = ctx->used;
- if (!used) {
+ if (!ctx->used) {
err = skcipher_wait_for_data(sk, flags);
if (err)
goto unlock;
}
- used = min_t(unsigned long, used, seglen);
+ used = min_t(unsigned long, ctx->used, seglen);
used = af_alg_make_sg(&ctx->rsgl, from, used, 1);
err = used;
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(&ctx->req);
skcipher_free_sgl(sk);
- sock_kfree_s(sk, ctx->iv, crypto_ablkcipher_ivsize(tfm));
+ sock_kzfree_s(sk, ctx->iv, crypto_ablkcipher_ivsize(tfm));
sock_kfree_s(sk, ctx, ctx->len);
af_alg_release_parent(sk);
}
#define PCI_VENDOR_ID_INTEL 0x8086
#define ADF_DH895XCC_DEVICE_NAME "dh895xcc"
#define ADF_DH895XCC_PCI_DEVICE_ID 0x435
- #define ADF_DH895XCC_PMISC_BAR 1
- #define ADF_DH895XCC_ETR_BAR 2
#define ADF_PCI_MAX_BARS 3
#define ADF_DEVICE_NAME_LENGTH 32
#define ADF_ETR_MAX_RINGS_PER_BANK 16
struct dentry *debugfs_dir;
struct list_head list;
struct module *owner;
- uint8_t accel_id;
- uint8_t numa_node;
struct adf_accel_pci accel_pci_dev;
+ uint8_t accel_id;
} __packed;
#endif
return 0;
}
- static void adf_enable_coalesc(struct adf_etr_bank_data *bank,
- const char *section, uint32_t bank_num_in_accel)
+ static void adf_get_coalesc_timer(struct adf_etr_bank_data *bank,
+ const char *section,
+ uint32_t bank_num_in_accel)
{
if (adf_get_cfg_int(bank->accel_dev, section,
ADF_ETRMGR_COALESCE_TIMER_FORMAT,
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
struct adf_etr_ring_data *ring;
struct adf_etr_ring_data *tx_ring;
- uint32_t i, coalesc_enabled;
+ uint32_t i, coalesc_enabled = 0;
memset(bank, 0, sizeof(*bank));
bank->bank_number = bank_num;
/* Enable IRQ coalescing always. This will allow to use
* the optimised flag and coalesc register.
* If it is disabled in the config file just use min time value */
- if (adf_get_cfg_int(accel_dev, "Accelerator0",
- ADF_ETRMGR_COALESCING_ENABLED_FORMAT,
- bank_num, &coalesc_enabled) && coalesc_enabled)
- adf_enable_coalesc(bank, "Accelerator0", bank_num);
+ if ((adf_get_cfg_int(accel_dev, "Accelerator0",
+ ADF_ETRMGR_COALESCING_ENABLED_FORMAT, bank_num,
+ &coalesc_enabled) == 0) && coalesc_enabled)
+ adf_get_coalesc_timer(bank, "Accelerator0", bank_num);
else
bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME;
WRITE_CSR_RING_BASE(csr_addr, bank_num, i, 0);
ring = &bank->rings[i];
if (hw_data->tx_rings_mask & (1 << i)) {
- ring->inflights = kzalloc_node(sizeof(atomic_t),
- GFP_KERNEL,
- accel_dev->numa_node);
+ ring->inflights =
+ kzalloc_node(sizeof(atomic_t),
+ GFP_KERNEL,
+ dev_to_node(&GET_DEV(accel_dev)));
if (!ring->inflights)
goto err;
} else {
int i, ret;
etr_data = kzalloc_node(sizeof(*etr_data), GFP_KERNEL,
- accel_dev->numa_node);
+ dev_to_node(&GET_DEV(accel_dev)));
if (!etr_data)
return -ENOMEM;
num_banks = GET_MAX_BANKS(accel_dev);
size = num_banks * sizeof(struct adf_etr_bank_data);
- etr_data->banks = kzalloc_node(size, GFP_KERNEL, accel_dev->numa_node);
+ etr_data->banks = kzalloc_node(size, GFP_KERNEL,
+ dev_to_node(&GET_DEV(accel_dev)));
if (!etr_data->banks) {
ret = -ENOMEM;
goto err_bank;
__be64 *hash512_state_out;
int i, offset;
- memset(auth_state.data, '\0', MAX_AUTH_STATE_SIZE + 64);
+ memzero_explicit(auth_state.data, MAX_AUTH_STATE_SIZE + 64);
shash->tfm = ctx->hash_tfm;
shash->flags = 0x0;
memcpy(ipad, buff, digest_size);
memcpy(opad, buff, digest_size);
- memset(ipad + digest_size, 0, block_size - digest_size);
- memset(opad + digest_size, 0, block_size - digest_size);
+ memzero_explicit(ipad + digest_size, block_size - digest_size);
+ memzero_explicit(opad + digest_size, block_size - digest_size);
} else {
memcpy(ipad, auth_key, auth_keylen);
memcpy(opad, auth_key, auth_keylen);
- memset(ipad + auth_keylen, 0, block_size - auth_keylen);
- memset(opad + auth_keylen, 0, block_size - auth_keylen);
+ memzero_explicit(ipad + auth_keylen, block_size - auth_keylen);
+ memzero_explicit(opad + auth_keylen, block_size - auth_keylen);
}
for (i = 0; i < block_size; i++) {
default:
return -EFAULT;
}
+ memzero_explicit(ipad, block_size);
+ memzero_explicit(opad, block_size);
return 0;
}
break;
default:
goto bad_key;
- break;
}
if (qat_alg_init_enc_session(ctx, alg, &keys))
if (ctx->enc_cd) {
/* rekeying */
dev = &GET_DEV(ctx->inst->accel_dev);
- memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
- memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
- memset(&ctx->enc_fw_req_tmpl, 0,
- sizeof(struct icp_qat_fw_la_bulk_req));
- memset(&ctx->dec_fw_req_tmpl, 0,
- sizeof(struct icp_qat_fw_la_bulk_req));
+ memzero_explicit(ctx->enc_cd, sizeof(struct qat_alg_cd));
+ memzero_explicit(ctx->dec_cd, sizeof(struct qat_alg_cd));
+ memzero_explicit(&ctx->enc_fw_req_tmpl,
+ sizeof(struct icp_qat_fw_la_bulk_req));
+ memzero_explicit(&ctx->dec_fw_req_tmpl,
+ sizeof(struct icp_qat_fw_la_bulk_req));
} else {
/* new key */
int node = get_current_node();
return 0;
out_free_all:
+ memzero_explicit(ctx->dec_cd, sizeof(struct qat_alg_cd));
dma_free_coherent(dev, sizeof(struct qat_alg_cd),
ctx->dec_cd, ctx->dec_cd_paddr);
ctx->dec_cd = NULL;
out_free_enc:
+ memzero_explicit(ctx->enc_cd, sizeof(struct qat_alg_cd));
dma_free_coherent(dev, sizeof(struct qat_alg_cd),
ctx->enc_cd, ctx->enc_cd_paddr);
ctx->enc_cd = NULL;
if (unlikely(!n))
return -EINVAL;
- bufl = kmalloc_node(sz, GFP_ATOMIC, inst->accel_dev->numa_node);
+ bufl = kmalloc_node(sz, GFP_ATOMIC,
+ dev_to_node(&GET_DEV(inst->accel_dev)));
if (unlikely(!bufl))
return -ENOMEM;
goto err;
for_each_sg(assoc, sg, assoc_n, i) {
+ if (!sg->length)
+ continue;
bufl->bufers[bufs].addr = dma_map_single(dev,
sg_virt(sg),
sg->length,
struct qat_alg_buf *bufers;
buflout = kmalloc_node(sz, GFP_ATOMIC,
- inst->accel_dev->numa_node);
+ dev_to_node(&GET_DEV(inst->accel_dev)));
if (unlikely(!buflout))
goto err;
bloutp = dma_map_single(dev, buflout, sz, DMA_TO_DEVICE);
{
struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
- memset(ctx, '\0', sizeof(*ctx));
+ memzero_explicit(ctx, sizeof(*ctx));
ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
if (IS_ERR(ctx->hash_tfm))
return -EFAULT;
return;
dev = &GET_DEV(inst->accel_dev);
- if (ctx->enc_cd)
+ if (ctx->enc_cd) {
+ memzero_explicit(ctx->enc_cd, sizeof(struct qat_alg_cd));
dma_free_coherent(dev, sizeof(struct qat_alg_cd),
ctx->enc_cd, ctx->enc_cd_paddr);
- if (ctx->dec_cd)
+ }
+ if (ctx->dec_cd) {
+ memzero_explicit(ctx->dec_cd, sizeof(struct qat_alg_cd));
dma_free_coherent(dev, sizeof(struct qat_alg_cd),
ctx->dec_cd, ctx->dec_cd_paddr);
+ }
qat_crypto_put_instance(inst);
}
uint32_t msix_num_entries = hw_data->num_banks + 1;
entries = kzalloc_node(msix_num_entries * sizeof(*entries),
- GFP_KERNEL, accel_dev->numa_node);
+ GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev)));
if (!entries)
return -ENOMEM;
accel_dev->accel_pci_dev.msix_entries.names = names;
return 0;
err:
- for (i = 0; i < msix_num_entries; i++) {
- if (*(names + i))
- kfree(*(names + i));
- }
+ for (i = 0; i < msix_num_entries; i++)
+ kfree(*(names + i));
kfree(entries);
kfree(names);
return -ENOMEM;
int i;
kfree(accel_dev->accel_pci_dev.msix_entries.entries);
- for (i = 0; i < msix_num_entries; i++) {
- if (*(names + i))
- kfree(*(names + i));
- }
+ for (i = 0; i < msix_num_entries; i++)
+ kfree(*(names + i));
kfree(names);
}
#include <linux/security.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
+#include <linux/page_counter.h>
#include <linux/memcontrol.h>
-#include <linux/res_counter.h>
#include <linux/static_key.h>
#include <linux/aio.h>
#include <linux/sched.h>
* @sk_rcvtimeo: %SO_RCVTIMEO setting
* @sk_sndtimeo: %SO_SNDTIMEO setting
* @sk_rxhash: flow hash received from netif layer
+ * @sk_incoming_cpu: record cpu processing incoming packets
* @sk_txhash: computed flow hash for use on transmit
* @sk_filter: socket filtering instructions
* @sk_protinfo: private area, net family specific, when not using slab
#ifdef CONFIG_RPS
__u32 sk_rxhash;
#endif
+ u16 sk_incoming_cpu;
+ /* 16bit hole
+ * Warned : sk_incoming_cpu can be set from softirq,
+ * Do not use this hole without fully understanding possible issues.
+ */
+
__u32 sk_txhash;
#ifdef CONFIG_NET_RX_BUSY_POLL
unsigned int sk_napi_id;
return sk->sk_backlog_rcv(sk, skb);
}
+static inline void sk_incoming_cpu_update(struct sock *sk)
+{
+ sk->sk_incoming_cpu = raw_smp_processor_id();
+}
+
static inline void sock_rps_record_flow_hash(__u32 hash)
{
#ifdef CONFIG_RPS
if (!__rc) { \
*(__timeo) = schedule_timeout(*(__timeo)); \
} \
+ sched_annotate_sleep(); \
lock_sock(__sk); \
__rc = __condition; \
__rc; \
};
struct cg_proto {
- struct res_counter memory_allocated; /* Current allocated memory. */
+ struct page_counter memory_allocated; /* Current allocated memory. */
struct percpu_counter sockets_allocated; /* Current number of sockets. */
int memory_pressure;
long sysctl_mem[3];
unsigned long amt,
int *parent_status)
{
- struct res_counter *fail;
- int ret;
+ page_counter_charge(&prot->memory_allocated, amt);
- ret = res_counter_charge_nofail(&prot->memory_allocated,
- amt << PAGE_SHIFT, &fail);
- if (ret < 0)
+ if (page_counter_read(&prot->memory_allocated) >
+ prot->memory_allocated.limit)
*parent_status = OVER_LIMIT;
}
static inline void memcg_memory_allocated_sub(struct cg_proto *prot,
unsigned long amt)
{
- res_counter_uncharge(&prot->memory_allocated, amt << PAGE_SHIFT);
-}
-
-static inline u64 memcg_memory_allocated_read(struct cg_proto *prot)
-{
- u64 ret;
- ret = res_counter_read_u64(&prot->memory_allocated, RES_USAGE);
- return ret >> PAGE_SHIFT;
+ page_counter_uncharge(&prot->memory_allocated, amt);
}
static inline long
sk_memory_allocated(const struct sock *sk)
{
struct proto *prot = sk->sk_prot;
+
if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
- return memcg_memory_allocated_read(sk->sk_cgrp);
+ return page_counter_read(&sk->sk_cgrp->memory_allocated);
return atomic_long_read(prot->memory_allocated);
}
memcg_memory_allocated_add(sk->sk_cgrp, amt, parent_status);
/* update the root cgroup regardless */
atomic_long_add_return(amt, prot->memory_allocated);
- return memcg_memory_allocated_read(sk->sk_cgrp);
+ return page_counter_read(&sk->sk_cgrp->memory_allocated);
}
return atomic_long_add_return(amt, prot->memory_allocated);
int *errcode, int max_page_order);
void *sock_kmalloc(struct sock *sk, int size, gfp_t priority);
void sock_kfree_s(struct sock *sk, void *mem, int size);
+ void sock_kzfree_s(struct sock *sk, void *mem, int size);
void sk_send_sigurg(struct sock *sk);
/*
return 0;
}
-static inline int skb_copy_to_page(struct sock *sk, char __user *from,
- struct sk_buff *skb, struct page *page,
- int off, int copy)
-{
- if (skb->ip_summed == CHECKSUM_NONE) {
- int err = 0;
- __wsum csum = csum_and_copy_from_user(from,
- page_address(page) + off,
- copy, 0, &err);
- if (err)
- return err;
- skb->csum = csum_block_add(skb->csum, csum, skb->len);
- } else if (copy_from_user(page_address(page) + off, from, copy))
- return -EFAULT;
-
- skb->len += copy;
- skb->data_len += copy;
- skb->truesize += copy;
- sk->sk_wmem_queued += copy;
- sk_mem_charge(sk, copy);
- return 0;
-}
-
/**
* sk_wmem_alloc_get - returns write allocations
* @sk: socket
bool sk_capable(const struct sock *sk, int cap);
bool sk_net_capable(const struct sock *sk, int cap);
-/*
- * Enable debug/info messages
- */
-extern int net_msg_warn;
-#define NETDEBUG(fmt, args...) \
- do { if (net_msg_warn) printk(fmt,##args); } while (0)
-
-#define LIMIT_NETDEBUG(fmt, args...) \
- do { if (net_msg_warn && net_ratelimit()) printk(fmt,##args); } while(0)
-
extern __u32 sysctl_wmem_max;
extern __u32 sysctl_rmem_max;
}
break;
+ case SO_ATTACH_BPF:
+ ret = -EINVAL;
+ if (optlen == sizeof(u32)) {
+ u32 ufd;
+
+ ret = -EFAULT;
+ if (copy_from_user(&ufd, optval, sizeof(ufd)))
+ break;
+
+ ret = sk_attach_bpf(ufd, sk);
+ }
+ break;
+
case SO_DETACH_FILTER:
ret = sk_detach_filter(sk);
break;
v.val = sk->sk_max_pacing_rate;
break;
+ case SO_INCOMING_CPU:
+ v.val = sk->sk_incoming_cpu;
+ break;
+
default:
return -ENOPROTOOPT;
}
newsk->sk_err = 0;
newsk->sk_priority = 0;
+ newsk->sk_incoming_cpu = raw_smp_processor_id();
/*
* Before updating sk_refcnt, we must commit prior changes to memory
* (Documentation/RCU/rculist_nulls.txt for details)
}
EXPORT_SYMBOL(sock_kmalloc);
- /*
- * Free an option memory block.
+ /* Free an option memory block. Note, we actually want the inline
+ * here as this allows gcc to detect the nullify and fold away the
+ * condition entirely.
*/
- void sock_kfree_s(struct sock *sk, void *mem, int size)
+ static inline void __sock_kfree_s(struct sock *sk, void *mem, int size,
+ const bool nullify)
{
if (WARN_ON_ONCE(!mem))
return;
- kfree(mem);
+ if (nullify)
+ kzfree(mem);
+ else
+ kfree(mem);
atomic_sub(size, &sk->sk_omem_alloc);
}
+
+ void sock_kfree_s(struct sock *sk, void *mem, int size)
+ {
+ __sock_kfree_s(sk, mem, size, false);
+ }
EXPORT_SYMBOL(sock_kfree_s);
+ void sock_kzfree_s(struct sock *sk, void *mem, int size)
+ {
+ __sock_kfree_s(sk, mem, size, true);
+ }
+ EXPORT_SYMBOL(sock_kzfree_s);
+
/* It is almost wait_for_tcp_memory minus release_sock/lock_sock.
I think, these locks should be removed for datagram sockets.
*/
msg->msg_flags |= MSG_TRUNC;
copied = len;
}
- err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
+ err = skb_copy_datagram_msg(skb, 0, msg, copied);
if (err)
goto out_free_skb;
projects / cascardo / linux.git / commitdiff