summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--backport-include/crypto/skcipher.h99
-rw-r--r--backport-include/linux/kernel.h4
-rw-r--r--backport-include/linux/kref.h13
-rw-r--r--backport-include/linux/skbuff.h17
-rw-r--r--backport-include/linux/string.h4
-rw-r--r--backport-include/linux/uio.h7
-rw-r--r--backport-include/net/sock.h10
-rw-r--r--backport-include/uapi/linux/cryptouser.h12
-rw-r--r--compat/Kconfig12
-rw-r--r--compat/Makefile3
-rw-r--r--compat/backport-4.0.c33
-rw-r--r--compat/backport-4.10.c37
-rw-r--r--compat/backport-4.4.c11
-rw-r--r--compat/crypto-kpp.c128
-rw-r--r--compat/crypto-skcipher.c35
-rw-r--r--compat/lib-iov_iter.c1473
-rw-r--r--include/crypto/backport-kpp.h350
-rw-r--r--include/crypto/internal/backport-kpp.h64
-rw-r--r--net/bluetooth/Kconfig3
-rw-r--r--net/bluetooth/a2mp.c4
-rw-r--r--net/bluetooth/af_bluetooth.c3
-rw-r--r--net/bluetooth/ecdh_helper.c4
-rw-r--r--net/bluetooth/l2cap_core.c38
-rw-r--r--net/bluetooth/l2cap_sock.c14
-rw-r--r--net/bluetooth/rfcomm/core.c4
-rw-r--r--net/bluetooth/rfcomm/sock.c6
-rw-r--r--net/bluetooth/sco.c6
27 files changed, 2383 insertions, 11 deletions
diff --git a/backport-include/crypto/skcipher.h b/backport-include/crypto/skcipher.h
index 73ba783..900c359 100644
--- a/backport-include/crypto/skcipher.h
+++ b/backport-include/crypto/skcipher.h
@@ -441,6 +441,105 @@ static inline void skcipher_request_set_crypt(
}
#endif /* < 4.3 */
+#if LINUX_VERSION_IS_LESS(4,8,0)
+#define skcipher_walk LINUX_BACKPORT(skcipher_walk)
+struct skcipher_walk {
+ union {
+ struct {
+ struct page *page;
+ unsigned long offset;
+ } phys;
+
+ struct {
+ u8 *page;
+ void *addr;
+ } virt;
+ } src, dst;
+
+ struct scatter_walk in;
+ unsigned int nbytes;
+
+ struct scatter_walk out;
+ unsigned int total;
+
+ struct list_head buffers;
+
+ u8 *page;
+ u8 *buffer;
+ u8 *oiv;
+ void *iv;
+
+ unsigned int ivsize;
+
+ int flags;
+ unsigned int blocksize;
+ unsigned int stride;
+ unsigned int alignmask;
+};
+
+struct skcipher_alg {
+ int (*setkey)(struct crypto_skcipher *tfm, const u8 *key,
+ unsigned int keylen);
+ int (*encrypt)(struct skcipher_request *req);
+ int (*decrypt)(struct skcipher_request *req);
+ int (*init)(struct crypto_skcipher *tfm);
+ void (*exit)(struct crypto_skcipher *tfm);
+
+ unsigned int min_keysize;
+ unsigned int max_keysize;
+ unsigned int ivsize;
+ unsigned int chunksize;
+ unsigned int walksize;
+
+ struct crypto_alg base;
+};
+
+struct skcipher_instance {
+ void (*free)(struct skcipher_instance *inst);
+ union {
+ struct {
+ char head[offsetof(struct skcipher_alg, base)];
+ struct crypto_instance base;
+ } s;
+ struct skcipher_alg alg;
+ };
+};
+
+static inline unsigned int crypto_skcipher_alg_walksize(
+ struct skcipher_alg *alg)
+{
+ if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) ==
+ CRYPTO_ALG_TYPE_BLKCIPHER)
+ return alg->base.cra_blocksize;
+
+ if (alg->base.cra_ablkcipher.encrypt)
+ return alg->base.cra_blocksize;
+
+ return alg->walksize;
+}
+
+static inline struct skcipher_alg *crypto_skcipher_alg(
+ struct crypto_skcipher *tfm)
+{
+ return container_of(crypto_skcipher_tfm(tfm)->__crt_alg,
+ struct skcipher_alg, base);
+}
+
+static inline unsigned int crypto_skcipher_walksize(
+ struct crypto_skcipher *tfm)
+{
+ return crypto_skcipher_alg_walksize(crypto_skcipher_alg(tfm));
+}
+
+static inline struct crypto_instance *skcipher_crypto_instance(
+ struct skcipher_instance *inst)
+{
+ return &inst->s.base;
+}
+
+#define CRYPTO_ALG_TYPE_SKCIPHER CRYPTO_ALG_TYPE_ABLKCIPHER
+#endif /* < 4.8 */
+
#if LINUX_VERSION_IS_LESS(4,6,0)
#define skcipher_request_zero LINUX_BACKPORT(skcipher_request_zero)
static inline void skcipher_request_zero(struct skcipher_request *req)
diff --git a/backport-include/linux/kernel.h b/backport-include/linux/kernel.h
index 3ddeb13..4b2f153 100644
--- a/backport-include/linux/kernel.h
+++ b/backport-include/linux/kernel.h
@@ -206,6 +206,10 @@ int __must_check kstrtobool_from_user(const char __user *s, size_t count, bool *
#endif
+#if LINUX_VERSION_IS_LESS(4,5,0)
+const char *kvasprintf_const(gfp_t gfp, const char *fmt, va_list args);
+#endif /* < 4.4 */
+
#if LINUX_VERSION_IS_LESS(3,14,0)
static inline u32 reciprocal_scale(u32 val, u32 ep_ro)
{
diff --git a/backport-include/linux/kref.h b/backport-include/linux/kref.h
new file mode 100644
index 0000000..631488f
--- /dev/null
+++ b/backport-include/linux/kref.h
@@ -0,0 +1,13 @@
+#ifndef __BACKPORT_LINUX_KREF_H
+#define __BACKPORT_LINUX_KREF_H
+#include_next <linux/kref.h>
+
+#if LINUX_VERSION_IS_LESS(4,11,0)
+#include <linux/refcount.h>
+static inline unsigned int kref_read(const struct kref *kref)
+{
+ return refcount_read((const refcount_t *)&kref->refcount);
+}
+#endif /* < 4.11 */
+
+#endif /* __BACKPORT_LINUX_KREF_H */
diff --git a/backport-include/linux/skbuff.h b/backport-include/linux/skbuff.h
index 034206b..af40b8d 100644
--- a/backport-include/linux/skbuff.h
+++ b/backport-include/linux/skbuff.h
@@ -371,6 +371,23 @@ static inline void skb_put_u8(struct sk_buff *skb, u8 val)
{
*(u8 *)skb_put(skb, 1) = val;
}
+
+static inline void *__skb_put_zero(struct sk_buff *skb, unsigned int len)
+{
+ void *tmp = __skb_put(skb, len);
+
+ memset(tmp, 0, len);
+ return tmp;
+}
+
+static inline void *__skb_put_data(struct sk_buff *skb, const void *data,
+ unsigned int len)
+{
+ void *tmp = __skb_put(skb, len);
+
+ memcpy(tmp, data, len);
+ return tmp;
+}
#endif
#endif /* __BACKPORT_SKBUFF_H */
diff --git a/backport-include/linux/string.h b/backport-include/linux/string.h
index b85d9c7..b1d26d1 100644
--- a/backport-include/linux/string.h
+++ b/backport-include/linux/string.h
@@ -29,4 +29,8 @@ void memzero_explicit(void *s, size_t count);
ssize_t strscpy(char *dest, const char *src, size_t count);
#endif
+#if LINUX_VERSION_IS_LESS(4,0,0)
+extern void kfree_const(const void *x);
+extern const char *kstrdup_const(const char *s, gfp_t gfp);
+#endif
#endif /* __BACKPORT_LINUX_STRING_H */
diff --git a/backport-include/linux/uio.h b/backport-include/linux/uio.h
new file mode 100644
index 0000000..fe27e68
--- /dev/null
+++ b/backport-include/linux/uio.h
@@ -0,0 +1,7 @@
+#ifndef _BP_LINUX_UIO_H
+#define _BP_LINUX_UIO_H
+#include_next <linux/uio.h>
+
+bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i);
+
+#endif /* _BP_LINUX_UIO_H */
diff --git a/backport-include/net/sock.h b/backport-include/net/sock.h
index 89191f3..ea3c474 100644
--- a/backport-include/net/sock.h
+++ b/backport-include/net/sock.h
@@ -63,4 +63,14 @@ static inline void sk_clear_bit(int nr, struct sock *sk)
}
#endif /* < 4.5 */
+#if LINUX_VERSION_IS_LESS(4,7,0)
+/* no reclassification while locks are held */
+static inline bool sock_allow_reclassification(const struct sock *csk)
+{
+ struct sock *sk = (struct sock *)csk;
+
+ return !sk->sk_lock.owned && !spin_is_locked(&sk->sk_lock.slock);
+}
+#endif /* < 4.7 */
+
#endif /* __BACKPORT_NET_SOCK_H */
diff --git a/backport-include/uapi/linux/cryptouser.h b/backport-include/uapi/linux/cryptouser.h
new file mode 100644
index 0000000..ea4a9a5
--- /dev/null
+++ b/backport-include/uapi/linux/cryptouser.h
@@ -0,0 +1,12 @@
+#ifndef __BACKPORT_LINUX_CRYPTOUSER_H
+#define __BACKPORT_LINUX_CRYPTOUSER_H
+#include_next <linux/cryptouser.h>
+#include <linux/version.h>
+
+#if LINUX_VERSION_IS_LESS(4,8,0)
+struct crypto_report_kpp {
+ char type[CRYPTO_MAX_NAME];
+};
+#endif /* < 4.8 */
+
+#endif
diff --git a/compat/Kconfig b/compat/Kconfig
index 6f972e2..9b41564 100644
--- a/compat/Kconfig
+++ b/compat/Kconfig
@@ -113,6 +113,12 @@ config BPAUTO_CRYPTO_SKCIPHER
#c-file crypto/skcipher.c
#module-name skcipher
+config BPAUTO_CRYPTO_KPP2
+ tristate
+ depends on m
+ depends on KERNEL_4_7
+ default y if BACKPORTED_BT
+
config BPAUTO_WANT_DEV_COREDUMP
bool
@@ -168,6 +174,12 @@ config BPAUTO_REFCOUNT
#h-file linux/refcount.h
#c-file lib/refcount.c
+config BPAUTO_IOV_ITER
+ bool
+ default y
+ depends on !KERNEL_4_6
+ #c-file lib/iov_iter.c
+
config BPAUTO_SYSTEM_DATA_VERIFICATION
bool
diff --git a/compat/Makefile b/compat/Makefile
index 4857d10..0352754 100644
--- a/compat/Makefile
+++ b/compat/Makefile
@@ -69,6 +69,8 @@ $(obj)/oid_registry_data.c: $(src)/../include/linux/backport-oid_registry.h \
quiet_cmd_build_OID_registry = GEN $@
cmd_build_OID_registry = perl $(src)/build_OID_registry $< $@
compat-$(CPTCFG_BPAUTO_ASN1_DECODER) += lib-asn1_decoder.o
+kpp-objs += crypto-kpp.o
+obj-$(CPTCFG_BPAUTO_CRYPTO_KPP2) += kpp.o
skcipher-objs += crypto-skcipher.o
obj-$(CPTCFG_BPAUTO_CRYPTO_SKCIPHER) += skcipher.o
compat-$(CPTCFG_BPAUTO_RHASHTABLE) += lib-rhashtable.o
@@ -78,3 +80,4 @@ compat-$(CPTCFG_BPAUTO_PUBLIC_KEY) +=
compat-$(CPTCFG_BPAUTO_PKCS7) +=
compat-$(CPTCFG_BPAUTO_BUILD_WANT_DEV_COREDUMP) += drivers-base-devcoredump.o
compat-$(CPTCFG_BPAUTO_REFCOUNT) += lib-refcount.o
+compat-$(CPTCFG_BPAUTO_IOV_ITER) += lib-iov_iter.o
diff --git a/compat/backport-4.0.c b/compat/backport-4.0.c
index eb95082..7f86c69 100644
--- a/compat/backport-4.0.c
+++ b/compat/backport-4.0.c
@@ -17,6 +17,7 @@
#include <linux/trace_seq.h>
#include <linux/ftrace_event.h>
#include <asm/unaligned.h>
+#include <linux/slab.h>
static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
struct mm_struct *mm,
@@ -375,3 +376,35 @@ ftrace_print_array_seq(struct trace_seq *p, const void *buf, int buf_len,
return ret;
}
EXPORT_SYMBOL(ftrace_print_array_seq);
+
+/**
+ * kfree_const - conditionally free memory
+ * @x: pointer to the memory
+ *
+ * Function calls kfree only if @x is not in .rodata section.
+ */
+void kfree_const(const void *x)
+{
+ /* if (!is_kernel_rodata((unsigned long)x)) */
+ kfree(x);
+}
+EXPORT_SYMBOL(kfree_const);
+
+/*
+* kstrdup_const - conditionally duplicate an existing const string
+* @s: the string to duplicate
+* @gfp: the GFP mask used in the kmalloc() call when allocating memory
+*
+* Function returns source string if it is in .rodata section otherwise it
+* fallbacks to kstrdup.
+* Strings allocated by kstrdup_const should be freed by kfree_const.
+*/
+
+const char *kstrdup_const(const char *s, gfp_t gfp)
+{
+/* if (is_kernel_rodata((unsigned long)s))
+ * return s;
+ */
+ return kstrdup(s, gfp);
+}
+EXPORT_SYMBOL(kstrdup_const);
diff --git a/compat/backport-4.10.c b/compat/backport-4.10.c
index 44e02dd..97d152b 100644
--- a/compat/backport-4.10.c
+++ b/compat/backport-4.10.c
@@ -251,4 +251,41 @@ int mii_ethtool_get_link_ksettings(struct mii_if_info *mii,
return 0;
}
EXPORT_SYMBOL(mii_ethtool_get_link_ksettings);
+
+void *kvmalloc_node(size_t size, gfp_t flags, int node)
+{
+ gfp_t kmalloc_flags = flags;
+ void *ret;
+
+ /*
+ * vmalloc uses GFP_KERNEL for some internal allocations (e.g page tables)
+ * so the given set of flags has to be compatible.
+ */
+ WARN_ON_ONCE((flags & GFP_KERNEL) != GFP_KERNEL);
+
+ /*
+ * We want to attempt a large physically contiguous block first because
+ * it is less likely to fragment multiple larger blocks and therefore
+ * contribute to a long term fragmentation less than vmalloc fallback.
+ * However make sure that larger requests are not too disruptive - no
+ * OOM killer and no allocation failure warnings as we have a fallback.
+ */
+ if (size > PAGE_SIZE) {
+ kmalloc_flags |= __GFP_NOWARN;
+ kmalloc_flags |= __GFP_NORETRY;
+ }
+
+ ret = kmalloc_node(size, kmalloc_flags, node);
+
+ /*
+ * It doesn't really make sense to fallback to vmalloc for sub page
+ * requests
+ */
+ if (ret || size <= PAGE_SIZE)
+ return ret;
+
+ return vmalloc(size);
+}
+EXPORT_SYMBOL(kvmalloc_node);
+
#endif /* LINUX_VERSION_IS_GEQ(4,6,0) */
diff --git a/compat/backport-4.4.c b/compat/backport-4.4.c
index 1c11a20..660d50b 100644
--- a/compat/backport-4.4.c
+++ b/compat/backport-4.4.c
@@ -152,3 +152,14 @@ void tso_start(struct sk_buff *skb, struct tso_t *tso)
}
}
EXPORT_SYMBOL(tso_start);
+
+const char *kvasprintf_const(gfp_t gfp, const char *fmt, va_list ap)
+{
+ if (!strchr(fmt, '%'))
+ return kstrdup_const(fmt, gfp);
+ if (!strcmp(fmt, "%s"))
+ return kstrdup_const(va_arg(ap, const char*), gfp);
+ return kvasprintf(gfp, fmt, ap);
+}
+EXPORT_SYMBOL(kvasprintf_const);
+
diff --git a/compat/crypto-kpp.c b/compat/crypto-kpp.c
new file mode 100644
index 0000000..ae11fa8
--- /dev/null
+++ b/compat/crypto-kpp.c
@@ -0,0 +1,128 @@
+/*
+ * Key-agreement Protocol Primitives (KPP)
+ *
+ * Copyright (c) 2016, Intel Corporation
+ * Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/crypto.h>
+#include <crypto/algapi.h>
+#include <linux/cryptouser.h>
+#include <linux/compiler.h>
+#include <net/netlink.h>
+#include <crypto/backport-kpp.h>
+#include <crypto/internal/backport-kpp.h>
+
+extern unsigned int crypto_alg_extsize(struct crypto_alg *alg);
+void *crypto_alloc_tfm(const char *alg_name,
+ const struct crypto_type *frontend, u32 type, u32 mask);
+
+#define CRYPTOCFGA_REPORT_KPP __CRYPTOCFGA_MAX
+#ifdef CONFIG_NET
+static int crypto_kpp_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+ struct crypto_report_kpp rkpp;
+
+ strncpy(rkpp.type, "kpp", sizeof(rkpp.type));
+
+ if (nla_put(skb, CRYPTOCFGA_REPORT_KPP,
+ sizeof(struct crypto_report_kpp), &rkpp))
+ goto nla_put_failure;
+ return 0;
+
+nla_put_failure:
+ return -EMSGSIZE;
+}
+#else
+static int crypto_kpp_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+ return -ENOSYS;
+}
+#endif
+
+static void crypto_kpp_show(struct seq_file *m, struct crypto_alg *alg)
+ __maybe_unused;
+
+static void crypto_kpp_show(struct seq_file *m, struct crypto_alg *alg)
+{
+ seq_puts(m, "type : kpp\n");
+}
+
+static void crypto_kpp_exit_tfm(struct crypto_tfm *tfm)
+{
+ struct crypto_kpp *kpp = __crypto_kpp_tfm(tfm);
+ struct kpp_alg *alg = crypto_kpp_alg(kpp);
+
+ alg->exit(kpp);
+}
+
+static int crypto_kpp_init_tfm(struct crypto_tfm *tfm)
+{
+ struct crypto_kpp *kpp = __crypto_kpp_tfm(tfm);
+ struct kpp_alg *alg = crypto_kpp_alg(kpp);
+
+ if (alg->exit)
+ kpp->base.exit = crypto_kpp_exit_tfm;
+
+ if (alg->init)
+ return alg->init(kpp);
+
+ return 0;
+}
+
+static const struct crypto_type crypto_kpp_type = {
+ .extsize = crypto_alg_extsize,
+ .init_tfm = crypto_kpp_init_tfm,
+#ifdef CONFIG_PROC_FS
+ .show = crypto_kpp_show,
+#endif
+ .report = crypto_kpp_report,
+ .maskclear = ~CRYPTO_ALG_TYPE_MASK,
+ .maskset = CRYPTO_ALG_TYPE_MASK,
+ .type = CRYPTO_ALG_TYPE_KPP,
+ .tfmsize = offsetof(struct crypto_kpp, base),
+};
+
+struct crypto_kpp *crypto_alloc_kpp(const char *alg_name, u32 type, u32 mask)
+{
+ return crypto_alloc_tfm(alg_name, &crypto_kpp_type, type, mask);
+}
+EXPORT_SYMBOL_GPL(crypto_alloc_kpp);
+
+static void kpp_prepare_alg(struct kpp_alg *alg)
+{
+ struct crypto_alg *base = &alg->base;
+
+ base->cra_type = &crypto_kpp_type;
+ base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
+ base->cra_flags |= CRYPTO_ALG_TYPE_KPP;
+}
+
+int crypto_register_kpp(struct kpp_alg *alg)
+{
+ struct crypto_alg *base = &alg->base;
+
+ kpp_prepare_alg(alg);
+ return crypto_register_alg(base);
+}
+EXPORT_SYMBOL_GPL(crypto_register_kpp);
+
+void crypto_unregister_kpp(struct kpp_alg *alg)
+{
+ crypto_unregister_alg(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_unregister_kpp);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Key-agreement Protocol Primitives");
diff --git a/compat/crypto-skcipher.c b/compat/crypto-skcipher.c
index 05b520d..3c53735 100644
--- a/compat/crypto-skcipher.c
+++ b/compat/crypto-skcipher.c
@@ -48,7 +48,7 @@ static inline u32 skcipher_request_flags(struct skcipher_request *req)
{
return req->base.flags;
}
-
+#if LINUX_VERSION_IS_GEQ(4,8,0)
enum {
SKCIPHER_WALK_PHYS = 1 << 0,
SKCIPHER_WALK_SLOW = 1 << 1,
@@ -593,6 +593,35 @@ int skcipher_walk_aead_decrypt(struct skcipher_walk *walk,
}
EXPORT_SYMBOL_GPL(skcipher_walk_aead_decrypt);
+#endif /* LINUX_VERSION_IS_GEQ(4,8,0) */
+
+#if LINUX_VERSION_IS_LESS(4,2,0)
+
+unsigned int crypto_alg_extsize(struct crypto_alg *alg)
+{
+ return alg->cra_ctxsize +
+ (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1));
+}
+EXPORT_SYMBOL_GPL(crypto_alg_extsize);
+
+int crypto_type_has_alg(const char *name, const struct crypto_type *frontend,
+ u32 type, u32 mask)
+{
+#if 0
+ int ret = 0;
+ struct crypto_alg *alg = crypto_find_alg(name, frontend, type, mask);
+
+ if (!IS_ERR(alg)) {
+ crypto_mod_put(alg);
+ ret = 1;
+ }
+#endif
+ return -EINVAL;
+}
+EXPORT_SYMBOL_GPL(crypto_type_has_alg);
+
+#endif
+
static unsigned int crypto_skcipher_extsize(struct crypto_alg *alg)
{
if (alg->cra_type == &crypto_blkcipher_type)
@@ -922,7 +951,9 @@ static int crypto_skcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
static const struct crypto_type crypto_skcipher_type2 = {
.extsize = crypto_skcipher_extsize,
.init_tfm = crypto_skcipher_init_tfm,
+#if LINUX_VERSION_IS_GEQ(4,3,0)
.free = crypto_skcipher_free_instance,
+#endif
#ifdef CONFIG_PROC_FS
.show = crypto_skcipher_show,
#endif
@@ -936,8 +967,10 @@ static const struct crypto_type crypto_skcipher_type2 = {
int crypto_grab_skcipher(struct crypto_skcipher_spawn *spawn,
const char *name, u32 type, u32 mask)
{
+#if 0
spawn->base.frontend = &crypto_skcipher_type2;
return crypto_grab_spawn(&spawn->base, name, type, mask);
+#endif
}
EXPORT_SYMBOL_GPL(crypto_grab_skcipher);
diff --git a/compat/lib-iov_iter.c b/compat/lib-iov_iter.c
new file mode 100644
index 0000000..070e5f0
--- /dev/null
+++ b/compat/lib-iov_iter.c
@@ -0,0 +1,1473 @@
+#include <linux/export.h>
+#include <linux/bvec.h>
+#include <linux/uio.h>
+#include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/splice.h>
+#include <net/checksum.h>
+
+#define PIPE_PARANOIA /* for now */
+
+#define iterate_iovec(i, n, __v, __p, skip, STEP) { \
+ size_t left; \
+ size_t wanted = n; \
+ __p = i->iov; \
+ __v.iov_len = min(n, __p->iov_len - skip); \
+ if (likely(__v.iov_len)) { \
+ __v.iov_base = __p->iov_base + skip; \
+ left = (STEP); \
+ __v.iov_len -= left; \
+ skip += __v.iov_len; \
+ n -= __v.iov_len; \
+ } else { \
+ left = 0; \
+ } \
+ while (unlikely(!left && n)) { \
+ __p++; \
+ __v.iov_len = min(n, __p->iov_len); \
+ if (unlikely(!__v.iov_len)) \
+ continue; \
+ __v.iov_base = __p->iov_base; \
+ left = (STEP); \
+ __v.iov_len -= left; \
+ skip = __v.iov_len; \
+ n -= __v.iov_len; \
+ } \
+ n = wanted - n; \
+}
+
+#define iterate_kvec(i, n, __v, __p, skip, STEP) { \
+ size_t wanted = n; \
+ __p = i->kvec; \
+ __v.iov_len = min(n, __p->iov_len - skip); \
+ if (likely(__v.iov_len)) { \
+ __v.iov_base = __p->iov_base + skip; \
+ (void)(STEP); \
+ skip += __v.iov_len; \
+ n -= __v.iov_len; \
+ } \
+ while (unlikely(n)) { \
+ __p++; \
+ __v.iov_len = min(n, __p->iov_len); \
+ if (unlikely(!__v.iov_len)) \
+ continue; \
+ __v.iov_base = __p->iov_base; \
+ (void)(STEP); \
+ skip = __v.iov_len; \
+ n -= __v.iov_len; \
+ } \
+ n = wanted; \
+}
+
+#define iterate_bvec(i, n, __v, __bi, skip, STEP) { \
+ struct bvec_iter __start; \
+ __start.bi_size = n; \
+ __start.bi_bvec_done = skip; \
+ __start.bi_idx = 0; \
+ for_each_bvec(__v, i->bvec, __bi, __start) { \
+ if (!__v.bv_len) \
+ continue; \
+ (void)(STEP); \
+ } \
+}
+
+#define iterate_all_kinds(i, n, v, I, B, K) { \
+ if (likely(n)) { \
+ size_t skip = i->iov_offset; \
+ if (unlikely(i->type & ITER_BVEC)) { \
+ struct bio_vec v; \
+ struct bvec_iter __bi; \
+ iterate_bvec(i, n, v, __bi, skip, (B)) \
+ } else if (unlikely(i->type & ITER_KVEC)) { \
+ const struct kvec *kvec; \
+ struct kvec v; \
+ iterate_kvec(i, n, v, kvec, skip, (K)) \
+ } else { \
+ const struct iovec *iov; \
+ struct iovec v; \
+ iterate_iovec(i, n, v, iov, skip, (I)) \
+ } \
+ } \
+}
+
+#define iterate_and_advance(i, n, v, I, B, K) { \
+ if (unlikely(i->count < n)) \
+ n = i->count; \
+ if (i->count) { \
+ size_t skip = i->iov_offset; \
+ if (unlikely(i->type & ITER_BVEC)) { \
+ const struct bio_vec *bvec = i->bvec; \
+ struct bio_vec v; \
+ struct bvec_iter __bi; \
+ iterate_bvec(i, n, v, __bi, skip, (B)) \
+ i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
+ i->nr_segs -= i->bvec - bvec; \
+ skip = __bi.bi_bvec_done; \
+ } else if (unlikely(i->type & ITER_KVEC)) { \
+ const struct kvec *kvec; \
+ struct kvec v; \
+ iterate_kvec(i, n, v, kvec, skip, (K)) \
+ if (skip == kvec->iov_len) { \
+ kvec++; \
+ skip = 0; \
+ } \
+ i->nr_segs -= kvec - i->kvec; \
+ i->kvec = kvec; \
+ } else { \
+ const struct iovec *iov; \
+ struct iovec v; \
+ iterate_iovec(i, n, v, iov, skip, (I)) \
+ if (skip == iov->iov_len) { \
+ iov++; \
+ skip = 0; \
+ } \
+ i->nr_segs -= iov - i->iov; \
+ i->iov = iov; \
+ } \
+ i->count -= n; \
+ i->iov_offset = skip; \
+ } \
+}
+
+static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
+ struct iov_iter *i)
+{
+ size_t skip, copy, left, wanted;
+ const struct iovec *iov;
+ char __user *buf;
+ void *kaddr, *from;
+
+ if (unlikely(bytes > i->count))
+ bytes = i->count;
+
+ if (unlikely(!bytes))
+ return 0;
+
+ might_fault();
+ wanted = bytes;
+ iov = i->iov;
+ skip = i->iov_offset;
+ buf = iov->iov_base + skip;
+ copy = min(bytes, iov->iov_len - skip);
+
+ if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
+ kaddr = kmap_atomic(page);
+ from = kaddr + offset;
+
+ /* first chunk, usually the only one */
+ left = __copy_to_user_inatomic(buf, from, copy);
+ copy -= left;
+ skip += copy;
+ from += copy;
+ bytes -= copy;
+
+ while (unlikely(!left && bytes)) {
+ iov++;
+ buf = iov->iov_base;
+ copy = min(bytes, iov->iov_len);
+ left = __copy_to_user_inatomic(buf, from, copy);
+ copy -= left;
+ skip = copy;
+ from += copy;
+ bytes -= copy;
+ }
+ if (likely(!bytes)) {
+ kunmap_atomic(kaddr);
+ goto done;
+ }
+ offset = from - kaddr;
+ buf += copy;
+ kunmap_atomic(kaddr);
+ copy = min(bytes, iov->iov_len - skip);
+ }
+ /* Too bad - revert to non-atomic kmap */
+
+ kaddr = kmap(page);
+ from = kaddr + offset;
+ left = __copy_to_user(buf, from, copy);
+ copy -= left;
+ skip += copy;
+ from += copy;
+ bytes -= copy;
+ while (unlikely(!left && bytes)) {
+ iov++;
+ buf = iov->iov_base;
+ copy = min(bytes, iov->iov_len);
+ left = __copy_to_user(buf, from, copy);
+ copy -= left;
+ skip = copy;
+ from += copy;
+ bytes -= copy;
+ }
+ kunmap(page);
+
+done:
+ if (skip == iov->iov_len) {
+ iov++;
+ skip = 0;
+ }
+ i->count -= wanted - bytes;
+ i->nr_segs -= iov - i->iov;
+ i->iov = iov;
+ i->iov_offset = skip;
+ return wanted - bytes;
+}
+
+static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
+ struct iov_iter *i)
+{
+ size_t skip, copy, left, wanted;
+ const struct iovec *iov;
+ char __user *buf;
+ void *kaddr, *to;
+
+ if (unlikely(bytes > i->count))
+ bytes = i->count;
+
+ if (unlikely(!bytes))
+ return 0;
+
+ might_fault();
+ wanted = bytes;
+ iov = i->iov;
+ skip = i->iov_offset;
+ buf = iov->iov_base + skip;
+ copy = min(bytes, iov->iov_len - skip);
+
+ if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
+ kaddr = kmap_atomic(page);
+ to = kaddr + offset;
+
+ /* first chunk, usually the only one */
+ left = __copy_from_user_inatomic(to, buf, copy);
+ copy -= left;
+ skip += copy;
+ to += copy;
+ bytes -= copy;
+
+ while (unlikely(!left && bytes)) {
+ iov++;
+ buf = iov->iov_base;
+ copy = min(bytes, iov->iov_len);
+ left = __copy_from_user_inatomic(to, buf, copy);
+ copy -= left;
+ skip = copy;
+ to += copy;
+ bytes -= copy;
+ }
+ if (likely(!bytes)) {
+ kunmap_atomic(kaddr);
+ goto done;
+ }
+ offset = to - kaddr;
+ buf += copy;
+ kunmap_atomic(kaddr);
+ copy = min(bytes, iov->iov_len - skip);
+ }
+ /* Too bad - revert to non-atomic kmap */
+
+ kaddr = kmap(page);
+ to = kaddr + offset;
+ left = __copy_from_user(to, buf, copy);
+ copy -= left;
+ skip += copy;
+ to += copy;
+ bytes -= copy;
+ while (unlikely(!left && bytes)) {
+ iov++;
+ buf = iov->iov_base;
+ copy = min(bytes, iov->iov_len);
+ left = __copy_from_user(to, buf, copy);
+ copy -= left;
+ skip = copy;
+ to += copy;
+ bytes -= copy;
+ }
+ kunmap(page);
+
+done:
+ if (skip == iov->iov_len) {
+ iov++;
+ skip = 0;
+ }
+ i->count -= wanted - bytes;
+ i->nr_segs -= iov - i->iov;
+ i->iov = iov;
+ i->iov_offset = skip;
+ return wanted - bytes;
+}
+
+#ifdef PIPE_PARANOIA
+static bool sanity(const struct iov_iter *i)
+{
+ struct pipe_inode_info *pipe = i->pipe;
+ int idx = i->idx;
+ int next = pipe->curbuf + pipe->nrbufs;
+ if (i->iov_offset) {
+ struct pipe_buffer *p;
+ if (unlikely(!pipe->nrbufs))
+ goto Bad; // pipe must be non-empty
+ if (unlikely(idx != ((next - 1) & (pipe->buffers - 1))))
+ goto Bad; // must be at the last buffer...
+
+ p = &pipe->bufs[idx];
+ if (unlikely(p->offset + p->len != i->iov_offset))
+ goto Bad; // ... at the end of segment
+ } else {
+ if (idx != (next & (pipe->buffers - 1)))
+ goto Bad; // must be right after the last buffer
+ }
+ return true;
+Bad:
+ printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset);
+ printk(KERN_ERR "curbuf = %d, nrbufs = %d, buffers = %d\n",
+ pipe->curbuf, pipe->nrbufs, pipe->buffers);
+ for (idx = 0; idx < pipe->buffers; idx++)
+ printk(KERN_ERR "[%p %p %d %d]\n",
+ pipe->bufs[idx].ops,
+ pipe->bufs[idx].page,
+ pipe->bufs[idx].offset,
+ pipe->bufs[idx].len);
+ WARN_ON(1);
+ return false;
+}
+#else
+#define sanity(i) true
+#endif
+
+static inline int next_idx(int idx, struct pipe_inode_info *pipe)
+{
+ return (idx + 1) & (pipe->buffers - 1);
+}
+
+static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
+ struct iov_iter *i)
+{
+ struct pipe_inode_info *pipe = i->pipe;
+ struct pipe_buffer *buf;
+ size_t off;
+ int idx;
+
+ if (unlikely(bytes > i->count))
+ bytes = i->count;
+
+ if (unlikely(!bytes))
+ return 0;
+
+ if (!sanity(i))
+ return 0;
+
+ off = i->iov_offset;
+ idx = i->idx;
+ buf = &pipe->bufs[idx];
+ if (off) {
+ if (offset == off && buf->page == page) {
+ /* merge with the last one */
+ buf->len += bytes;
+ i->iov_offset += bytes;
+ goto out;
+ }
+ idx = next_idx(idx, pipe);
+ buf = &pipe->bufs[idx];
+ }
+ if (idx == pipe->curbuf && pipe->nrbufs)
+ return 0;
+ pipe->nrbufs++;
+ buf->ops = &page_cache_pipe_buf_ops;
+ get_page(buf->page = page);
+ buf->offset = offset;
+ buf->len = bytes;
+ i->iov_offset = offset + bytes;
+ i->idx = idx;
+out:
+ i->count -= bytes;
+ return bytes;
+}
+
+#if LINUX_VERSION_IS_LESS(4,7,0)
+/*
+ * Fault in one or more iovecs of the given iov_iter, to a maximum length of
+ * bytes. For each iovec, fault in each page that constitutes the iovec.
+ *
+ * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
+ * because it is an invalid address).
+ */
+int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
+{
+ size_t skip = i->iov_offset;
+ const struct iovec *iov;
+ int err;
+ struct iovec v;
+
+ if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
+ iterate_iovec(i, bytes, v, iov, skip, ({
+ err = fault_in_pages_readable(v.iov_base, v.iov_len);
+ if (unlikely(err))
+ return err;
+ 0;}))
+ }
+ return 0;
+}
+EXPORT_SYMBOL(iov_iter_fault_in_readable);
+
+#define uaccess_kernel() segment_eq(get_fs(), KERNEL_DS)
+
+void iov_iter_init(struct iov_iter *i, int direction,
+ const struct iovec *iov, unsigned long nr_segs,
+ size_t count)
+{
+ /* It will get better. Eventually... */
+ if (uaccess_kernel()) {
+ direction |= ITER_KVEC;
+ i->type = direction;
+ i->kvec = (struct kvec *)iov;
+ } else {
+ i->type = direction;
+ i->iov = iov;
+ }
+ i->nr_segs = nr_segs;
+ i->iov_offset = 0;
+ i->count = count;
+}
+EXPORT_SYMBOL(iov_iter_init);
+#endif
+
+static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
+{
+ char *from = kmap_atomic(page);
+ memcpy(to, from + offset, len);
+ kunmap_atomic(from);
+}
+
+static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
+{
+ char *to = kmap_atomic(page);
+ memcpy(to + offset, from, len);
+ kunmap_atomic(to);
+}
+
+static void memzero_page(struct page *page, size_t offset, size_t len)
+{
+ char *addr = kmap_atomic(page);
+ memset(addr + offset, 0, len);
+ kunmap_atomic(addr);
+}
+
+#if 0
+static inline bool allocated(struct pipe_buffer *buf)
+{
+ return buf->ops == &default_pipe_buf_ops;
+}
+
+static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp)
+{
+ size_t off = i->iov_offset;
+ int idx = i->idx;
+ if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) {
+ idx = next_idx(idx, i->pipe);
+ off = 0;
+ }
+ *idxp = idx;
+ *offp = off;
+}
+
+static size_t push_pipe(struct iov_iter *i, size_t size,
+ int *idxp, size_t *offp)
+{
+ struct pipe_inode_info *pipe = i->pipe;
+ size_t off;
+ int idx;
+ ssize_t left;
+
+ if (unlikely(size > i->count))
+ size = i->count;
+ if (unlikely(!size))
+ return 0;
+
+ left = size;
+ data_start(i, &idx, &off);
+ *idxp = idx;
+ *offp = off;
+ if (off) {
+ left -= PAGE_SIZE - off;
+ if (left <= 0) {
+ pipe->bufs[idx].len += size;
+ return size;
+ }
+ pipe->bufs[idx].len = PAGE_SIZE;
+ idx = next_idx(idx, pipe);
+ }
+ while (idx != pipe->curbuf || !pipe->nrbufs) {
+ struct page *page = alloc_page(GFP_USER);
+ if (!page)
+ break;
+ pipe->nrbufs++;
+ pipe->bufs[idx].ops = &default_pipe_buf_ops;
+ pipe->bufs[idx].page = page;
+ pipe->bufs[idx].offset = 0;
+ if (left <= PAGE_SIZE) {
+ pipe->bufs[idx].len = left;
+ return size;
+ }
+ pipe->bufs[idx].len = PAGE_SIZE;
+ left -= PAGE_SIZE;
+ idx = next_idx(idx, pipe);
+ }
+ return size - left;
+}
+
+static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
+ struct iov_iter *i)
+{
+ struct pipe_inode_info *pipe = i->pipe;
+ size_t n, off;
+ int idx;
+
+ if (!sanity(i))
+ return 0;
+
+ bytes = n = push_pipe(i, bytes, &idx, &off);
+ if (unlikely(!n))
+ return 0;
+ for ( ; n; idx = next_idx(idx, pipe), off = 0) {
+ size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
+ memcpy_to_page(pipe->bufs[idx].page, off, addr, chunk);
+ i->idx = idx;
+ i->iov_offset = off + chunk;
+ n -= chunk;
+ addr += chunk;
+ }
+ i->count -= bytes;
+ return bytes;
+}
+#endif
+
+size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
+{
+ const char *from = addr;
+#if 0
+ if (unlikely(i->type & ITER_PIPE))
+ return copy_pipe_to_iter(addr, bytes, i);
+#endif
+ if (iter_is_iovec(i))
+ might_fault();
+ iterate_and_advance(i, bytes, v,
+ __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len,
+ v.iov_len),
+ memcpy_to_page(v.bv_page, v.bv_offset,
+ (from += v.bv_len) - v.bv_len, v.bv_len),
+ memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
+ )
+
+ return bytes;
+}
+EXPORT_SYMBOL(_copy_to_iter);
+
+size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
+{
+ char *to = addr;
+ if (unlikely(i->type & ITER_PIPE)) {
+ WARN_ON(1);
+ return 0;
+ }
+ if (iter_is_iovec(i))
+ might_fault();
+ iterate_and_advance(i, bytes, v,
+ __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base,
+ v.iov_len),
+ memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
+ v.bv_offset, v.bv_len),
+ memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
+ )
+
+ return bytes;
+}
+EXPORT_SYMBOL(_copy_from_iter);
+
+bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
+{
+ char *to = addr;
+ if (unlikely(i->type & ITER_PIPE)) {
+ WARN_ON(1);
+ return false;
+ }
+ if (unlikely(i->count < bytes))
+ return false;
+
+ if (iter_is_iovec(i))
+ might_fault();
+ iterate_all_kinds(i, bytes, v, ({
+ if (__copy_from_user((to += v.iov_len) - v.iov_len,
+ v.iov_base, v.iov_len))
+ return false;
+ 0;}),
+ memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
+ v.bv_offset, v.bv_len),
+ memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
+ )
+
+ iov_iter_advance(i, bytes);
+ return true;
+}
+EXPORT_SYMBOL(_copy_from_iter_full);
+
+size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
+{
+ char *to = addr;
+ if (unlikely(i->type & ITER_PIPE)) {
+ WARN_ON(1);
+ return 0;
+ }
+ iterate_and_advance(i, bytes, v,
+ __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
+ v.iov_base, v.iov_len),
+ memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
+ v.bv_offset, v.bv_len),
+ memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
+ )
+
+ return bytes;
+}
+EXPORT_SYMBOL(_copy_from_iter_nocache);
+
+#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
+size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
+{
+ char *to = addr;
+ if (unlikely(i->type & ITER_PIPE)) {
+ WARN_ON(1);
+ return 0;
+ }
+ iterate_and_advance(i, bytes, v,
+ __copy_from_user_flushcache((to += v.iov_len) - v.iov_len,
+ v.iov_base, v.iov_len),
+ memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
+ v.bv_offset, v.bv_len),
+ memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base,
+ v.iov_len)
+ )
+
+ return bytes;
+}
+EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache);
+#endif
+
+bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
+{
+ char *to = addr;
+ if (unlikely(i->type & ITER_PIPE)) {
+ WARN_ON(1);
+ return false;
+ }
+ if (unlikely(i->count < bytes))
+ return false;
+ iterate_all_kinds(i, bytes, v, ({
+ if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
+ v.iov_base, v.iov_len))
+ return false;
+ 0;}),
+ memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
+ v.bv_offset, v.bv_len),
+ memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
+ )
+
+ iov_iter_advance(i, bytes);
+ return true;
+}
+EXPORT_SYMBOL(_copy_from_iter_full_nocache);
+
+static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
+{
+ struct page *head = compound_head(page);
+ size_t v = n + offset + page_address(page) - page_address(head);
+
+ if (likely(n <= v && v <= (PAGE_SIZE << compound_order(head))))
+ return true;
+ WARN_ON(1);
+ return false;
+}
+
+#if LINUX_VERSION_IS_LESS(4,7,0)
+size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
+ struct iov_iter *i)
+{
+ if (unlikely(!page_copy_sane(page, offset, bytes)))
+ return 0;
+ if (i->type & (ITER_BVEC|ITER_KVEC)) {
+ void *kaddr = kmap_atomic(page);
+ size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
+ kunmap_atomic(kaddr);
+ return wanted;
+ } else if (likely(!(i->type & ITER_PIPE)))
+ return copy_page_to_iter_iovec(page, offset, bytes, i);
+ else
+ return copy_page_to_iter_pipe(page, offset, bytes, i);
+}
+EXPORT_SYMBOL(copy_page_to_iter);
+
+size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
+ struct iov_iter *i)
+{
+ if (unlikely(!page_copy_sane(page, offset, bytes)))
+ return 0;
+ if (unlikely(i->type & ITER_PIPE)) {
+ WARN_ON(1);
+ return 0;
+ }
+ if (i->type & (ITER_BVEC|ITER_KVEC)) {
+ void *kaddr = kmap_atomic(page);
+ size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);
+ kunmap_atomic(kaddr);
+ return wanted;
+ } else
+ return copy_page_from_iter_iovec(page, offset, bytes, i);
+}
+EXPORT_SYMBOL(copy_page_from_iter);
+#endif
+
+#if 0
+static size_t pipe_zero(size_t bytes, struct iov_iter *i)
+{
+ struct pipe_inode_info *pipe = i->pipe;
+ size_t n, off;
+ int idx;
+
+ if (!sanity(i))
+ return 0;
+
+ bytes = n = push_pipe(i, bytes, &idx, &off);
+ if (unlikely(!n))
+ return 0;
+
+ for ( ; n; idx = next_idx(idx, pipe), off = 0) {
+ size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
+ memzero_page(pipe->bufs[idx].page, off, chunk);
+ i->idx = idx;
+ i->iov_offset = off + chunk;
+ n -= chunk;
+ }
+ i->count -= bytes;
+ return bytes;
+}
+#endif
+
+#if LINUX_VERSION_IS_LESS(4,7,0)
+size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
+{
+ if (unlikely(i->type & ITER_PIPE))
+ return pipe_zero(bytes, i);
+ iterate_and_advance(i, bytes, v,
+ clear_user(v.iov_base, v.iov_len),
+ memzero_page(v.bv_page, v.bv_offset, v.bv_len),
+ memset(v.iov_base, 0, v.iov_len)
+ )
+
+ return bytes;
+}
+EXPORT_SYMBOL(iov_iter_zero);
+
+size_t iov_iter_copy_from_user_atomic(struct page *page,
+ struct iov_iter *i, unsigned long offset, size_t bytes)
+{
+ char *kaddr = kmap_atomic(page), *p = kaddr + offset;
+ if (unlikely(!page_copy_sane(page, offset, bytes))) {
+ kunmap_atomic(kaddr);
+ return 0;
+ }
+ if (unlikely(i->type & ITER_PIPE)) {
+ kunmap_atomic(kaddr);
+ WARN_ON(1);
+ return 0;
+ }
+ iterate_all_kinds(i, bytes, v,
+ __copy_from_user_inatomic((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
+ memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
+ v.bv_offset, v.bv_len),
+ memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
+ )
+ kunmap_atomic(kaddr);
+ return bytes;
+}
+EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
+#endif
+
+static inline void pipe_truncate(struct iov_iter *i)
+{
+ struct pipe_inode_info *pipe = i->pipe;
+ if (pipe->nrbufs) {
+ size_t off = i->iov_offset;
+ int idx = i->idx;
+ int nrbufs = (idx - pipe->curbuf) & (pipe->buffers - 1);
+ if (off) {
+ pipe->bufs[idx].len = off - pipe->bufs[idx].offset;
+ idx = next_idx(idx, pipe);
+ nrbufs++;
+ }
+ while (pipe->nrbufs > nrbufs) {
+ pipe_buf_release(pipe, &pipe->bufs[idx]);
+ idx = next_idx(idx, pipe);
+ pipe->nrbufs--;
+ }
+ }
+}
+
+static void pipe_advance(struct iov_iter *i, size_t size)
+{
+ struct pipe_inode_info *pipe = i->pipe;
+ if (unlikely(i->count < size))
+ size = i->count;
+ if (size) {
+ struct pipe_buffer *buf;
+ size_t off = i->iov_offset, left = size;
+ int idx = i->idx;
+ if (off) /* make it relative to the beginning of buffer */
+ left += off - pipe->bufs[idx].offset;
+ while (1) {
+ buf = &pipe->bufs[idx];
+ if (left <= buf->len)
+ break;
+ left -= buf->len;
+ idx = next_idx(idx, pipe);
+ }
+ i->idx = idx;
+ i->iov_offset = buf->offset + left;
+ }
+ i->count -= size;
+ /* ... and discard everything past that point */
+ pipe_truncate(i);
+}
+
+#if LINUX_VERSION_IS_LESS(4,7,0)
+void iov_iter_advance(struct iov_iter *i, size_t size)
+{
+ if (unlikely(i->type & ITER_PIPE)) {
+ pipe_advance(i, size);
+ return;
+ }
+ iterate_and_advance(i, size, v, 0, 0, 0)
+}
+EXPORT_SYMBOL(iov_iter_advance);
+
+void iov_iter_revert(struct iov_iter *i, size_t unroll)
+{
+ if (!unroll)
+ return;
+ if (WARN_ON(unroll > MAX_RW_COUNT))
+ return;
+ i->count += unroll;
+ if (unlikely(i->type & ITER_PIPE)) {
+ struct pipe_inode_info *pipe = i->pipe;
+ int idx = i->idx;
+ size_t off = i->iov_offset;
+ while (1) {
+ size_t n = off - pipe->bufs[idx].offset;
+ if (unroll < n) {
+ off -= unroll;
+ break;
+ }
+ unroll -= n;
+ if (!unroll && idx == i->start_idx) {
+ off = 0;
+ break;
+ }
+ if (!idx--)
+ idx = pipe->buffers - 1;
+ off = pipe->bufs[idx].offset + pipe->bufs[idx].len;
+ }
+ i->iov_offset = off;
+ i->idx = idx;
+ pipe_truncate(i);
+ return;
+ }
+ if (unroll <= i->iov_offset) {
+ i->iov_offset -= unroll;
+ return;
+ }
+ unroll -= i->iov_offset;
+ if (i->type & ITER_BVEC) {
+ const struct bio_vec *bvec = i->bvec;
+ while (1) {
+ size_t n = (--bvec)->bv_len;
+ i->nr_segs++;
+ if (unroll <= n) {
+ i->bvec = bvec;
+ i->iov_offset = n - unroll;
+ return;
+ }
+ unroll -= n;
+ }
+ } else { /* same logics for iovec and kvec */
+ const struct iovec *iov = i->iov;
+ while (1) {
+ size_t n = (--iov)->iov_len;
+ i->nr_segs++;
+ if (unroll <= n) {
+ i->iov = iov;
+ i->iov_offset = n - unroll;
+ return;
+ }
+ unroll -= n;
+ }
+ }
+}
+EXPORT_SYMBOL(iov_iter_revert);
+
+/*
+ * Return the count of just the current iov_iter segment.
+ */
+size_t iov_iter_single_seg_count(const struct iov_iter *i)
+{
+ if (unlikely(i->type & ITER_PIPE))
+ return i->count; // it is a silly place, anyway
+ if (i->nr_segs == 1)
+ return i->count;
+ else if (i->type & ITER_BVEC)
+ return min(i->count, i->bvec->bv_len - i->iov_offset);
+ else
+ return min(i->count, i->iov->iov_len - i->iov_offset);
+}
+EXPORT_SYMBOL(iov_iter_single_seg_count);
+
+void iov_iter_kvec(struct iov_iter *i, int direction,
+ const struct kvec *kvec, unsigned long nr_segs,
+ size_t count)
+{
+ BUG_ON(!(direction & ITER_KVEC));
+ i->type = direction;
+ i->kvec = kvec;
+ i->nr_segs = nr_segs;
+ i->iov_offset = 0;
+ i->count = count;
+}
+EXPORT_SYMBOL(iov_iter_kvec);
+
+void iov_iter_bvec(struct iov_iter *i, int direction,
+ const struct bio_vec *bvec, unsigned long nr_segs,
+ size_t count)
+{
+ BUG_ON(!(direction & ITER_BVEC));
+ i->type = direction;
+ i->bvec = bvec;
+ i->nr_segs = nr_segs;
+ i->iov_offset = 0;
+ i->count = count;
+}
+EXPORT_SYMBOL(iov_iter_bvec);
+
+void iov_iter_pipe(struct iov_iter *i, int direction,
+ struct pipe_inode_info *pipe,
+ size_t count)
+{
+ BUG_ON(direction != ITER_PIPE);
+ WARN_ON(pipe->nrbufs == pipe->buffers);
+ i->type = direction;
+ i->pipe = pipe;
+ i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
+ i->iov_offset = 0;
+ i->count = count;
+ i->start_idx = i->idx;
+}
+EXPORT_SYMBOL(iov_iter_pipe);
+
+unsigned long iov_iter_alignment(const struct iov_iter *i)
+{
+ unsigned long res = 0;
+ size_t size = i->count;
+
+ if (unlikely(i->type & ITER_PIPE)) {
+ if (size && i->iov_offset && allocated(&i->pipe->bufs[i->idx]))
+ return size | i->iov_offset;
+ return size;
+ }
+ iterate_all_kinds(i, size, v,
+ (res |= (unsigned long)v.iov_base | v.iov_len, 0),
+ res |= v.bv_offset | v.bv_len,
+ res |= (unsigned long)v.iov_base | v.iov_len
+ )
+ return res;
+}
+EXPORT_SYMBOL(iov_iter_alignment);
+
+unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
+{
+ unsigned long res = 0;
+ size_t size = i->count;
+
+ if (unlikely(i->type & ITER_PIPE)) {
+ WARN_ON(1);
+ return ~0U;
+ }
+
+ iterate_all_kinds(i, size, v,
+ (res |= (!res ? 0 : (unsigned long)v.iov_base) |
+ (size != v.iov_len ? size : 0), 0),
+ (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
+ (size != v.bv_len ? size : 0)),
+ (res |= (!res ? 0 : (unsigned long)v.iov_base) |
+ (size != v.iov_len ? size : 0))
+ );
+ return res;
+}
+EXPORT_SYMBOL(iov_iter_gap_alignment);
+#endif
+
+#if 0
+static inline size_t __pipe_get_pages(struct iov_iter *i,
+ size_t maxsize,
+ struct page **pages,
+ int idx,
+ size_t *start)
+{
+ struct pipe_inode_info *pipe = i->pipe;
+ ssize_t n = push_pipe(i, maxsize, &idx, start);
+ if (!n)
+ return -EFAULT;
+
+ maxsize = n;
+ n += *start;
+ while (n > 0) {
+ get_page(*pages++ = pipe->bufs[idx].page);
+ idx = next_idx(idx, pipe);
+ n -= PAGE_SIZE;
+ }
+
+ return maxsize;
+}
+
+static ssize_t pipe_get_pages(struct iov_iter *i,
+ struct page **pages, size_t maxsize, unsigned maxpages,
+ size_t *start)
+{
+ unsigned npages;
+ size_t capacity;
+ int idx;
+
+ if (!maxsize)
+ return 0;
+
+ if (!sanity(i))
+ return -EFAULT;
+
+ data_start(i, &idx, start);
+ /* some of this one + all after this one */
+ npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
+ capacity = min(npages,maxpages) * PAGE_SIZE - *start;
+
+ return __pipe_get_pages(i, min(maxsize, capacity), pages, idx, start);
+}
+#endif
+#if LINUX_VERSION_IS_LESS(4,7,0)
+ssize_t iov_iter_get_pages(struct iov_iter *i,
+ struct page **pages, size_t maxsize, unsigned maxpages,
+ size_t *start)
+{
+ if (maxsize > i->count)
+ maxsize = i->count;
+
+ if (unlikely(i->type & ITER_PIPE))
+ return pipe_get_pages(i, pages, maxsize, maxpages, start);
+ iterate_all_kinds(i, maxsize, v, ({
+ unsigned long addr = (unsigned long)v.iov_base;
+ size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
+ int n;
+ int res;
+
+ if (len > maxpages * PAGE_SIZE)
+ len = maxpages * PAGE_SIZE;
+ addr &= ~(PAGE_SIZE - 1);
+ n = DIV_ROUND_UP(len, PAGE_SIZE);
+ res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
+ if (unlikely(res < 0))
+ return res;
+ return (res == n ? len : res * PAGE_SIZE) - *start;
+ 0;}),({
+ /* can't be more than PAGE_SIZE */
+ *start = v.bv_offset;
+ get_page(*pages = v.bv_page);
+ return v.bv_len;
+ }),({
+ return -EFAULT;
+ })
+ )
+ return 0;
+}
+EXPORT_SYMBOL(iov_iter_get_pages);
+#endif
+
+extern void *kvmalloc_node(size_t size, gfp_t flags, int node);
+static inline void *kvmalloc(size_t size, gfp_t flags)
+{
+ return kvmalloc_node(size, flags, NUMA_NO_NODE);
+}
+
+static inline void *kvmalloc_array(size_t n, size_t size, gfp_t flags)
+{
+ if (size != 0 && n > SIZE_MAX / size)
+ return NULL;
+
+ return kvmalloc(n * size, flags);
+}
+
+static struct page **get_pages_array(size_t n)
+{
+ return kvmalloc_array(n, sizeof(struct page *), GFP_KERNEL);
+}
+
+#if 0
+static ssize_t pipe_get_pages_alloc(struct iov_iter *i,
+ struct page ***pages, size_t maxsize,
+ size_t *start)
+{
+ struct page **p;
+ size_t n;
+ int idx;
+ int npages;
+
+ if (!maxsize)
+ return 0;
+
+ if (!sanity(i))
+ return -EFAULT;
+
+ data_start(i, &idx, start);
+ /* some of this one + all after this one */
+ npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
+ n = npages * PAGE_SIZE - *start;
+ if (maxsize > n)
+ maxsize = n;
+ else
+ npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
+ p = get_pages_array(npages);
+ if (!p)
+ return -ENOMEM;
+ n = __pipe_get_pages(i, maxsize, p, idx, start);
+ if (n > 0)
+ *pages = p;
+ else
+ kvfree(p);
+ return n;
+}
+#endif
+
+#if LINUX_VERSION_IS_LESS(4,7,0)
+ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
+ struct page ***pages, size_t maxsize,
+ size_t *start)
+{
+ struct page **p;
+
+ if (maxsize > i->count)
+ maxsize = i->count;
+
+ if (unlikely(i->type & ITER_PIPE))
+ return pipe_get_pages_alloc(i, pages, maxsize, start);
+ iterate_all_kinds(i, maxsize, v, ({
+ unsigned long addr = (unsigned long)v.iov_base;
+ size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
+ int n;
+ int res;
+
+ addr &= ~(PAGE_SIZE - 1);
+ n = DIV_ROUND_UP(len, PAGE_SIZE);
+ p = get_pages_array(n);
+ if (!p)
+ return -ENOMEM;
+ res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
+ if (unlikely(res < 0)) {
+ kvfree(p);
+ return res;
+ }
+ *pages = p;
+ return (res == n ? len : res * PAGE_SIZE) - *start;
+ 0;}),({
+ /* can't be more than PAGE_SIZE */
+ *start = v.bv_offset;
+ *pages = p = get_pages_array(1);
+ if (!p)
+ return -ENOMEM;
+ get_page(*p = v.bv_page);
+ return v.bv_len;
+ }),({
+ return -EFAULT;
+ })
+ )
+ return 0;
+}
+EXPORT_SYMBOL(iov_iter_get_pages_alloc);
+
+size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
+ struct iov_iter *i)
+{
+ char *to = addr;
+ __wsum sum, next;
+ size_t off = 0;
+ sum = *csum;
+ if (unlikely(i->type & ITER_PIPE)) {
+ WARN_ON(1);
+ return 0;
+ }
+ iterate_and_advance(i, bytes, v, ({
+ int err = 0;
+ next = csum_and_copy_from_user(v.iov_base,
+ (to += v.iov_len) - v.iov_len,
+ v.iov_len, 0, &err);
+ if (!err) {
+ sum = csum_block_add(sum, next, off);
+ off += v.iov_len;
+ }
+ err ? v.iov_len : 0;
+ }), ({
+ char *p = kmap_atomic(v.bv_page);
+ next = csum_partial_copy_nocheck(p + v.bv_offset,
+ (to += v.bv_len) - v.bv_len,
+ v.bv_len, 0);
+ kunmap_atomic(p);
+ sum = csum_block_add(sum, next, off);
+ off += v.bv_len;
+ }),({
+ next = csum_partial_copy_nocheck(v.iov_base,
+ (to += v.iov_len) - v.iov_len,
+ v.iov_len, 0);
+ sum = csum_block_add(sum, next, off);
+ off += v.iov_len;
+ })
+ )
+ *csum = sum;
+ return bytes;
+}
+EXPORT_SYMBOL(csum_and_copy_from_iter);
+#endif
+
+bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,
+ struct iov_iter *i)
+{
+ char *to = addr;
+ __wsum sum, next;
+ size_t off = 0;
+ sum = *csum;
+ if (unlikely(i->type & ITER_PIPE)) {
+ WARN_ON(1);
+ return false;
+ }
+ if (unlikely(i->count < bytes))
+ return false;
+ iterate_all_kinds(i, bytes, v, ({
+ int err = 0;
+ next = csum_and_copy_from_user(v.iov_base,
+ (to += v.iov_len) - v.iov_len,
+ v.iov_len, 0, &err);
+ if (err)
+ return false;
+ sum = csum_block_add(sum, next, off);
+ off += v.iov_len;
+ 0;
+ }), ({
+ char *p = kmap_atomic(v.bv_page);
+ next = csum_partial_copy_nocheck(p + v.bv_offset,
+ (to += v.bv_len) - v.bv_len,
+ v.bv_len, 0);
+ kunmap_atomic(p);
+ sum = csum_block_add(sum, next, off);
+ off += v.bv_len;
+ }),({
+ next = csum_partial_copy_nocheck(v.iov_base,
+ (to += v.iov_len) - v.iov_len,
+ v.iov_len, 0);
+ sum = csum_block_add(sum, next, off);
+ off += v.iov_len;
+ })
+ )
+ *csum = sum;
+ iov_iter_advance(i, bytes);
+ return true;
+}
+EXPORT_SYMBOL(csum_and_copy_from_iter_full);
+
+#if LINUX_VERSION_IS_LESS(4,7,0)
+size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
+ struct iov_iter *i)
+{
+ const char *from = addr;
+ __wsum sum, next;
+ size_t off = 0;
+ sum = *csum;
+ if (unlikely(i->type & ITER_PIPE)) {
+ WARN_ON(1); /* for now */
+ return 0;
+ }
+ iterate_and_advance(i, bytes, v, ({
+ int err = 0;
+ next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
+ v.iov_base,
+ v.iov_len, 0, &err);
+ if (!err) {
+ sum = csum_block_add(sum, next, off);
+ off += v.iov_len;
+ }
+ err ? v.iov_len : 0;
+ }), ({
+ char *p = kmap_atomic(v.bv_page);
+ next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
+ p + v.bv_offset,
+ v.bv_len, 0);
+ kunmap_atomic(p);
+ sum = csum_block_add(sum, next, off);
+ off += v.bv_len;
+ }),({
+ next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
+ v.iov_base,
+ v.iov_len, 0);
+ sum = csum_block_add(sum, next, off);
+ off += v.iov_len;
+ })
+ )
+ *csum = sum;
+ return bytes;
+}
+EXPORT_SYMBOL(csum_and_copy_to_iter);
+
+int iov_iter_npages(const struct iov_iter *i, int maxpages)
+{
+ size_t size = i->count;
+ int npages = 0;
+
+ if (!size)
+ return 0;
+
+ if (unlikely(i->type & ITER_PIPE)) {
+ struct pipe_inode_info *pipe = i->pipe;
+ size_t off;
+ int idx;
+
+ if (!sanity(i))
+ return 0;
+
+ data_start(i, &idx, &off);
+ /* some of this one + all after this one */
+ npages = ((pipe->curbuf - idx - 1) & (pipe->buffers - 1)) + 1;
+ if (npages >= maxpages)
+ return maxpages;
+ } else iterate_all_kinds(i, size, v, ({
+ unsigned long p = (unsigned long)v.iov_base;
+ npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
+ - p / PAGE_SIZE;
+ if (npages >= maxpages)
+ return maxpages;
+ 0;}),({
+ npages++;
+ if (npages >= maxpages)
+ return maxpages;
+ }),({
+ unsigned long p = (unsigned long)v.iov_base;
+ npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
+ - p / PAGE_SIZE;
+ if (npages >= maxpages)
+ return maxpages;
+ })
+ )
+ return npages;
+}
+EXPORT_SYMBOL(iov_iter_npages);
+
+const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
+{
+ *new = *old;
+ if (unlikely(new->type & ITER_PIPE)) {
+ WARN_ON(1);
+ return NULL;
+ }
+ if (new->type & ITER_BVEC)
+ return new->bvec = kmemdup(new->bvec,
+ new->nr_segs * sizeof(struct bio_vec),
+ flags);
+ else
+ /* iovec and kvec have identical layout */
+ return new->iov = kmemdup(new->iov,
+ new->nr_segs * sizeof(struct iovec),
+ flags);
+}
+EXPORT_SYMBOL(dup_iter);
+
+/**
+ * import_iovec() - Copy an array of &struct iovec from userspace
+ * into the kernel, check that it is valid, and initialize a new
+ * &struct iov_iter iterator to access it.
+ *
+ * @type: One of %READ or %WRITE.
+ * @uvector: Pointer to the userspace array.
+ * @nr_segs: Number of elements in userspace array.
+ * @fast_segs: Number of elements in @iov.
+ * @iov: (input and output parameter) Pointer to pointer to (usually small
+ * on-stack) kernel array.
+ * @i: Pointer to iterator that will be initialized on success.
+ *
+ * If the array pointed to by *@iov is large enough to hold all @nr_segs,
+ * then this function places %NULL in *@iov on return. Otherwise, a new
+ * array will be allocated and the result placed in *@iov. This means that
+ * the caller may call kfree() on *@iov regardless of whether the small
+ * on-stack array was used or not (and regardless of whether this function
+ * returns an error or not).
+ *
+ * Return: 0 on success or negative error code on error.
+ */
+int import_iovec(int type, const struct iovec __user * uvector,
+ unsigned nr_segs, unsigned fast_segs,
+ struct iovec **iov, struct iov_iter *i)
+{
+ ssize_t n;
+ struct iovec *p;
+ n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
+ *iov, &p);
+ if (n < 0) {
+ if (p != *iov)
+ kfree(p);
+ *iov = NULL;
+ return n;
+ }
+ iov_iter_init(i, type, p, nr_segs, n);
+ *iov = p == *iov ? NULL : p;
+ return 0;
+}
+EXPORT_SYMBOL(import_iovec);
+#endif
+
+#ifdef CONFIG_COMPAT
+#include <linux/compat.h>
+
+int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
+ unsigned nr_segs, unsigned fast_segs,
+ struct iovec **iov, struct iov_iter *i)
+{
+ ssize_t n;
+ struct iovec *p;
+ n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
+ *iov, &p);
+ if (n < 0) {
+ if (p != *iov)
+ kfree(p);
+ *iov = NULL;
+ return n;
+ }
+ iov_iter_init(i, type, p, nr_segs, n);
+ *iov = p == *iov ? NULL : p;
+ return 0;
+}
+#endif
+
+#if LINUX_VERSION_IS_LESS(4,7,0)
+int import_single_range(int rw, void __user *buf, size_t len,
+ struct iovec *iov, struct iov_iter *i)
+{
+ if (len > MAX_RW_COUNT)
+ len = MAX_RW_COUNT;
+ if (unlikely(!access_ok(!rw, buf, len)))
+ return -EFAULT;
+
+ iov->iov_base = buf;
+ iov->iov_len = len;
+ iov_iter_init(i, rw, iov, 1, len);
+ return 0;
+}
+EXPORT_SYMBOL(import_single_range);
+#endif
diff --git a/include/crypto/backport-kpp.h b/include/crypto/backport-kpp.h
new file mode 100644
index 0000000..4597d74
--- /dev/null
+++ b/include/crypto/backport-kpp.h
@@ -0,0 +1,350 @@
+#ifndef _BACKPORT_CRYPTO_KPP_H
+#define _BACKPORT_CRYPTO_KPP_H
+/*
+ * Key-agreement Protocol Primitives (KPP)
+ *
+ * Copyright (c) 2016, Intel Corporation
+ * Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+
+#include <linux/crypto.h>
+#define CRYPTO_ALG_TYPE_KPP 0x0000000b
+
+/**
+ * struct kpp_request
+ *
+ * @base: Common attributes for async crypto requests
+ * @src: Source data
+ * @dst: Destination data
+ * @src_len: Size of the input buffer
+ * @dst_len: Size of the output buffer. It needs to be at least
+ * as big as the expected result depending on the operation
+ * After operation it will be updated with the actual size of the
+ * result. In case of error where the dst sgl size was insufficient,
+ * it will be updated to the size required for the operation.
+ * @__ctx: Start of private context data
+ */
+struct kpp_request {
+ struct crypto_async_request base;
+ struct scatterlist *src;
+ struct scatterlist *dst;
+ unsigned int src_len;
+ unsigned int dst_len;
+ void *__ctx[] CRYPTO_MINALIGN_ATTR;
+};
+
+/**
+ * struct crypto_kpp - user-instantiated object which encapsulate
+ * algorithms and core processing logic
+ *
+ * @base: Common crypto API algorithm data structure
+ */
+struct crypto_kpp {
+ struct crypto_tfm base;
+};
+
+/**
+ * struct kpp_alg - generic key-agreement protocol primitives
+ *
+ * @set_secret: Function invokes the protocol specific function to
+ * store the secret private key along with parameters.
+ * The implementation knows how to decode the buffer
+ * @generate_public_key: Function generate the public key to be sent to the
+ * counterpart. In case of error, where output is not big
+ * enough req->dst_len will be updated to the size
+ * required
+ * @compute_shared_secret: Function compute the shared secret as defined by
+ * the algorithm. The result is given back to the user.
+ * In case of error, where output is not big enough,
+ * req->dst_len will be updated to the size required
+ * @max_size: Function returns the size of the output buffer
+ * @init: Initialize the object. This is called only once at
+ * instantiation time. In case the cryptographic hardware
+ * needs to be initialized. Software fallback should be
+ * put in place here.
+ * @exit: Undo everything @init did.
+ *
+ * @reqsize: Request context size required by algorithm
+ * implementation
+ * @base: Common crypto API algorithm data structure
+ */
+struct kpp_alg {
+ int (*set_secret)(struct crypto_kpp *tfm, const void *buffer,
+ unsigned int len);
+ int (*generate_public_key)(struct kpp_request *req);
+ int (*compute_shared_secret)(struct kpp_request *req);
+
+ unsigned int (*max_size)(struct crypto_kpp *tfm);
+
+ int (*init)(struct crypto_kpp *tfm);
+ void (*exit)(struct crypto_kpp *tfm);
+
+ unsigned int reqsize;
+ struct crypto_alg base;
+};
+
+/**
+ * DOC: Generic Key-agreement Protocol Primitives API
+ *
+ * The KPP API is used with the algorithm type
+ * CRYPTO_ALG_TYPE_KPP (listed as type "kpp" in /proc/crypto)
+ */
+
+/**
+ * crypto_alloc_kpp() - allocate KPP tfm handle
+ * @alg_name: is the name of the kpp algorithm (e.g. "dh", "ecdh")
+ * @type: specifies the type of the algorithm
+ * @mask: specifies the mask for the algorithm
+ *
+ * Allocate a handle for kpp algorithm. The returned struct crypto_kpp
+ * is required for any following API invocation
+ *
+ * Return: allocated handle in case of success; IS_ERR() is true in case of
+ * an error, PTR_ERR() returns the error code.
+ */
+struct crypto_kpp *crypto_alloc_kpp(const char *alg_name, u32 type, u32 mask);
+
+static inline struct crypto_tfm *crypto_kpp_tfm(struct crypto_kpp *tfm)
+{
+ return &tfm->base;
+}
+
+static inline struct kpp_alg *__crypto_kpp_alg(struct crypto_alg *alg)
+{
+ return container_of(alg, struct kpp_alg, base);
+}
+
+static inline struct crypto_kpp *__crypto_kpp_tfm(struct crypto_tfm *tfm)
+{
+ return container_of(tfm, struct crypto_kpp, base);
+}
+
+static inline struct kpp_alg *crypto_kpp_alg(struct crypto_kpp *tfm)
+{
+ return __crypto_kpp_alg(crypto_kpp_tfm(tfm)->__crt_alg);
+}
+
+static inline unsigned int crypto_kpp_reqsize(struct crypto_kpp *tfm)
+{
+ return crypto_kpp_alg(tfm)->reqsize;
+}
+
+static inline void kpp_request_set_tfm(struct kpp_request *req,
+ struct crypto_kpp *tfm)
+{
+ req->base.tfm = crypto_kpp_tfm(tfm);
+}
+
+static inline struct crypto_kpp *crypto_kpp_reqtfm(struct kpp_request *req)
+{
+ return __crypto_kpp_tfm(req->base.tfm);
+}
+
+static inline u32 crypto_kpp_get_flags(struct crypto_kpp *tfm)
+{
+ return crypto_tfm_get_flags(crypto_kpp_tfm(tfm));
+}
+
+static inline void crypto_kpp_set_flags(struct crypto_kpp *tfm, u32 flags)
+{
+ crypto_tfm_set_flags(crypto_kpp_tfm(tfm), flags);
+}
+
+/**
+ * crypto_free_kpp() - free KPP tfm handle
+ *
+ * @tfm: KPP tfm handle allocated with crypto_alloc_kpp()
+ */
+static inline void crypto_free_kpp(struct crypto_kpp *tfm)
+{
+ crypto_destroy_tfm(tfm, crypto_kpp_tfm(tfm));
+}
+
+/**
+ * kpp_request_alloc() - allocates kpp request
+ *
+ * @tfm: KPP tfm handle allocated with crypto_alloc_kpp()
+ * @gfp: allocation flags
+ *
+ * Return: allocated handle in case of success or NULL in case of an error.
+ */
+static inline struct kpp_request *kpp_request_alloc(struct crypto_kpp *tfm,
+ gfp_t gfp)
+{
+ struct kpp_request *req;
+
+ req = kmalloc(sizeof(*req) + crypto_kpp_reqsize(tfm), gfp);
+ if (likely(req))
+ kpp_request_set_tfm(req, tfm);
+
+ return req;
+}
+
+/**
+ * kpp_request_free() - zeroize and free kpp request
+ *
+ * @req: request to free
+ */
+static inline void kpp_request_free(struct kpp_request *req)
+{
+ kzfree(req);
+}
+
+/**
+ * kpp_request_set_callback() - Sets an asynchronous callback.
+ *
+ * Callback will be called when an asynchronous operation on a given
+ * request is finished.
+ *
+ * @req: request that the callback will be set for
+ * @flgs: specify for instance if the operation may backlog
+ * @cmpl: callback which will be called
+ * @data: private data used by the caller
+ */
+static inline void kpp_request_set_callback(struct kpp_request *req,
+ u32 flgs,
+ crypto_completion_t cmpl,
+ void *data)
+{
+ req->base.complete = cmpl;
+ req->base.data = data;
+ req->base.flags = flgs;
+}
+
+/**
+ * kpp_request_set_input() - Sets input buffer
+ *
+ * Sets parameters required by generate_public_key
+ *
+ * @req: kpp request
+ * @input: ptr to input scatter list
+ * @input_len: size of the input scatter list
+ */
+static inline void kpp_request_set_input(struct kpp_request *req,
+ struct scatterlist *input,
+ unsigned int input_len)
+{
+ req->src = input;
+ req->src_len = input_len;
+}
+
+/**
+ * kpp_request_set_output() - Sets output buffer
+ *
+ * Sets parameters required by kpp operation
+ *
+ * @req: kpp request
+ * @output: ptr to output scatter list
+ * @output_len: size of the output scatter list
+ */
+static inline void kpp_request_set_output(struct kpp_request *req,
+ struct scatterlist *output,
+ unsigned int output_len)
+{
+ req->dst = output;
+ req->dst_len = output_len;
+}
+
+enum {
+ CRYPTO_KPP_SECRET_TYPE_UNKNOWN,
+ CRYPTO_KPP_SECRET_TYPE_DH,
+ CRYPTO_KPP_SECRET_TYPE_ECDH,
+};
+
+/**
+ * struct kpp_secret - small header for packing secret buffer
+ *
+ * @type: define type of secret. Each kpp type will define its own
+ * @len: specify the len of the secret, include the header, that
+ * follows the struct
+ */
+struct kpp_secret {
+ unsigned short type;
+ unsigned short len;
+};
+
+/**
+ * crypto_kpp_set_secret() - Invoke kpp operation
+ *
+ * Function invokes the specific kpp operation for a given alg.
+ *
+ * @tfm: tfm handle
+ * @buffer: Buffer holding the packet representation of the private
+ * key. The structure of the packet key depends on the particular
+ * KPP implementation. Packing and unpacking helpers are provided
+ * for ECDH and DH (see the respective header files for those
+ * implementations).
+ * @len: Length of the packet private key buffer.
+ *
+ * Return: zero on success; error code in case of error
+ */
+static inline int crypto_kpp_set_secret(struct crypto_kpp *tfm,
+ const void *buffer, unsigned int len)
+{
+ struct kpp_alg *alg = crypto_kpp_alg(tfm);
+
+ return alg->set_secret(tfm, buffer, len);
+}
+
+/**
+ * crypto_kpp_generate_public_key() - Invoke kpp operation
+ *
+ * Function invokes the specific kpp operation for generating the public part
+ * for a given kpp algorithm.
+ *
+ * To generate a private key, the caller should use a random number generator.
+ * The output of the requested length serves as the private key.
+ *
+ * @req: kpp key request
+ *
+ * Return: zero on success; error code in case of error
+ */
+static inline int crypto_kpp_generate_public_key(struct kpp_request *req)
+{
+ struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
+ struct kpp_alg *alg = crypto_kpp_alg(tfm);
+
+ return alg->generate_public_key(req);
+}
+
+/**
+ * crypto_kpp_compute_shared_secret() - Invoke kpp operation
+ *
+ * Function invokes the specific kpp operation for computing the shared secret
+ * for a given kpp algorithm.
+ *
+ * @req: kpp key request
+ *
+ * Return: zero on success; error code in case of error
+ */
+static inline int crypto_kpp_compute_shared_secret(struct kpp_request *req)
+{
+ struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
+ struct kpp_alg *alg = crypto_kpp_alg(tfm);
+
+ return alg->compute_shared_secret(req);
+}
+
+/**
+ * crypto_kpp_maxsize() - Get len for output buffer
+ *
+ * Function returns the output buffer size required for a given key.
+ * Function assumes that the key is already set in the transformation. If this
+ * function is called without a setkey or with a failed setkey, you will end up
+ * in a NULL dereference.
+ *
+ * @tfm: KPP tfm handle allocated with crypto_alloc_kpp()
+ */
+static inline unsigned int crypto_kpp_maxsize(struct crypto_kpp *tfm)
+{
+ struct kpp_alg *alg = crypto_kpp_alg(tfm);
+
+ return alg->max_size(tfm);
+}
+#endif
diff --git a/include/crypto/internal/backport-kpp.h b/include/crypto/internal/backport-kpp.h
new file mode 100644
index 0000000..bd16788
--- /dev/null
+++ b/include/crypto/internal/backport-kpp.h
@@ -0,0 +1,64 @@
+/*
+ * Key-agreement Protocol Primitives (KPP)
+ *
+ * Copyright (c) 2016, Intel Corporation
+ * Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+#ifndef _CRYPTO_KPP_INT_H
+#define _CRYPTO_KPP_INT_H
+#include <crypto/backport-kpp.h>
+#include <crypto/algapi.h>
+
+/*
+ * Transform internal helpers.
+ */
+static inline void *kpp_request_ctx(struct kpp_request *req)
+{
+ return req->__ctx;
+}
+
+static inline void *kpp_tfm_ctx(struct crypto_kpp *tfm)
+{
+ return tfm->base.__crt_ctx;
+}
+
+static inline void kpp_request_complete(struct kpp_request *req, int err)
+{
+ req->base.complete(&req->base, err);
+}
+
+static inline const char *kpp_alg_name(struct crypto_kpp *tfm)
+{
+ return crypto_kpp_tfm(tfm)->__crt_alg->cra_name;
+}
+
+/**
+ * crypto_register_kpp() -- Register key-agreement protocol primitives algorithm
+ *
+ * Function registers an implementation of a key-agreement protocol primitive
+ * algorithm
+ *
+ * @alg: algorithm definition
+ *
+ * Return: zero on success; error code in case of error
+ */
+int crypto_register_kpp(struct kpp_alg *alg);
+
+/**
+ * crypto_unregister_kpp() -- Unregister key-agreement protocol primitive
+ * algorithm
+ *
+ * Function unregisters an implementation of a key-agreement protocol primitive
+ * algorithm
+ *
+ * @alg: algorithm definition
+ */
+void crypto_unregister_kpp(struct kpp_alg *alg);
+
+#endif
diff --git a/net/bluetooth/Kconfig b/net/bluetooth/Kconfig
index 08f5808..0f68a23 100644
--- a/net/bluetooth/Kconfig
+++ b/net/bluetooth/Kconfig
@@ -14,7 +14,8 @@ menuconfig BT
depends on CRYPTO_CMAC
depends on CRYPTO_ECB
depends on CRYPTO_SHA256
- depends on CRYPTO_ECDH
+ depends on CRYPTO_ECDH
+ depends on !KERNEL_4_7
help
Bluetooth is low-cost, low-power, short-range wireless technology.
It was designed as a replacement for cables and other short-range
diff --git a/net/bluetooth/a2mp.c b/net/bluetooth/a2mp.c
index ab3bb87..e6d3b82 100644
--- a/net/bluetooth/a2mp.c
+++ b/net/bluetooth/a2mp.c
@@ -244,7 +244,7 @@ static int a2mp_discover_rsp(struct amp_mgr *mgr, struct sk_buff *skb,
}
len -= sizeof(*cl);
- cl = skb_pull(skb, sizeof(*cl));
+ cl = (void *)skb_pull(skb, sizeof(*cl));
}
/* Fall back to L2CAP init sequence */
@@ -284,7 +284,7 @@ static int a2mp_change_notify(struct amp_mgr *mgr, struct sk_buff *skb,
while (skb->len >= sizeof(*cl)) {
BT_DBG("Controller id %d type %d status %d", cl->id, cl->type,
cl->status);
- cl = skb_pull(skb, sizeof(*cl));
+ cl = (void *) skb_pull(skb, sizeof(*cl));
}
/* TODO send A2MP_CHANGE_RSP */
diff --git a/net/bluetooth/af_bluetooth.c b/net/bluetooth/af_bluetooth.c
index 91e3ba2..1c10269 100644
--- a/net/bluetooth/af_bluetooth.c
+++ b/net/bluetooth/af_bluetooth.c
@@ -28,6 +28,7 @@
#include <linux/debugfs.h>
#include <linux/stringify.h>
#include <linux/sched/signal.h>
+#include <linux/refcount.h>
#include <asm/ioctls.h>
@@ -657,7 +658,7 @@ static int bt_seq_show(struct seq_file *seq, void *v)
seq_printf(seq,
"%pK %-6d %-6u %-6u %-6u %-6lu %-6lu",
sk,
- refcount_read(&sk->sk_refcnt),
+ refcount_read((const refcount_t *)&sk->sk_refcnt),
sk_rmem_alloc_get(sk),
sk_wmem_alloc_get(sk),
from_kuid(seq_user_ns(seq), sock_i_uid(sk)),
diff --git a/net/bluetooth/ecdh_helper.c b/net/bluetooth/ecdh_helper.c
index c7b1a9a..ce6c546 100644
--- a/net/bluetooth/ecdh_helper.c
+++ b/net/bluetooth/ecdh_helper.c
@@ -23,7 +23,11 @@
#include "ecdh_helper.h"
#include <linux/scatterlist.h>
+#if LINUX_VERSION_IS_LESS(4,8,0)
+#include <crypto/backport-kpp.h>
+#else
#include <crypto/kpp.h>
+#endif
#include <crypto/ecdh.h>
struct ecdh_completion {
diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c
index 9800324..a1a9b4c 100644
--- a/net/bluetooth/l2cap_core.c
+++ b/net/bluetooth/l2cap_core.c
@@ -33,7 +33,7 @@
#include <linux/debugfs.h>
#include <linux/crc16.h>
#include <linux/filter.h>
-
+#include <linux/uio.h>
#include <net/bluetooth/bluetooth.h>
#include <net/bluetooth/hci_core.h>
#include <net/bluetooth/l2cap.h>
@@ -2119,6 +2119,42 @@ static void l2cap_send_ack(struct l2cap_chan *chan)
}
}
+extern void __compiletime_error("copy source size is too small")
+__bad_copy_from(void);
+extern void __compiletime_error("copy destination size is too small")
+__bad_copy_to(void);
+
+static inline void copy_overflow(int size, unsigned long count)
+{
+ WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);
+}
+
+static __always_inline bool
+check_copy_size(const void *addr, size_t bytes, bool is_source)
+{
+ int sz = __compiletime_object_size(addr);
+ if (unlikely(sz >= 0 && sz < bytes)) {
+ if (!__builtin_constant_p(bytes))
+ copy_overflow(sz, bytes);
+ else if (is_source)
+ __bad_copy_from();
+ else
+ __bad_copy_to();
+ return false;
+ }
+ check_object_size(addr, bytes, is_source);
+ return true;
+}
+
+static __always_inline __must_check
+bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
+{
+ if (unlikely(!check_copy_size(addr, bytes, false)))
+ return false;
+ else
+ return _copy_from_iter_full(addr, bytes, i);
+}
+
static inline int l2cap_skbuff_fromiovec(struct l2cap_chan *chan,
struct msghdr *msg, int len,
int count, struct sk_buff *skb)
diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c
index 72022c5..f357d16 100644
--- a/net/bluetooth/l2cap_sock.c
+++ b/net/bluetooth/l2cap_sock.c
@@ -300,9 +300,13 @@ done:
release_sock(sk);
return err;
}
-
+#if LINUX_VERSION_IS_LESS(4,12,0)
+static int l2cap_sock_accept(struct socket *sock, struct socket *newsock,
+ int flags)
+#else
static int l2cap_sock_accept(struct socket *sock, struct socket *newsock,
int flags, bool kern)
+#endif
{
DEFINE_WAIT_FUNC(wait, woken_wake_function);
struct sock *sk = sock->sk, *nsk;
@@ -1028,7 +1032,11 @@ static int l2cap_sock_recvmsg(struct socket *sock, struct msghdr *msg,
goto done;
if (pi->rx_busy_skb) {
+#if LINUX_VERSION_IS_GEQ(4,7,0)
if (!__sock_queue_rcv_skb(sk, pi->rx_busy_skb))
+#else
+ if (!sock_queue_rcv_skb(sk, pi->rx_busy_skb))
+#endif
pi->rx_busy_skb = NULL;
else
goto done;
@@ -1297,7 +1305,11 @@ static int l2cap_sock_recv_cb(struct l2cap_chan *chan, struct sk_buff *skb)
goto done;
}
+#if LINUX_VERSION_IS_GEQ(4,7,0)
err = __sock_queue_rcv_skb(sk, skb);
+#else
+ err = sock_queue_rcv_skb(sk, skb);
+#endif
/* For ERTM, handle one skb that doesn't fit into the recv
* buffer. This is important to do because the data frames
diff --git a/net/bluetooth/rfcomm/core.c b/net/bluetooth/rfcomm/core.c
index 7184dad..20df249 100644
--- a/net/bluetooth/rfcomm/core.c
+++ b/net/bluetooth/rfcomm/core.c
@@ -351,7 +351,7 @@ static void rfcomm_dlc_unlink(struct rfcomm_dlc *d)
{
struct rfcomm_session *s = d->session;
- BT_DBG("dlc %p refcnt %d session %p", d, refcount_read(&d->refcnt), s);
+ BT_DBG("dlc %p refcnt %d session %p", d, refcount_read((const refcount_t *)&d->refcnt), s);
list_del(&d->list);
d->session = NULL;
@@ -872,7 +872,7 @@ static int rfcomm_queue_disc(struct rfcomm_dlc *d)
if (!skb)
return -ENOMEM;
- cmd = __skb_put(skb, sizeof(*cmd));
+ cmd = (void *)__skb_put(skb, sizeof(*cmd));
cmd->addr = d->addr;
cmd->ctrl = __ctrl(RFCOMM_DISC, 1);
cmd->len = __len8(0);
diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c
index 6b18b56..8b5c51b 100644
--- a/net/bluetooth/rfcomm/sock.c
+++ b/net/bluetooth/rfcomm/sock.c
@@ -205,7 +205,7 @@ static void rfcomm_sock_kill(struct sock *sk)
if (!sock_flag(sk, SOCK_ZAPPED) || sk->sk_socket)
return;
- BT_DBG("sk %p state %d refcnt %d", sk, sk->sk_state, refcount_read(&sk->sk_refcnt));
+ BT_DBG("sk %p state %d refcnt %d", sk, sk->sk_state, refcount_read((const refcount_t *)&sk->sk_refcnt));
/* Kill poor orphan */
bt_sock_unlink(&rfcomm_sk_list, sk);
@@ -480,8 +480,12 @@ done:
return err;
}
+#if LINUX_VERSION_IS_LESS(4,12,0)
+static int rfcomm_sock_accept(struct socket *sock, struct socket *newsock, int flags)
+#else
static int rfcomm_sock_accept(struct socket *sock, struct socket *newsock, int flags,
bool kern)
+#endif
{
DEFINE_WAIT_FUNC(wait, woken_wake_function);
struct sock *sk = sock->sk, *nsk;
diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c
index b1f215c..86a245f 100644
--- a/net/bluetooth/sco.c
+++ b/net/bluetooth/sco.c
@@ -623,9 +623,13 @@ done:
release_sock(sk);
return err;
}
-
+#if LINUX_VERSION_IS_LESS(4,12,0)
+static int sco_sock_accept(struct socket *sock, struct socket *newsock,
+ int flags)
+#else
static int sco_sock_accept(struct socket *sock, struct socket *newsock,
int flags, bool kern)
+#endif
{
DEFINE_WAIT_FUNC(wait, woken_wake_function);
struct sock *sk = sock->sk, *ch;