summaryrefslogtreecommitdiff
path: root/net/core/skbuff.c
diff options
context:
space:
mode:
authorWillem de Bruijn <willemb@google.com>2017-08-03 16:29:38 -0400
committerDavid S. Miller <davem@davemloft.net>2017-08-03 21:37:29 -0700
commit3ece782693c4b64d588dd217868558ab9a19bfe7 (patch)
treec18f48b0b880177bd081d3f365ce296b6f96f777 /net/core/skbuff.c
parent98ba0bd5505dcbb90322a4be07bcfe6b8a18c73f (diff)
sock: skb_copy_ubufs support for compound pages
Refine skb_copy_ubufs to support compound pages. With upcoming TCP zerocopy sendmsg, such fragments may appear. The existing code replaces each page one for one. Splitting each compound page into an independent number of regular pages can result in exceeding limit MAX_SKB_FRAGS if data is not exactly page aligned. Instead, fill all destination pages but the last to PAGE_SIZE. Split the existing alloc + copy loop into separate stages: 1. compute bytelength and minimum number of pages to store this. 2. allocate 3. copy, filling each page except the last to PAGE_SIZE bytes 4. update skb frag array Signed-off-by: Willem de Bruijn <willemb@google.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'net/core/skbuff.c')
-rw-r--r--net/core/skbuff.c53
1 files changed, 38 insertions, 15 deletions
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 0f0933b338d7..a95877a8ac8b 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -932,17 +932,20 @@ EXPORT_SYMBOL_GPL(skb_morph);
*/
int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
{
- int i;
+ struct ubuf_info *uarg = skb_shinfo(skb)->destructor_arg;
int num_frags = skb_shinfo(skb)->nr_frags;
struct page *page, *head = NULL;
- struct ubuf_info *uarg = skb_shinfo(skb)->destructor_arg;
+ int i, new_frags;
+ u32 d_off;
- for (i = 0; i < num_frags; i++) {
- skb_frag_t *f = &skb_shinfo(skb)->frags[i];
- u32 p_off, p_len, copied;
- struct page *p;
- u8 *vaddr;
+ if (!num_frags)
+ return 0;
+
+ if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
+ return -EINVAL;
+ new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT;
+ for (i = 0; i < new_frags; i++) {
page = alloc_page(gfp_mask);
if (!page) {
while (head) {
@@ -952,17 +955,36 @@ int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
}
return -ENOMEM;
}
+ set_page_private(page, (unsigned long)head);
+ head = page;
+ }
+
+ page = head;
+ d_off = 0;
+ for (i = 0; i < num_frags; i++) {
+ skb_frag_t *f = &skb_shinfo(skb)->frags[i];
+ u32 p_off, p_len, copied;
+ struct page *p;
+ u8 *vaddr;
skb_frag_foreach_page(f, f->page_offset, skb_frag_size(f),
p, p_off, p_len, copied) {
+ u32 copy, done = 0;
vaddr = kmap_atomic(p);
- memcpy(page_address(page) + copied, vaddr + p_off,
- p_len);
+
+ while (done < p_len) {
+ if (d_off == PAGE_SIZE) {
+ d_off = 0;
+ page = (struct page *)page_private(page);
+ }
+ copy = min_t(u32, PAGE_SIZE - d_off, p_len - done);
+ memcpy(page_address(page) + d_off,
+ vaddr + p_off + done, copy);
+ done += copy;
+ d_off += copy;
+ }
kunmap_atomic(vaddr);
}
-
- set_page_private(page, (unsigned long)head);
- head = page;
}
/* skb frags release userspace buffers */
@@ -972,11 +994,12 @@ int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
uarg->callback(uarg, false);
/* skb frags point to kernel buffers */
- for (i = num_frags - 1; i >= 0; i--) {
- __skb_fill_page_desc(skb, i, head, 0,
- skb_shinfo(skb)->frags[i].size);
+ for (i = 0; i < new_frags - 1; i++) {
+ __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE);
head = (struct page *)page_private(head);
}
+ __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off);
+ skb_shinfo(skb)->nr_frags = new_frags;
skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
return 0;