net: extend ubuf_info callback to ops structure
We'll need to associate additional callbacks with ubuf_info, introduce a structure holding ubuf_info callbacks. Apart from a more smarter io_uring notification management introduced in next patches, it can be used to generalise msg_zerocopy_put_abort() and also store ->sg_from_iter, which is currently passed in struct msghdr. Reviewed-by: Jens Axboe <axboe@kernel.dk> Reviewed-by: David Ahern <dsahern@kernel.org> Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> Reviewed-by: Willem de Bruijn <willemb@google.com> Link: https://lore.kernel.org/all/a62015541de49c0e2a8a0377a1d5d0a5aeb07016.1713369317.git.asml.silence@gmail.com Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Pavel Begunkov
Jakub Kicinski
parent
0bbac3facb
commit
7ab4f16f9e
|
|
@ -754,7 +754,7 @@ static ssize_t tap_get_user(struct tap_queue *q, void *msg_control,
|
|||
skb_zcopy_init(skb, msg_control);
|
||||
} else if (msg_control) {
|
||||
struct ubuf_info *uarg = msg_control;
|
||||
uarg->callback(NULL, uarg, false);
|
||||
uarg->ops->complete(NULL, uarg, false);
|
||||
}
|
||||
|
||||
dev_queue_xmit(skb);
|
||||
|
|
|
|||
|
|
@ -1906,7 +1906,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
|
|||
skb_zcopy_init(skb, msg_control);
|
||||
} else if (msg_control) {
|
||||
struct ubuf_info *uarg = msg_control;
|
||||
uarg->callback(NULL, uarg, false);
|
||||
uarg->ops->complete(NULL, uarg, false);
|
||||
}
|
||||
|
||||
skb_reset_network_header(skb);
|
||||
|
|
|
|||
|
|
@ -390,9 +390,8 @@ bool xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb);
|
|||
|
||||
void xenvif_carrier_on(struct xenvif *vif);
|
||||
|
||||
/* Callback from stack when TX packet can be released */
|
||||
void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf,
|
||||
bool zerocopy_success);
|
||||
/* Callbacks from stack when TX packet can be released */
|
||||
extern const struct ubuf_info_ops xenvif_ubuf_ops;
|
||||
|
||||
static inline pending_ring_idx_t nr_pending_reqs(struct xenvif_queue *queue)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -593,7 +593,7 @@ int xenvif_init_queue(struct xenvif_queue *queue)
|
|||
|
||||
for (i = 0; i < MAX_PENDING_REQS; i++) {
|
||||
queue->pending_tx_info[i].callback_struct = (struct ubuf_info_msgzc)
|
||||
{ { .callback = xenvif_zerocopy_callback },
|
||||
{ { .ops = &xenvif_ubuf_ops },
|
||||
{ { .ctx = NULL,
|
||||
.desc = i } } };
|
||||
queue->grant_tx_handle[i] = NETBACK_INVALID_HANDLE;
|
||||
|
|
|
|||
|
|
@ -1156,7 +1156,7 @@ static int xenvif_handle_frag_list(struct xenvif_queue *queue, struct sk_buff *s
|
|||
uarg = skb_shinfo(skb)->destructor_arg;
|
||||
/* increase inflight counter to offset decrement in callback */
|
||||
atomic_inc(&queue->inflight_packets);
|
||||
uarg->callback(NULL, uarg, true);
|
||||
uarg->ops->complete(NULL, uarg, true);
|
||||
skb_shinfo(skb)->destructor_arg = NULL;
|
||||
|
||||
/* Fill the skb with the new (local) frags. */
|
||||
|
|
@ -1278,8 +1278,9 @@ static int xenvif_tx_submit(struct xenvif_queue *queue)
|
|||
return work_done;
|
||||
}
|
||||
|
||||
void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base,
|
||||
bool zerocopy_success)
|
||||
static void xenvif_zerocopy_callback(struct sk_buff *skb,
|
||||
struct ubuf_info *ubuf_base,
|
||||
bool zerocopy_success)
|
||||
{
|
||||
unsigned long flags;
|
||||
pending_ring_idx_t index;
|
||||
|
|
@ -1312,6 +1313,10 @@ void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base,
|
|||
xenvif_skb_zerocopy_complete(queue);
|
||||
}
|
||||
|
||||
const struct ubuf_info_ops xenvif_ubuf_ops = {
|
||||
.complete = xenvif_zerocopy_callback,
|
||||
};
|
||||
|
||||
static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue)
|
||||
{
|
||||
struct gnttab_unmap_grant_ref *gop;
|
||||
|
|
|
|||
|
|
@ -380,7 +380,7 @@ static void vhost_zerocopy_signal_used(struct vhost_net *net,
|
|||
}
|
||||
}
|
||||
|
||||
static void vhost_zerocopy_callback(struct sk_buff *skb,
|
||||
static void vhost_zerocopy_complete(struct sk_buff *skb,
|
||||
struct ubuf_info *ubuf_base, bool success)
|
||||
{
|
||||
struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base);
|
||||
|
|
@ -408,6 +408,10 @@ static void vhost_zerocopy_callback(struct sk_buff *skb,
|
|||
rcu_read_unlock_bh();
|
||||
}
|
||||
|
||||
static const struct ubuf_info_ops vhost_ubuf_ops = {
|
||||
.complete = vhost_zerocopy_complete,
|
||||
};
|
||||
|
||||
static inline unsigned long busy_clock(void)
|
||||
{
|
||||
return local_clock() >> 10;
|
||||
|
|
@ -879,7 +883,7 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
|
|||
vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
|
||||
ubuf->ctx = nvq->ubufs;
|
||||
ubuf->desc = nvq->upend_idx;
|
||||
ubuf->ubuf.callback = vhost_zerocopy_callback;
|
||||
ubuf->ubuf.ops = &vhost_ubuf_ops;
|
||||
ubuf->ubuf.flags = SKBFL_ZEROCOPY_FRAG;
|
||||
refcount_set(&ubuf->ubuf.refcnt, 1);
|
||||
msg.msg_control = &ctl;
|
||||
|
|
|
|||
|
|
@ -527,6 +527,11 @@ enum {
|
|||
#define SKBFL_ALL_ZEROCOPY (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \
|
||||
SKBFL_DONT_ORPHAN | SKBFL_MANAGED_FRAG_REFS)
|
||||
|
||||
struct ubuf_info_ops {
|
||||
void (*complete)(struct sk_buff *, struct ubuf_info *,
|
||||
bool zerocopy_success);
|
||||
};
|
||||
|
||||
/*
|
||||
* The callback notifies userspace to release buffers when skb DMA is done in
|
||||
* lower device, the skb last reference should be 0 when calling this.
|
||||
|
|
@ -536,8 +541,7 @@ enum {
|
|||
* The desc field is used to track userspace buffer index.
|
||||
*/
|
||||
struct ubuf_info {
|
||||
void (*callback)(struct sk_buff *, struct ubuf_info *,
|
||||
bool zerocopy_success);
|
||||
const struct ubuf_info_ops *ops;
|
||||
refcount_t refcnt;
|
||||
u8 flags;
|
||||
};
|
||||
|
|
@ -1662,14 +1666,13 @@ static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
|
|||
}
|
||||
#endif
|
||||
|
||||
extern const struct ubuf_info_ops msg_zerocopy_ubuf_ops;
|
||||
|
||||
struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
|
||||
struct ubuf_info *uarg);
|
||||
|
||||
void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref);
|
||||
|
||||
void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
|
||||
bool success);
|
||||
|
||||
int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
|
||||
struct sk_buff *skb, struct iov_iter *from,
|
||||
size_t length);
|
||||
|
|
@ -1757,13 +1760,13 @@ static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb)
|
|||
static inline void net_zcopy_put(struct ubuf_info *uarg)
|
||||
{
|
||||
if (uarg)
|
||||
uarg->callback(NULL, uarg, true);
|
||||
uarg->ops->complete(NULL, uarg, true);
|
||||
}
|
||||
|
||||
static inline void net_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref)
|
||||
{
|
||||
if (uarg) {
|
||||
if (uarg->callback == msg_zerocopy_callback)
|
||||
if (uarg->ops == &msg_zerocopy_ubuf_ops)
|
||||
msg_zerocopy_put_abort(uarg, have_uref);
|
||||
else if (have_uref)
|
||||
net_zcopy_put(uarg);
|
||||
|
|
@ -1777,7 +1780,7 @@ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success)
|
|||
|
||||
if (uarg) {
|
||||
if (!skb_zcopy_is_nouarg(skb))
|
||||
uarg->callback(skb, uarg, zerocopy_success);
|
||||
uarg->ops->complete(skb, uarg, zerocopy_success);
|
||||
|
||||
skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ static void io_notif_complete_tw_ext(struct io_kiocb *notif, struct io_tw_state
|
|||
io_req_task_complete(notif, ts);
|
||||
}
|
||||
|
||||
static void io_tx_ubuf_callback(struct sk_buff *skb, struct ubuf_info *uarg,
|
||||
static void io_tx_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg,
|
||||
bool success)
|
||||
{
|
||||
struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);
|
||||
|
|
@ -45,19 +45,27 @@ static void io_tx_ubuf_callback_ext(struct sk_buff *skb, struct ubuf_info *uarg,
|
|||
else if (!success && !nd->zc_copied)
|
||||
WRITE_ONCE(nd->zc_copied, true);
|
||||
}
|
||||
io_tx_ubuf_callback(skb, uarg, success);
|
||||
io_tx_ubuf_complete(skb, uarg, success);
|
||||
}
|
||||
|
||||
static const struct ubuf_info_ops io_ubuf_ops = {
|
||||
.complete = io_tx_ubuf_complete,
|
||||
};
|
||||
|
||||
static const struct ubuf_info_ops io_ubuf_ops_ext = {
|
||||
.complete = io_tx_ubuf_callback_ext,
|
||||
};
|
||||
|
||||
void io_notif_set_extended(struct io_kiocb *notif)
|
||||
{
|
||||
struct io_notif_data *nd = io_notif_to_data(notif);
|
||||
|
||||
if (nd->uarg.callback != io_tx_ubuf_callback_ext) {
|
||||
if (nd->uarg.ops != &io_ubuf_ops_ext) {
|
||||
nd->account_pages = 0;
|
||||
nd->zc_report = false;
|
||||
nd->zc_used = false;
|
||||
nd->zc_copied = false;
|
||||
nd->uarg.callback = io_tx_ubuf_callback_ext;
|
||||
nd->uarg.ops = &io_ubuf_ops_ext;
|
||||
notif->io_task_work.func = io_notif_complete_tw_ext;
|
||||
}
|
||||
}
|
||||
|
|
@ -80,7 +88,7 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
|
|||
|
||||
nd = io_notif_to_data(notif);
|
||||
nd->uarg.flags = IO_NOTIF_UBUF_FLAGS;
|
||||
nd->uarg.callback = io_tx_ubuf_callback;
|
||||
nd->uarg.ops = &io_ubuf_ops;
|
||||
refcount_set(&nd->uarg.refcnt, 1);
|
||||
return notif;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1708,7 +1708,7 @@ static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size)
|
|||
return NULL;
|
||||
}
|
||||
|
||||
uarg->ubuf.callback = msg_zerocopy_callback;
|
||||
uarg->ubuf.ops = &msg_zerocopy_ubuf_ops;
|
||||
uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
|
||||
uarg->len = 1;
|
||||
uarg->bytelen = size;
|
||||
|
|
@ -1734,7 +1734,7 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
|
|||
u32 bytelen, next;
|
||||
|
||||
/* there might be non MSG_ZEROCOPY users */
|
||||
if (uarg->callback != msg_zerocopy_callback)
|
||||
if (uarg->ops != &msg_zerocopy_ubuf_ops)
|
||||
return NULL;
|
||||
|
||||
/* realloc only when socket is locked (TCP, UDP cork),
|
||||
|
|
@ -1845,8 +1845,8 @@ release:
|
|||
sock_put(sk);
|
||||
}
|
||||
|
||||
void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
|
||||
bool success)
|
||||
static void msg_zerocopy_complete(struct sk_buff *skb, struct ubuf_info *uarg,
|
||||
bool success)
|
||||
{
|
||||
struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg);
|
||||
|
||||
|
|
@ -1855,7 +1855,6 @@ void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
|
|||
if (refcount_dec_and_test(&uarg->refcnt))
|
||||
__msg_zerocopy_callback(uarg_zc);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(msg_zerocopy_callback);
|
||||
|
||||
void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
|
||||
{
|
||||
|
|
@ -1865,10 +1864,15 @@ void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
|
|||
uarg_to_msgzc(uarg)->len--;
|
||||
|
||||
if (have_uref)
|
||||
msg_zerocopy_callback(NULL, uarg, true);
|
||||
msg_zerocopy_complete(NULL, uarg, true);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort);
|
||||
|
||||
const struct ubuf_info_ops msg_zerocopy_ubuf_ops = {
|
||||
.complete = msg_zerocopy_complete,
|
||||
};
|
||||
EXPORT_SYMBOL_GPL(msg_zerocopy_ubuf_ops);
|
||||
|
||||
int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
|
||||
struct msghdr *msg, int len,
|
||||
struct ubuf_info *uarg)
|
||||
|
|
|
|||
Loading…
Reference in New Issue