From d69e8f3e06e4b2156827ce5e45373713da57abed Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Thu, 31 Jan 2013 15:46:43 +0100 Subject: [PATCH 01/76] tevent: Fix a comment typo Signed-off-by: Volker Lendecke Reviewed-by: Ira Cooper Reviewed-by: Richard Sharpe Autobuild-User(master): Richard Sharpe Autobuild-Date(master): Thu Jan 31 17:52:39 CET 2013 on sn-devel-104 --- lib/tevent/tevent_poll.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index da8cc01..44e108f 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -56,7 +56,7 @@ struct poll_event_context { }; /* - create a select_event_context structure. + create a poll_event_context structure. */ static int poll_event_context_init(struct tevent_context *ev) { -- 1.8.1.3 From eb4ce1d0cfe584d7c3f0676600014fe65ad68a93 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Thu, 31 Jan 2013 14:33:15 +0100 Subject: [PATCH 02/76] tevent_poll: Fix a crash in the poll backend If tevent_add_fd is immediately followed by tevent_fd_set_flags, the poll backend crashes. This was introduced when the poll backend was prepared for the multi-threaded python extension. Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index 44e108f..fcf97c4 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -256,7 +256,7 @@ static struct tevent_fd *poll_event_add_fd(struct tevent_context *ev, fde->private_data = private_data; fde->handler_name = handler_name; fde->location = location; - fde->additional_flags = 0; + fde->additional_flags = UINT64_MAX; fde->additional_data = NULL; DLIST_ADD(poll_ev->fresh, fde); @@ -278,7 +278,20 @@ static void poll_event_set_fd_flags(struct tevent_fd *fde, uint16_t flags) struct poll_event_context *poll_ev = talloc_get_type_abort( fde->event_ctx->additional_data, struct poll_event_context); uint64_t idx = fde->additional_flags; - uint16_t pollflags = 0; + uint16_t pollflags; + + fde->flags = flags; + + if (idx == UINT64_MAX) { + /* + * poll_event_setup_fresh not yet called after this fde was + * added. We don't have to do anything to transfer the changed + * flags to the array passed to poll(2) + */ + return; + } + + pollflags = 0; if (flags & TEVENT_FD_READ) { pollflags |= (POLLIN|POLLHUP); @@ -286,10 +299,8 @@ static void poll_event_set_fd_flags(struct tevent_fd *fde, uint16_t flags) if (flags & TEVENT_FD_WRITE) { pollflags |= (POLLOUT); } - poll_ev->fds[idx].events = pollflags; - fde->flags = flags; poll_event_wake_pollthread(poll_ev); } -- 1.8.1.3 From a0749d74d5ab415e7db036354be580a0be93e345 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Thu, 31 Jan 2013 16:25:23 +0100 Subject: [PATCH 03/76] tevent_poll: Re-order routines Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index fcf97c4..b6265fd 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -55,6 +55,22 @@ struct poll_event_context { int exit_code; }; +static int poll_event_mt_destructor(struct poll_event_context *poll_ev) +{ + if (poll_ev->signal_fd != -1) { + close(poll_ev->signal_fd); + poll_ev->signal_fd = -1; + } + if (poll_ev->num_fds == 0) { + return 0; + } + if (poll_ev->fds[0].fd != -1) { + close(poll_ev->fds[0].fd); + poll_ev->fds[0].fd = -1; + } + return 0; +} + /* create a poll_event_context structure. */ @@ -72,22 +88,6 @@ static int poll_event_context_init(struct tevent_context *ev) return 0; } -static int poll_event_mt_destructor(struct poll_event_context *poll_ev) -{ - if (poll_ev->signal_fd != -1) { - close(poll_ev->signal_fd); - poll_ev->signal_fd = -1; - } - if (poll_ev->num_fds == 0) { - return 0; - } - if (poll_ev->fds[0].fd != -1) { - close(poll_ev->fds[0].fd); - poll_ev->fds[0].fd = -1; - } - return 0; -} - static bool set_nonblock(int fd) { int val; -- 1.8.1.3 From fa15fa8f888791d08fcf75a555783303d3042286 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Thu, 31 Jan 2013 16:28:08 +0100 Subject: [PATCH 04/76] tevent_poll: Use the poll_event_context_destructor always This is in preparation for the next patch to NULL out the event_ctx pointers in the poll_ev->fresh list Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index b6265fd..f64dd38 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -55,12 +55,18 @@ struct poll_event_context { int exit_code; }; -static int poll_event_mt_destructor(struct poll_event_context *poll_ev) +static int poll_event_context_destructor(struct poll_event_context *poll_ev) { - if (poll_ev->signal_fd != -1) { - close(poll_ev->signal_fd); - poll_ev->signal_fd = -1; + if (poll_ev->signal_fd == -1) { + /* + * Non-threaded, no signal pipe + */ + return 0; } + + close(poll_ev->signal_fd); + poll_ev->signal_fd = -1; + if (poll_ev->num_fds == 0) { return 0; } @@ -85,6 +91,7 @@ static int poll_event_context_init(struct tevent_context *ev) poll_ev->ev = ev; poll_ev->signal_fd = -1; ev->additional_data = poll_ev; + talloc_set_destructor(poll_ev, poll_event_context_destructor); return 0; } @@ -140,7 +147,7 @@ static int poll_event_context_init_mt(struct tevent_context *ev) poll_ev->num_fds = 1; - talloc_set_destructor(poll_ev, poll_event_mt_destructor); + talloc_set_destructor(poll_ev, poll_event_context_destructor); return 0; } -- 1.8.1.3 From 0dc0d138f3ec71eafd679c4be3738b38fd9b7375 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Thu, 31 Jan 2013 16:31:45 +0100 Subject: [PATCH 05/76] tevent_poll: NULL out fde->event_ctx for "fresh" poll fdes This is done in tevent_common_context_destructor for the non-fresh fdes already Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index f64dd38..0c580e4 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -57,6 +57,14 @@ struct poll_event_context { static int poll_event_context_destructor(struct poll_event_context *poll_ev) { + struct tevent_fd *fd, *fn; + + for (fd = poll_ev->fresh; fd; fd = fn) { + fn = fd->next; + fd->event_ctx = NULL; + DLIST_REMOVE(poll_ev->fresh, fd); + } + if (poll_ev->signal_fd == -1) { /* * Non-threaded, no signal pipe -- 1.8.1.3 From e20eb41ee3e200af206bab2de049de50aa31b7f4 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Thu, 31 Jan 2013 16:34:41 +0100 Subject: [PATCH 06/76] tevent_poll: Avoid a crash in poll_fresh_fde_destructor The event context might have been freed before the fde Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index 0c580e4..f631737 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -218,8 +218,15 @@ done: static int poll_fresh_fde_destructor(struct tevent_fd *fde) { - struct poll_event_context *poll_ev = talloc_get_type_abort( - fde->event_ctx->additional_data, struct poll_event_context); + struct tevent_context *ev = fde->event_ctx; + struct poll_event_context *poll_ev; + + if (ev == NULL) { + return 0; + } + poll_ev = talloc_get_type_abort( + ev->additional_data, struct poll_event_context); + DLIST_REMOVE(poll_ev->fresh, fde); return 0; } -- 1.8.1.3 From 1c57a1a0aee487ca43d3113f25ec057cc686468d Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Thu, 31 Jan 2013 16:34:41 +0100 Subject: [PATCH 07/76] tevent_poll: Avoid a crash in poll_event_set_fd_flags The event context might have been freed before the fde Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index f631737..7d10665 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -297,11 +297,17 @@ static struct tevent_fd *poll_event_add_fd(struct tevent_context *ev, */ static void poll_event_set_fd_flags(struct tevent_fd *fde, uint16_t flags) { - struct poll_event_context *poll_ev = talloc_get_type_abort( - fde->event_ctx->additional_data, struct poll_event_context); + struct tevent_context *ev = fde->event_ctx; + struct poll_event_context *poll_ev; uint64_t idx = fde->additional_flags; uint16_t pollflags; + if (ev == NULL) { + return; + } + poll_ev = talloc_get_type_abort( + ev->additional_data, struct poll_event_context); + fde->flags = flags; if (idx == UINT64_MAX) { -- 1.8.1.3 From 5798b965689b537f799bc6e417025b02af9f16b6 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 1 Feb 2013 15:56:34 +0100 Subject: [PATCH 08/76] tevent_poll: call tevent_common_fd_destructor() from poll_fresh_fde_destructor() We need to trigger the fde->close_fn(). Signed-off-by: Stefan Metzmacher Reviewed-by: Volker Lendecke Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index 7d10665..b8221a4 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -222,13 +222,14 @@ static int poll_fresh_fde_destructor(struct tevent_fd *fde) struct poll_event_context *poll_ev; if (ev == NULL) { - return 0; + goto done; } poll_ev = talloc_get_type_abort( ev->additional_data, struct poll_event_context); DLIST_REMOVE(poll_ev->fresh, fde); - return 0; +done: + return tevent_common_fd_destructor(fde); } static void poll_event_schedule_immediate(struct tevent_immediate *im, -- 1.8.1.3 From d068f291bbdceb2b4119d70355d42e5b6506cd00 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 1 Feb 2013 15:27:10 +0100 Subject: [PATCH 09/76] tevent_poll: fix the usage of tevent_re_initialise() Signed-off-by: Stefan Metzmacher Reviewed-by: Volker Lendecke Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index b8221a4..ac567ce 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -92,6 +92,14 @@ static int poll_event_context_init(struct tevent_context *ev) { struct poll_event_context *poll_ev; + /* + * we might be called during tevent_re_initialise() + * which means we need to free our old additional_data + * in order to detach old fd events from the + * poll_ev->fresh list + */ + TALLOC_FREE(ev->additional_data); + poll_ev = talloc_zero(ev, struct poll_event_context); if (poll_ev == NULL) { return -1; -- 1.8.1.3 From 3d5f1d9346c7d99fa402a398dcd11a68525b6128 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 1 Feb 2013 15:50:08 +0100 Subject: [PATCH 10/76] tevent_poll: add poll_event_loop_wait() We can't use tevent_common_loop_wait() because new fd events will be added to the poll_ev->fresh list instead of ev->fd_events. Signed-off-by: Stefan Metzmacher Reviewed-by: Volker Lendecke Reviewed-by: Jeremy Allison Autobuild-User(master): Jeremy Allison Autobuild-Date(master): Sun Feb 3 06:40:09 CET 2013 on sn-devel-104 --- lib/tevent/tevent_poll.c | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index ac567ce..5479f2f 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -547,6 +547,35 @@ static int poll_event_loop_once(struct tevent_context *ev, return poll_event_loop_poll(ev, &tval); } +static int poll_event_loop_wait(struct tevent_context *ev, + const char *location) +{ + struct poll_event_context *poll_ev = talloc_get_type_abort( + ev->additional_data, struct poll_event_context); + + /* + * loop as long as we have events pending + */ + while (ev->fd_events || + ev->timer_events || + ev->immediate_events || + ev->signal_events || + poll_ev->fresh) { + int ret; + ret = _tevent_loop_once(ev, location); + if (ret != 0) { + tevent_debug(ev, TEVENT_DEBUG_FATAL, + "_tevent_loop_once() failed: %d - %s\n", + ret, strerror(errno)); + return ret; + } + } + + tevent_debug(ev, TEVENT_DEBUG_WARNING, + "poll_event_loop_wait() out of events\n"); + return 0; +} + static const struct tevent_ops poll_event_ops = { .context_init = poll_event_context_init, .add_fd = poll_event_add_fd, @@ -557,7 +586,7 @@ static const struct tevent_ops poll_event_ops = { .schedule_immediate = tevent_common_schedule_immediate, .add_signal = tevent_common_add_signal, .loop_once = poll_event_loop_once, - .loop_wait = tevent_common_loop_wait, + .loop_wait = poll_event_loop_wait, }; _PRIVATE_ bool tevent_poll_init(void) @@ -575,7 +604,7 @@ static const struct tevent_ops poll_event_mt_ops = { .schedule_immediate = poll_event_schedule_immediate, .add_signal = tevent_common_add_signal, .loop_once = poll_event_loop_once, - .loop_wait = tevent_common_loop_wait, + .loop_wait = poll_event_loop_wait, }; _PRIVATE_ bool tevent_poll_mt_init(void) -- 1.8.1.3 From 6e6346f2a7a83d9185b3cde12c6b1e8b86ca042f Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Mon, 11 Feb 2013 10:36:52 -0800 Subject: [PATCH 11/76] tevent: Preparing to fix "standard" backend fallback. Initialize standard after epoll. Signed-off-by: Jeremy Allison Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tevent/tevent.c b/lib/tevent/tevent.c index fa842e4..e307dba 100644 --- a/lib/tevent/tevent.c +++ b/lib/tevent/tevent.c @@ -115,10 +115,10 @@ static void tevent_backend_init(void) tevent_select_init(); tevent_poll_init(); tevent_poll_mt_init(); - tevent_standard_init(); #ifdef HAVE_EPOLL tevent_epoll_init(); #endif + tevent_standard_init(); } /* -- 1.8.1.3 From 5ba56174184e833fec9d96436406f442d652a4c8 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Mon, 11 Feb 2013 10:38:01 -0800 Subject: [PATCH 12/76] tevent: Ensure we return after every call to epoll_panic(). Currently we can't return from this, but the new fallback code will change this. Signed-off-by: Jeremy Allison Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 5f93de2..8e7bc4d 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -152,6 +152,7 @@ static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_ event.data.ptr = fde; if (epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_ADD, fde->fd, &event) != 0) { epoll_panic(epoll_ev, "EPOLL_CTL_ADD failed"); + return; } fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; @@ -201,6 +202,7 @@ static void epoll_mod_event(struct epoll_event_context *epoll_ev, struct tevent_ event.data.ptr = fde; if (epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_MOD, fde->fd, &event) != 0) { epoll_panic(epoll_ev, "EPOLL_CTL_MOD failed"); + return; } /* only if we want to read we want to tell the event handler about errors */ -- 1.8.1.3 From b098c8d4df32b1bfaba450728ef73212a9855271 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 14 Feb 2013 10:58:55 +0100 Subject: [PATCH 13/76] tevent: pass 'bool replay' to epoll_panic() A fallback panic handler will need to know if there was an error while waiting for events (replay=true) or if the error happened on modify (replay=false). Signed-off-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 8e7bc4d..c7936d3 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -44,7 +44,8 @@ struct epoll_event_context { /* called when a epoll call fails */ -static void epoll_panic(struct epoll_event_context *epoll_ev, const char *reason) +static void epoll_panic(struct epoll_event_context *epoll_ev, + const char *reason, bool replay) { tevent_debug(epoll_ev->ev, TEVENT_DEBUG_FATAL, "%s (%s) - calling abort()\n", reason, strerror(errno)); @@ -151,7 +152,7 @@ static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_ event.events = epoll_map_flags(fde->flags); event.data.ptr = fde; if (epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_ADD, fde->fd, &event) != 0) { - epoll_panic(epoll_ev, "EPOLL_CTL_ADD failed"); + epoll_panic(epoll_ev, "EPOLL_CTL_ADD failed", false); return; } fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; @@ -201,7 +202,7 @@ static void epoll_mod_event(struct epoll_event_context *epoll_ev, struct tevent_ event.events = epoll_map_flags(fde->flags); event.data.ptr = fde; if (epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_MOD, fde->fd, &event) != 0) { - epoll_panic(epoll_ev, "EPOLL_CTL_MOD failed"); + epoll_panic(epoll_ev, "EPOLL_CTL_MOD failed", false); return; } @@ -277,7 +278,7 @@ static int epoll_event_loop(struct epoll_event_context *epoll_ev, struct timeval } if (ret == -1 && errno != EINTR) { - epoll_panic(epoll_ev, "epoll_wait() failed"); + epoll_panic(epoll_ev, "epoll_wait() failed", true); return -1; } @@ -293,7 +294,7 @@ static int epoll_event_loop(struct epoll_event_context *epoll_ev, struct timeval uint16_t flags = 0; if (fde == NULL) { - epoll_panic(epoll_ev, "epoll_wait() gave bad data"); + epoll_panic(epoll_ev, "epoll_wait() gave bad data", true); return -1; } if (events[i].events & (EPOLLHUP|EPOLLERR)) { -- 1.8.1.3 From d6a16337ad1e0a5ff0561f176089a67c377be994 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Mon, 11 Feb 2013 10:43:39 -0800 Subject: [PATCH 14/76] tevent: Add an internal function tevent_epoll_set_panic_fallback(). Can be set externally, allows us to fallback if epoll fails at runtime. Signed-off-by: Jeremy Allison Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 24 ++++++++++++++++++++++++ lib/tevent/tevent_internal.h | 3 +++ 2 files changed, 27 insertions(+) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index c7936d3..8a48d1d 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -39,9 +39,33 @@ struct epoll_event_context { int epoll_fd; pid_t pid; + + bool (*panic_fallback)(struct tevent_context *ev, bool replay); }; /* + called to set the panic fallback function. +*/ +_PRIVATE_ bool tevent_epoll_set_panic_fallback(struct tevent_context *ev, + bool (*panic_fallback)(struct tevent_context *ev, + bool replay)) +{ + struct epoll_event_context *epoll_ev; + + if (ev->additional_data == NULL) { + return false; + } + + epoll_ev = talloc_get_type(ev->additional_data, + struct epoll_event_context); + if (epoll_ev == NULL) { + return false; + } + epoll_ev->panic_fallback = panic_fallback; + return true; +} + +/* called when a epoll call fails */ static void epoll_panic(struct epoll_event_context *epoll_ev, diff --git a/lib/tevent/tevent_internal.h b/lib/tevent/tevent_internal.h index f09cf57..4e9cd69 100644 --- a/lib/tevent/tevent_internal.h +++ b/lib/tevent/tevent_internal.h @@ -318,6 +318,9 @@ bool tevent_poll_init(void); bool tevent_poll_mt_init(void); #ifdef HAVE_EPOLL bool tevent_epoll_init(void); +bool tevent_epoll_set_panic_fallback(struct tevent_context *ev, + bool (*panic_fallback)(struct tevent_context *ev, + bool replay)); #endif void tevent_trace_point_callback(struct tevent_context *ev, -- 1.8.1.3 From 8c530e47a8ab6e1cbb04b79901884853bbe7c2f4 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Mon, 11 Feb 2013 10:48:02 -0800 Subject: [PATCH 15/76] tevent: Plumb in the panic fallback code into the epoll_panic() runtime call. Signed-off-by: Jeremy Allison Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 8a48d1d..0ef1ffe 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -71,9 +71,26 @@ _PRIVATE_ bool tevent_epoll_set_panic_fallback(struct tevent_context *ev, static void epoll_panic(struct epoll_event_context *epoll_ev, const char *reason, bool replay) { - tevent_debug(epoll_ev->ev, TEVENT_DEBUG_FATAL, - "%s (%s) - calling abort()\n", reason, strerror(errno)); - abort(); + struct tevent_context *ev = epoll_ev->ev; + + if (epoll_ev->panic_fallback == NULL) { + tevent_debug(ev, TEVENT_DEBUG_FATAL, + "%s (%s) replay[%u] - calling abort()\n", + reason, strerror(errno), (unsigned)replay); + abort(); + } + + tevent_debug(ev, TEVENT_DEBUG_WARNING, + "%s (%s) replay[%u] - calling panic_fallback\n", + reason, strerror(errno), (unsigned)replay); + + if (!epoll_ev->panic_fallback(ev, replay)) { + /* Fallback failed. */ + tevent_debug(ev, TEVENT_DEBUG_FATAL, + "%s (%s) replay[%u] - calling abort()\n", + reason, strerror(errno), (unsigned)replay); + abort(); + } } /* -- 1.8.1.3 From 0bd15616788923cdc3bffa4fcb87eb92c4383d7e Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Mon, 11 Feb 2013 10:52:30 -0800 Subject: [PATCH 16/76] tevent: Add in some test code to allow the panic fallback path to be tested. Signed-off-by: Jeremy Allison Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 0ef1ffe..5a17331 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -43,6 +43,24 @@ struct epoll_event_context { bool (*panic_fallback)(struct tevent_context *ev, bool replay); }; +#ifdef TEST_PANIC_FALLBACK +static int epoll_wait_panic_fallback(int epfd, + struct epoll_event *events, + int maxevents, + int timeout) +{ + /* 50% of the time, fail... */ + if ((random() % 2) == 0) { + errno = EINVAL; + return -1; + } + + return epoll_wait(epfd, events, maxevents, timeout); +} + +#define epoll_wait epoll_wait_panic_fallback +#endif + /* called to set the panic fallback function. */ -- 1.8.1.3 From 55f361418fe7585fe58f281c8398b067247587ed Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Mon, 11 Feb 2013 10:53:15 -0800 Subject: [PATCH 17/76] tevent: Add in the same tevent_re_initialise() fix Metze put in the tevent_poll backend. We might be called during tevent_re_initialise() which means we need to free our old additional_data. Signed-off-by: Jeremy Allison Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 6 ++++++ lib/tevent/tevent_select.c | 6 ++++++ lib/tevent/tevent_standard.c | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 5a17331..8696215 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -389,6 +389,12 @@ static int epoll_event_context_init(struct tevent_context *ev) int ret; struct epoll_event_context *epoll_ev; + /* + * We might be called during tevent_re_initialise() + * which means we need to free our old additional_data. + */ + TALLOC_FREE(ev->additional_data); + epoll_ev = talloc_zero(ev, struct epoll_event_context); if (!epoll_ev) return -1; epoll_ev->ev = ev; diff --git a/lib/tevent/tevent_select.c b/lib/tevent/tevent_select.c index c11f0e8..7e0c927 100644 --- a/lib/tevent/tevent_select.c +++ b/lib/tevent/tevent_select.c @@ -47,6 +47,12 @@ static int select_event_context_init(struct tevent_context *ev) { struct select_event_context *select_ev; + /* + * We might be called during tevent_re_initialise() + * which means we need to free our old additional_data. + */ + TALLOC_FREE(ev->additional_data); + select_ev = talloc_zero(ev, struct select_event_context); if (!select_ev) return -1; select_ev->ev = ev; diff --git a/lib/tevent/tevent_standard.c b/lib/tevent/tevent_standard.c index 1e33720..938b223 100644 --- a/lib/tevent/tevent_standard.c +++ b/lib/tevent/tevent_standard.c @@ -348,6 +348,12 @@ static int std_event_context_init(struct tevent_context *ev) { struct std_event_context *std_ev; + /* + * We might be called during tevent_re_initialise() + * which means we need to free our old additional_data. + */ + TALLOC_FREE(ev->additional_data); + std_ev = talloc_zero(ev, struct std_event_context); if (!std_ev) return -1; std_ev->ev = ev; -- 1.8.1.3 From 90a44aa3f4258971ce75f71f5e1cfe3c2183515d Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Mon, 11 Feb 2013 10:56:58 -0800 Subject: [PATCH 18/76] tevent: Add a utility function tevent_find_ops_byname(). Returns an event ops struct given a string name. Not yet used, but will be part of the new "standard" fallback code. Signed-off-by: Jeremy Allison Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent.c | 22 ++++++++++++++++++++++ lib/tevent/tevent_internal.h | 1 + 2 files changed, 23 insertions(+) diff --git a/lib/tevent/tevent.c b/lib/tevent/tevent.c index e307dba..6352471 100644 --- a/lib/tevent/tevent.c +++ b/lib/tevent/tevent.c @@ -121,6 +121,28 @@ static void tevent_backend_init(void) tevent_standard_init(); } +_PRIVATE_ const struct tevent_ops *tevent_find_ops_byname(const char *name) +{ + struct tevent_ops_list *e; + + tevent_backend_init(); + + if (name == NULL) { + name = tevent_default_backend; + } + if (name == NULL) { + name = "standard"; + } + + for (e = tevent_backends; e != NULL; e = e->next) { + if (0 == strcmp(e->name, name)) { + return e->ops; + } + } + + return NULL; +} + /* list available backends */ diff --git a/lib/tevent/tevent_internal.h b/lib/tevent/tevent_internal.h index 4e9cd69..fa6652c 100644 --- a/lib/tevent/tevent_internal.h +++ b/lib/tevent/tevent_internal.h @@ -265,6 +265,7 @@ struct tevent_context { } tracing; }; +const struct tevent_ops *tevent_find_ops_byname(const char *name); int tevent_common_context_destructor(struct tevent_context *ev); int tevent_common_loop_wait(struct tevent_context *ev, -- 1.8.1.3 From 548465bf393691c3495ab346bdaf3c6e348d9ec8 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 14 Feb 2013 09:29:57 +0100 Subject: [PATCH 19/76] tevent: make sure tevent_backend_init() only runs once Signed-off-by: Stefan Metzmacher --- lib/tevent/tevent.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/tevent/tevent.c b/lib/tevent/tevent.c index 6352471..669b778 100644 --- a/lib/tevent/tevent.c +++ b/lib/tevent/tevent.c @@ -112,6 +112,14 @@ void tevent_set_default_backend(const char *backend) */ static void tevent_backend_init(void) { + static bool done; + + if (done) { + return; + } + + done = true; + tevent_select_init(); tevent_poll_init(); tevent_poll_mt_init(); -- 1.8.1.3 From 3f12695dc3c707e4ca6904ea0dd970dc528db36b Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 14 Feb 2013 09:30:31 +0100 Subject: [PATCH 20/76] tevent: make use of tevent_find_ops_byname() in tevent_context_init_byname() Signed-off-by: Stefan Metzmacher --- lib/tevent/tevent.c | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/lib/tevent/tevent.c b/lib/tevent/tevent.c index 669b778..aa758de 100644 --- a/lib/tevent/tevent.c +++ b/lib/tevent/tevent.c @@ -273,23 +273,14 @@ struct tevent_context *tevent_context_init_ops(TALLOC_CTX *mem_ctx, struct tevent_context *tevent_context_init_byname(TALLOC_CTX *mem_ctx, const char *name) { - struct tevent_ops_list *e; - - tevent_backend_init(); + struct tevent_ops *ops; - if (name == NULL) { - name = tevent_default_backend; - } - if (name == NULL) { - name = "standard"; + ops = tevent_find_ops_byname(name); + if (ops == NULL) { + return NULL; } - for (e=tevent_backends;e;e=e->next) { - if (strcmp(name, e->name) == 0) { - return tevent_context_init_ops(mem_ctx, e->ops, NULL); - } - } - return NULL; + return tevent_context_init_ops(mem_ctx, ops, NULL); } -- 1.8.1.3 From eebba27e3d1a6ff836e50413e6567f2f307c3d69 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Mon, 11 Feb 2013 11:20:28 -0800 Subject: [PATCH 21/76] tevent: Add a private function tevent_poll_event_add_fd_internal(). Not yet used, but will be called by the "standard" fallback from epoll -> poll backends. Signed-off-by: Jeremy Allison Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent_internal.h | 3 +++ lib/tevent/tevent_poll.c | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/lib/tevent/tevent_internal.h b/lib/tevent/tevent_internal.h index fa6652c..8433333 100644 --- a/lib/tevent/tevent_internal.h +++ b/lib/tevent/tevent_internal.h @@ -316,6 +316,8 @@ void tevent_cleanup_pending_signal_handlers(struct tevent_signal *se); bool tevent_standard_init(void); bool tevent_select_init(void); bool tevent_poll_init(void); +void tevent_poll_event_add_fd_internal(struct tevent_context *ev, + struct tevent_fd *fde); bool tevent_poll_mt_init(void); #ifdef HAVE_EPOLL bool tevent_epoll_init(void); @@ -324,5 +326,6 @@ bool tevent_epoll_set_panic_fallback(struct tevent_context *ev, bool replay)); #endif + void tevent_trace_point_callback(struct tevent_context *ev, enum tevent_trace_point); diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index 5479f2f..89b3bbc 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -256,6 +256,22 @@ static void poll_event_schedule_immediate(struct tevent_immediate *im, } /* + Private function called by "standard" backend fallback. + Note this only allows fallback to "poll" backend, not "poll-mt". +*/ +_PRIVATE_ void tevent_poll_event_add_fd_internal(struct tevent_context *ev, + struct tevent_fd *fde) +{ + struct poll_event_context *poll_ev = talloc_get_type_abort( + ev->additional_data, struct poll_event_context); + + fde->additional_flags = UINT64_MAX; + fde->additional_data = NULL; + DLIST_ADD(poll_ev->fresh, fde); + talloc_set_destructor(fde, poll_fresh_fde_destructor); +} + +/* add a fd based event return NULL on failure (memory allocation error) */ -- 1.8.1.3 From ea41d78562bd4d46e5cc7850e68486e2b89f038f Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Mon, 11 Feb 2013 11:40:49 -0800 Subject: [PATCH 22/76] tevent: Add in the new implementation of "standard" tevent backend. Falls back cleanly from epoll -> poll, or uses poll if epoll not available. Signed-off-by: Jeremy Allison Signed-off-by: Stefan Metzmacher Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent_standard.c | 186 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 180 insertions(+), 6 deletions(-) diff --git a/lib/tevent/tevent_standard.c b/lib/tevent/tevent_standard.c index 938b223..0c72226 100644 --- a/lib/tevent/tevent_standard.c +++ b/lib/tevent/tevent_standard.c @@ -2,7 +2,8 @@ Unix SMB/CIFS implementation. main select loop and event handling Copyright (C) Andrew Tridgell 2003-2005 - Copyright (C) Stefan Metzmacher 2005-2009 + Copyright (C) Stefan Metzmacher 2005-2013 + Copyright (C) Jeremy Allison 2013 ** NOTE! The following LGPL license applies to the tevent ** library. This does NOT imply that all of Samba is released @@ -26,18 +27,17 @@ This is SAMBA's default event loop code - we try to use epoll if configure detected support for it - otherwise we use select() + otherwise we use poll() - if epoll is broken on the system or the kernel doesn't support it - at runtime we fallback to select() + at runtime we fallback to poll() */ #include "replace.h" -#include "system/filesys.h" -#include "system/select.h" #include "tevent.h" #include "tevent_util.h" #include "tevent_internal.h" +#if 0 struct std_event_context { /* a pointer back to the generic event_context */ struct tevent_context *ev; @@ -589,10 +589,184 @@ static const struct tevent_ops std_event_ops = { .loop_once = std_event_loop_once, .loop_wait = tevent_common_loop_wait, }; +#endif + +struct std_event_glue { + const struct tevent_ops *epoll_ops; + const struct tevent_ops *poll_ops; + struct tevent_ops *glue_ops; + bool fallback_replay; +}; + +static int std_event_context_init(struct tevent_context *ev); + +static const struct tevent_ops std_event_ops = { + .context_init = std_event_context_init, +}; + +/* + If this function gets called. epoll failed at runtime. + Move us to using poll instead. If we return false here, + caller should abort(). +*/ +static bool std_fallback_to_poll(struct tevent_context *ev, bool replay) +{ + void *glue_ptr = talloc_parent(ev->ops); + struct std_event_glue *glue = + talloc_get_type_abort(glue_ptr, + struct std_event_glue); + int ret; + struct tevent_fd *fde; + struct tevent_fd *fde_next; + + glue->fallback_replay = replay; + + /* First switch all the ops to poll. */ + glue->epoll_ops = NULL; + TALLOC_FREE(ev->additional_data); + + /* + * Set custom_ops the same as poll. + */ + *glue->glue_ops = *glue->poll_ops; + glue->glue_ops->context_init = std_event_context_init; + /* Next initialize the poll backend. */ + ret = glue->poll_ops->context_init(ev); + if (ret != 0) { + return false; + } + + /* + * Now we have to change all the existing file descriptor + * events from the epoll backend to the poll backend. + */ + for (fde = ev->fd_events; fde; fde = fde_next) { + /* + * We must remove this fde off the ev->fd_events list. + */ + fde_next = fde->next; + + /* Remove from the ev->fd_events list. */ + DLIST_REMOVE(ev->fd_events, fde); + + /* Re-add this event as a poll backend event. */ + tevent_poll_event_add_fd_internal(ev, fde); + } + + return true; +} + +static int std_event_loop_once(struct tevent_context *ev, const char *location) +{ + void *glue_ptr = talloc_parent(ev->ops); + struct std_event_glue *glue = + talloc_get_type_abort(glue_ptr, + struct std_event_glue); + int ret; + + ret = glue->epoll_ops->loop_once(ev, location); + if (glue->epoll_ops != NULL) { + /* No fallback */ + return ret; + } + + if (!glue->fallback_replay) { + /* + * The problem happened while modifying an event. + * An event handler was triggered in this case + * and there is no need to call loop_once() again. + */ + return ret; + } + + return glue->poll_ops->loop_once(ev, location); +} + +/* + Initialize the epoll backend and allow it to call a + switch function if epoll fails at runtime. +*/ +static int std_event_context_init(struct tevent_context *ev) +{ + struct std_event_glue *glue; + int ret; + + /* + * If this is the first initialization + * we need to set up the allocated ops + * pointers. + */ + + if (ev->ops == &std_event_ops) { + glue = talloc_zero(ev, struct std_event_glue); + if (glue == NULL) { + return -1; + } + + glue->epoll_ops = tevent_find_ops_byname("epoll"); + + glue->poll_ops = tevent_find_ops_byname("poll"); + if (glue->poll_ops == NULL) { + return -1; + } + + /* + * Allocate space for our custom ops. + * Allocate as a child of our epoll_ops pointer + * so we can easily get to it using talloc_parent. + */ + glue->glue_ops = talloc_zero(glue, struct tevent_ops); + if (glue->glue_ops == NULL) { + talloc_free(glue); + return -1; + } + + ev->ops = glue->glue_ops; + } else { + void *glue_ptr = talloc_parent(ev->ops); + glue = talloc_get_type_abort(glue_ptr, struct std_event_glue); + } + + if (glue->epoll_ops != NULL) { + /* + * Set custom_ops the same as epoll, + * except re-init using std_event_context_init() + * and use std_event_loop_once() to add the + * ability to fallback to a poll backend on + * epoll runtime error. + */ + *glue->glue_ops = *glue->epoll_ops; + glue->glue_ops->context_init = std_event_context_init; + glue->glue_ops->loop_once = std_event_loop_once; + + ret = glue->epoll_ops->context_init(ev); + if (ret == -1) { + goto fallback; + } +#ifdef HAVE_EPOLL + if (!tevent_epoll_set_panic_fallback(ev, std_fallback_to_poll)) { + TALLOC_FREE(ev->additional_data); + goto fallback; + } +#endif + + return ret; + } + +fallback: + glue->epoll_ops = NULL; + + /* + * Set custom_ops the same as poll. + */ + *glue->glue_ops = *glue->poll_ops; + glue->glue_ops->context_init = std_event_context_init; + + return glue->poll_ops->context_init(ev); +} _PRIVATE_ bool tevent_standard_init(void) { return tevent_register_backend("standard", &std_event_ops); } - -- 1.8.1.3 From a8cba6dae23156e967531f5ae0dd6dcd789ce9a8 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Mon, 11 Feb 2013 11:42:08 -0800 Subject: [PATCH 23/76] tevent: Remove the previous "standard" tevent backend implementation. This was a horrible hybrid of duplicated epoll and select() code. Signed-off-by: Jeremy Allison Reviewed-by: Stefan Metzmacher Autobuild-User(master): Jeremy Allison Autobuild-Date(master): Thu Feb 14 22:40:30 CET 2013 on sn-devel-104 --- lib/tevent/tevent_standard.c | 557 +------------------------------------------ 1 file changed, 1 insertion(+), 556 deletions(-) diff --git a/lib/tevent/tevent_standard.c b/lib/tevent/tevent_standard.c index 0c72226..d7a5bd7 100644 --- a/lib/tevent/tevent_standard.c +++ b/lib/tevent/tevent_standard.c @@ -1,8 +1,7 @@ /* Unix SMB/CIFS implementation. main select loop and event handling - Copyright (C) Andrew Tridgell 2003-2005 - Copyright (C) Stefan Metzmacher 2005-2013 + Copyright (C) Stefan Metzmacher 2013 Copyright (C) Jeremy Allison 2013 ** NOTE! The following LGPL license applies to the tevent @@ -37,560 +36,6 @@ #include "tevent_util.h" #include "tevent_internal.h" -#if 0 -struct std_event_context { - /* a pointer back to the generic event_context */ - struct tevent_context *ev; - - /* the maximum file descriptor number in fd_events */ - int maxfd; - - /* information for exiting from the event loop */ - int exit_code; - - /* when using epoll this is the handle from epoll_create */ - int epoll_fd; - - /* our pid at the time the epoll_fd was created */ - pid_t pid; -}; - -/* use epoll if it is available */ -#if HAVE_EPOLL -/* - called when a epoll call fails, and we should fallback - to using select -*/ -static void epoll_fallback_to_select(struct std_event_context *std_ev, const char *reason) -{ - tevent_debug(std_ev->ev, TEVENT_DEBUG_FATAL, - "%s (%s) - falling back to select()\n", - reason, strerror(errno)); - close(std_ev->epoll_fd); - std_ev->epoll_fd = -1; - talloc_set_destructor(std_ev, NULL); -} - -/* - map from TEVENT_FD_* to EPOLLIN/EPOLLOUT -*/ -static uint32_t epoll_map_flags(uint16_t flags) -{ - uint32_t ret = 0; - if (flags & TEVENT_FD_READ) ret |= (EPOLLIN | EPOLLERR | EPOLLHUP); - if (flags & TEVENT_FD_WRITE) ret |= (EPOLLOUT | EPOLLERR | EPOLLHUP); - return ret; -} - -/* - free the epoll fd -*/ -static int epoll_ctx_destructor(struct std_event_context *std_ev) -{ - if (std_ev->epoll_fd != -1) { - close(std_ev->epoll_fd); - } - std_ev->epoll_fd = -1; - return 0; -} - -/* - init the epoll fd -*/ -static void epoll_init_ctx(struct std_event_context *std_ev) -{ - std_ev->epoll_fd = epoll_create(64); - if (std_ev->epoll_fd == -1) { - tevent_debug(std_ev->ev, TEVENT_DEBUG_FATAL, - "Failed to create epoll handle.\n"); - return; - } - - if (!ev_set_close_on_exec(std_ev->epoll_fd)) { - tevent_debug(std_ev->ev, TEVENT_DEBUG_WARNING, - "Failed to set close-on-exec, file descriptor may be leaked to children.\n"); - } - - std_ev->pid = getpid(); - talloc_set_destructor(std_ev, epoll_ctx_destructor); -} - -static void epoll_add_event(struct std_event_context *std_ev, struct tevent_fd *fde); - -/* - reopen the epoll handle when our pid changes - see http://junkcode.samba.org/ftp/unpacked/junkcode/epoll_fork.c for an - demonstration of why this is needed - */ -static void epoll_check_reopen(struct std_event_context *std_ev) -{ - struct tevent_fd *fde; - - if (std_ev->pid == getpid()) { - return; - } - - close(std_ev->epoll_fd); - std_ev->epoll_fd = epoll_create(64); - if (std_ev->epoll_fd == -1) { - tevent_debug(std_ev->ev, TEVENT_DEBUG_FATAL, - "Failed to recreate epoll handle after fork\n"); - return; - } - - if (!ev_set_close_on_exec(std_ev->epoll_fd)) { - tevent_debug(std_ev->ev, TEVENT_DEBUG_WARNING, - "Failed to set close-on-exec, file descriptor may be leaked to children.\n"); - } - - std_ev->pid = getpid(); - for (fde=std_ev->ev->fd_events;fde;fde=fde->next) { - epoll_add_event(std_ev, fde); - } -} - -#define EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT (1<<0) -#define EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR (1<<1) -#define EPOLL_ADDITIONAL_FD_FLAG_GOT_ERROR (1<<2) - -/* - add the epoll event to the given fd_event -*/ -static void epoll_add_event(struct std_event_context *std_ev, struct tevent_fd *fde) -{ - struct epoll_event event; - if (std_ev->epoll_fd == -1) return; - - fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; - - /* if we don't want events yet, don't add an epoll_event */ - if (fde->flags == 0) return; - - ZERO_STRUCT(event); - event.events = epoll_map_flags(fde->flags); - event.data.ptr = fde; - if (epoll_ctl(std_ev->epoll_fd, EPOLL_CTL_ADD, fde->fd, &event) != 0) { - epoll_fallback_to_select(std_ev, "EPOLL_CTL_ADD failed"); - } - fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; - - /* only if we want to read we want to tell the event handler about errors */ - if (fde->flags & TEVENT_FD_READ) { - fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; - } -} - -/* - delete the epoll event for given fd_event -*/ -static void epoll_del_event(struct std_event_context *std_ev, struct tevent_fd *fde) -{ - struct epoll_event event; - if (std_ev->epoll_fd == -1) return; - - fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; - - /* if there's no epoll_event, we don't need to delete it */ - if (!(fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT)) return; - - ZERO_STRUCT(event); - event.events = epoll_map_flags(fde->flags); - event.data.ptr = fde; - epoll_ctl(std_ev->epoll_fd, EPOLL_CTL_DEL, fde->fd, &event); - fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; -} - -/* - change the epoll event to the given fd_event -*/ -static void epoll_mod_event(struct std_event_context *std_ev, struct tevent_fd *fde) -{ - struct epoll_event event; - if (std_ev->epoll_fd == -1) return; - - fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; - - ZERO_STRUCT(event); - event.events = epoll_map_flags(fde->flags); - event.data.ptr = fde; - if (epoll_ctl(std_ev->epoll_fd, EPOLL_CTL_MOD, fde->fd, &event) != 0) { - epoll_fallback_to_select(std_ev, "EPOLL_CTL_MOD failed"); - } - - /* only if we want to read we want to tell the event handler about errors */ - if (fde->flags & TEVENT_FD_READ) { - fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; - } -} - -static void epoll_change_event(struct std_event_context *std_ev, struct tevent_fd *fde) -{ - bool got_error = (fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_GOT_ERROR); - bool want_read = (fde->flags & TEVENT_FD_READ); - bool want_write= (fde->flags & TEVENT_FD_WRITE); - - if (std_ev->epoll_fd == -1) return; - - fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; - - /* there's already an event */ - if (fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT) { - if (want_read || (want_write && !got_error)) { - epoll_mod_event(std_ev, fde); - return; - } - /* - * if we want to match the select behavior, we need to remove the epoll_event - * when the caller isn't interested in events. - * - * this is because epoll reports EPOLLERR and EPOLLHUP, even without asking for them - */ - epoll_del_event(std_ev, fde); - return; - } - - /* there's no epoll_event attached to the fde */ - if (want_read || (want_write && !got_error)) { - epoll_add_event(std_ev, fde); - return; - } -} - -/* - event loop handling using epoll -*/ -static int epoll_event_loop(struct std_event_context *std_ev, struct timeval *tvalp) -{ - int ret, i; -#define MAXEVENTS 1 - struct epoll_event events[MAXEVENTS]; - int timeout = -1; - - if (std_ev->epoll_fd == -1) return -1; - - if (tvalp) { - /* it's better to trigger timed events a bit later than to early */ - timeout = ((tvalp->tv_usec+999) / 1000) + (tvalp->tv_sec*1000); - } - - if (std_ev->ev->signal_events && - tevent_common_check_signal(std_ev->ev)) { - return 0; - } - - tevent_trace_point_callback(std_ev->ev, TEVENT_TRACE_BEFORE_WAIT); - ret = epoll_wait(std_ev->epoll_fd, events, MAXEVENTS, timeout); - tevent_trace_point_callback(std_ev->ev, TEVENT_TRACE_AFTER_WAIT); - - if (ret == -1 && errno == EINTR && std_ev->ev->signal_events) { - if (tevent_common_check_signal(std_ev->ev)) { - return 0; - } - } - - if (ret == -1 && errno != EINTR) { - epoll_fallback_to_select(std_ev, "epoll_wait() failed"); - return -1; - } - - if (ret == 0 && tvalp) { - /* we don't care about a possible delay here */ - tevent_common_loop_timer_delay(std_ev->ev); - return 0; - } - - for (i=0;iadditional_flags |= EPOLL_ADDITIONAL_FD_FLAG_GOT_ERROR; - /* - * if we only wait for TEVENT_FD_WRITE, we should not tell the - * event handler about it, and remove the epoll_event, - * as we only report errors when waiting for read events, - * to match the select() behavior - */ - if (!(fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR)) { - epoll_del_event(std_ev, fde); - continue; - } - flags |= TEVENT_FD_READ; - } - if (events[i].events & EPOLLIN) flags |= TEVENT_FD_READ; - if (events[i].events & EPOLLOUT) flags |= TEVENT_FD_WRITE; - if (flags) { - fde->handler(std_ev->ev, fde, flags, fde->private_data); - break; - } - } - - return 0; -} -#else -#define epoll_init_ctx(std_ev) -#define epoll_add_event(std_ev,fde) -#define epoll_del_event(std_ev,fde) -#define epoll_change_event(std_ev,fde) -#define epoll_event_loop(std_ev,tvalp) (-1) -#define epoll_check_reopen(std_ev) -#endif - -/* - create a std_event_context structure. -*/ -static int std_event_context_init(struct tevent_context *ev) -{ - struct std_event_context *std_ev; - - /* - * We might be called during tevent_re_initialise() - * which means we need to free our old additional_data. - */ - TALLOC_FREE(ev->additional_data); - - std_ev = talloc_zero(ev, struct std_event_context); - if (!std_ev) return -1; - std_ev->ev = ev; - std_ev->epoll_fd = -1; - - epoll_init_ctx(std_ev); - - ev->additional_data = std_ev; - return 0; -} - -/* - recalculate the maxfd -*/ -static void calc_maxfd(struct std_event_context *std_ev) -{ - struct tevent_fd *fde; - - std_ev->maxfd = 0; - for (fde = std_ev->ev->fd_events; fde; fde = fde->next) { - if (fde->fd > std_ev->maxfd) { - std_ev->maxfd = fde->fd; - } - } -} - - -/* to mark the ev->maxfd invalid - * this means we need to recalculate it - */ -#define EVENT_INVALID_MAXFD (-1) - -/* - destroy an fd_event -*/ -static int std_event_fd_destructor(struct tevent_fd *fde) -{ - struct tevent_context *ev = fde->event_ctx; - struct std_event_context *std_ev = NULL; - - if (ev) { - std_ev = talloc_get_type(ev->additional_data, - struct std_event_context); - - epoll_check_reopen(std_ev); - - if (std_ev->maxfd == fde->fd) { - std_ev->maxfd = EVENT_INVALID_MAXFD; - } - - epoll_del_event(std_ev, fde); - } - - return tevent_common_fd_destructor(fde); -} - -/* - add a fd based event - return NULL on failure (memory allocation error) -*/ -static struct tevent_fd *std_event_add_fd(struct tevent_context *ev, TALLOC_CTX *mem_ctx, - int fd, uint16_t flags, - tevent_fd_handler_t handler, - void *private_data, - const char *handler_name, - const char *location) -{ - struct std_event_context *std_ev = talloc_get_type(ev->additional_data, - struct std_event_context); - struct tevent_fd *fde; - - epoll_check_reopen(std_ev); - - fde = tevent_common_add_fd(ev, mem_ctx, fd, flags, - handler, private_data, - handler_name, location); - if (!fde) return NULL; - - if ((std_ev->maxfd != EVENT_INVALID_MAXFD) - && (fde->fd > std_ev->maxfd)) { - std_ev->maxfd = fde->fd; - } - talloc_set_destructor(fde, std_event_fd_destructor); - - epoll_add_event(std_ev, fde); - - return fde; -} - -/* - set the fd event flags -*/ -static void std_event_set_fd_flags(struct tevent_fd *fde, uint16_t flags) -{ - struct tevent_context *ev; - struct std_event_context *std_ev; - - if (fde->flags == flags) return; - - ev = fde->event_ctx; - std_ev = talloc_get_type(ev->additional_data, struct std_event_context); - - fde->flags = flags; - - epoll_check_reopen(std_ev); - - epoll_change_event(std_ev, fde); -} - -/* - event loop handling using select() -*/ -static int std_event_loop_select(struct std_event_context *std_ev, struct timeval *tvalp) -{ - fd_set r_fds, w_fds; - struct tevent_fd *fde; - int selrtn; - - /* we maybe need to recalculate the maxfd */ - if (std_ev->maxfd == EVENT_INVALID_MAXFD) { - calc_maxfd(std_ev); - } - - FD_ZERO(&r_fds); - FD_ZERO(&w_fds); - - /* setup any fd events */ - for (fde = std_ev->ev->fd_events; fde; fde = fde->next) { - if (fde->fd < 0 || fde->fd >= FD_SETSIZE) { - std_ev->exit_code = EBADF; - return -1; - } - if (fde->flags & TEVENT_FD_READ) { - FD_SET(fde->fd, &r_fds); - } - if (fde->flags & TEVENT_FD_WRITE) { - FD_SET(fde->fd, &w_fds); - } - } - - if (std_ev->ev->signal_events && - tevent_common_check_signal(std_ev->ev)) { - return 0; - } - - selrtn = select(std_ev->maxfd+1, &r_fds, &w_fds, NULL, tvalp); - - if (selrtn == -1 && errno == EINTR && - std_ev->ev->signal_events) { - tevent_common_check_signal(std_ev->ev); - return 0; - } - - if (selrtn == -1 && errno == EBADF) { - /* the socket is dead! this should never - happen as the socket should have first been - made readable and that should have removed - the event, so this must be a bug. This is a - fatal error. */ - tevent_debug(std_ev->ev, TEVENT_DEBUG_FATAL, - "ERROR: EBADF on std_event_loop_once\n"); - std_ev->exit_code = EBADF; - return -1; - } - - if (selrtn == 0 && tvalp) { - /* we don't care about a possible delay here */ - tevent_common_loop_timer_delay(std_ev->ev); - return 0; - } - - if (selrtn > 0) { - /* at least one file descriptor is ready - check - which ones and call the handler, being careful to allow - the handler to remove itself when called */ - for (fde = std_ev->ev->fd_events; fde; fde = fde->next) { - uint16_t flags = 0; - - if (FD_ISSET(fde->fd, &r_fds)) flags |= TEVENT_FD_READ; - if (FD_ISSET(fde->fd, &w_fds)) flags |= TEVENT_FD_WRITE; - if (flags & fde->flags) { - fde->handler(std_ev->ev, fde, flags, fde->private_data); - break; - } - } - } - - return 0; -} - -/* - do a single event loop using the events defined in ev -*/ -static int std_event_loop_once(struct tevent_context *ev, const char *location) -{ - struct std_event_context *std_ev = talloc_get_type(ev->additional_data, - struct std_event_context); - struct timeval tval; - - if (ev->signal_events && - tevent_common_check_signal(ev)) { - return 0; - } - - if (ev->immediate_events && - tevent_common_loop_immediate(ev)) { - return 0; - } - - tval = tevent_common_loop_timer_delay(ev); - if (tevent_timeval_is_zero(&tval)) { - return 0; - } - - epoll_check_reopen(std_ev); - - if (epoll_event_loop(std_ev, &tval) == 0) { - return 0; - } - - return std_event_loop_select(std_ev, &tval); -} - -static const struct tevent_ops std_event_ops = { - .context_init = std_event_context_init, - .add_fd = std_event_add_fd, - .set_fd_close_fn = tevent_common_fd_set_close_fn, - .get_fd_flags = tevent_common_fd_get_flags, - .set_fd_flags = std_event_set_fd_flags, - .add_timer = tevent_common_add_timer, - .schedule_immediate = tevent_common_schedule_immediate, - .add_signal = tevent_common_add_signal, - .loop_once = std_event_loop_once, - .loop_wait = tevent_common_loop_wait, -}; -#endif - struct std_event_glue { const struct tevent_ops *epoll_ops; const struct tevent_ops *poll_ops; -- 1.8.1.3 From 723fe47d487c64ba0227fad3e6aa8dbcff9eb307 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 1 Mar 2013 09:27:29 +0100 Subject: [PATCH 24/76] s3:lib/events: make use of tevent_common_loop_timer_delay() This is critical as we also use tevent_common_add_timer(). And if the common code does internal changes, it may affects both tevent_common_add_timer() and tevent_common_loop_timer_delay() together! Without this we rely on a specific behavior and version of tevent. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- source3/lib/events.c | 36 ++++++------------------------------ 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/source3/lib/events.c b/source3/lib/events.c index cafe598..947bcd4 100644 --- a/source3/lib/events.c +++ b/source3/lib/events.c @@ -182,7 +182,6 @@ bool run_events_poll(struct tevent_context *ev, int pollrtn, struct tevent_poll_private *state; int *pollfd_idx; struct tevent_fd *fde; - struct timeval now; if (ev->signal_events && tevent_common_check_signal(ev)) { @@ -194,37 +193,14 @@ bool run_events_poll(struct tevent_context *ev, int pollrtn, return true; } - GetTimeOfDay(&now); - - if ((ev->timer_events != NULL) - && (timeval_compare(&now, &ev->timer_events->next_event) >= 0)) { - /* this older events system did not auto-free timed - events on running them, and had a race condition - where the event could be called twice if the - talloc_free of the te happened after the callback - made a call which invoked the event loop. To avoid - this while still allowing old code which frees the - te, we need to create a temporary context which - will be used to ensure the te is freed. We also - remove the te from the timed event list before we - call the handler, to ensure we can't loop */ - - struct tevent_timer *te = ev->timer_events; - TALLOC_CTX *tmp_ctx = talloc_new(ev); - - DEBUG(10, ("Running timed event \"%s\" %p\n", - ev->timer_events->handler_name, ev->timer_events)); - - DLIST_REMOVE(ev->timer_events, te); - talloc_steal(tmp_ctx, te); - - te->handler(ev, te, now, te->private_data); + if (pollrtn <= 0) { + struct timeval tval; - talloc_free(tmp_ctx); - return true; - } + tval = tevent_common_loop_timer_delay(ev); + if (tevent_timeval_is_zero(&tval)) { + return true; + } - if (pollrtn <= 0) { /* * No fd ready */ -- 1.8.1.3 From dd76af4761380d85c23aca242734b0ff61567f29 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 22 Feb 2013 14:20:07 +0100 Subject: [PATCH 25/76] wafsamba: add CHECK_VALUEOF() helper Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- buildtools/wafsamba/samba_autoconf.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/buildtools/wafsamba/samba_autoconf.py b/buildtools/wafsamba/samba_autoconf.py index 76316d2..65c66f3 100644 --- a/buildtools/wafsamba/samba_autoconf.py +++ b/buildtools/wafsamba/samba_autoconf.py @@ -323,7 +323,25 @@ def CHECK_SIZEOF(conf, vars, headers=None, define=None): ret = False return ret - +@conf +def CHECK_VALUEOF(conf, v, headers=None, define=None): + '''check the value of a variable/define''' + ret = True + v_define = define + if v_define is None: + v_define = 'VALUEOF_%s' % v.upper().replace(' ', '_') + if CHECK_CODE(conf, + 'printf("%%u", (unsigned)(%s))' % v, + define=v_define, + execute=True, + define_ret=True, + quote=False, + headers=headers, + local_include=False, + msg="Checking value of %s" % v): + return int(conf.env[v_define]) + + return None @conf def CHECK_CODE(conf, code, define, -- 1.8.1.3 From 2e1aad8ebc3be47d6b587f655f64e27e6d03d837 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 22 Feb 2013 14:53:07 +0100 Subject: [PATCH 26/76] lib/replace: add AC_CHECK_VALUEOF() macro Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/replace/libreplace_macros.m4 | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/replace/libreplace_macros.m4 b/lib/replace/libreplace_macros.m4 index 42b8847..f3753c4 100644 --- a/lib/replace/libreplace_macros.m4 +++ b/lib/replace/libreplace_macros.m4 @@ -330,3 +330,18 @@ fi AC_DEFINE_UNQUOTED($ac_tr_hdr, 1, [Define if you have type `$1']) fi ]) + +# AC_CHECK_VALUEOF(TYPE, [INCLUDES = DEFAULT-INCLUDES]) +# --------------------------------------------------------------- +AC_DEFUN([AC_CHECK_VALUEOF], +[AS_LITERAL_IF(m4_translit([[$1]], [*], [p]), [], + [m4_fatal([$0: requires literal arguments])])]dnl +[ +_AC_CACHE_CHECK_INT([value of $1], [AS_TR_SH([ac_cv_valueof_$1])], + [(long int) ($1)], + [AC_INCLUDES_DEFAULT([$2])], + []) + +AC_DEFINE_UNQUOTED(AS_TR_CPP(valueof_$1), $AS_TR_SH([ac_cv_valueof_$1]), + [The value of `$1'.]) +])# AC_CHECK_VALUEOF -- 1.8.1.3 From 2749a93db2803d95932daf4de621f77b92ddf977 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 22 Feb 2013 14:26:16 +0100 Subject: [PATCH 27/76] tevent: define TEVENT_NUM_SIGNALS based on configure checks Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/libtevent.m4 | 29 +++++++++++++++++++++++++++++ lib/tevent/tevent_signal.c | 2 -- lib/tevent/wscript | 17 +++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/lib/tevent/libtevent.m4 b/lib/tevent/libtevent.m4 index 5c5969b..4d4fd2b 100644 --- a/lib/tevent/libtevent.m4 +++ b/lib/tevent/libtevent.m4 @@ -39,6 +39,35 @@ if test x"$ac_cv_header_sys_epoll_h" = x"yes" -a x"$ac_cv_func_epoll_create" = x AC_DEFINE(HAVE_EPOLL, 1, [Whether epoll available]) fi +tevent_num_signals_includes="$ac_includes_default +#include +" +tevent_num_signals=64 +AC_CHECK_VALUEOF(NSIG, [$tevent_num_signals_includes]) +v=$ac_cv_valueof_NSIG +test -n "$v" && test "$v" -gt "$tevent_num_signals" && { + tevent_num_signals=$v +} +AC_CHECK_VALUEOF(_NSIG, [$tevent_num_signals_includes]) +v=$ac_cv_valueof__NSIG +test -n "$v" && test "$v" -gt "$tevent_num_signals" && { + tevent_num_signals=$v +} +AC_CHECK_VALUEOF(SIGRTMAX, [$tevent_num_signals_includes]) +v=$ac_cv_valueof_SIGRTMAX +test -n "$v" && test "$v" -gt "$tevent_num_signals" && { + tevent_num_signals=$v +} +AC_CHECK_VALUEOF(SIGRTMIN, [$tevent_num_signals_includes]) +v=$ac_cv_valueof_SIGRTMIN +test -n "$v" && { + v=`expr $v + $v` +} +test -n "$v" && test "$v" -gt "$tevent_num_signals" && { + tevent_num_signals=$v +} +AC_DEFINE_UNQUOTED(TEVENT_NUM_SIGNALS, $tevent_num_signals, [Max signal number value]) + if test x"$VERSIONSCRIPT" != "x"; then EXPORTSFILE=tevent.exports AC_SUBST(EXPORTSFILE) diff --git a/lib/tevent/tevent_signal.c b/lib/tevent/tevent_signal.c index 4be54f2..0b637dc 100644 --- a/lib/tevent/tevent_signal.c +++ b/lib/tevent/tevent_signal.c @@ -30,8 +30,6 @@ #include "tevent_internal.h" #include "tevent_util.h" -#define TEVENT_NUM_SIGNALS 64 - /* maximum number of SA_SIGINFO signals to hold in the queue. NB. This *MUST* be a power of 2, in order for the ring buffer wrap to work correctly. Thanks to Petr Vandrovec diff --git a/lib/tevent/wscript b/lib/tevent/wscript index 684286d..c62c3de 100755 --- a/lib/tevent/wscript +++ b/lib/tevent/wscript @@ -44,6 +44,23 @@ def configure(conf): if conf.CHECK_FUNCS('epoll_create', headers='sys/epoll.h'): conf.DEFINE('HAVE_EPOLL', 1) + tevent_num_signals = 64 + v = conf.CHECK_VALUEOF('NSIG', headers='signal.h') + if v is not None: + tevent_num_signals = max(tevent_num_signals, v) + v = conf.CHECK_VALUEOF('_NSIG', headers='signal.h') + if v is not None: + tevent_num_signals = max(tevent_num_signals, v) + v = conf.CHECK_VALUEOF('SIGRTMAX', headers='signal.h') + if v is not None: + tevent_num_signals = max(tevent_num_signals, v) + v = conf.CHECK_VALUEOF('SIGRTMIN', headers='signal.h') + if v is not None: + tevent_num_signals = max(tevent_num_signals, v*2) + + if not conf.CONFIG_SET('USING_SYSTEM_TEVENT'): + conf.DEFINE('TEVENT_NUM_SIGNALS', tevent_num_signals) + conf.env.disable_python = getattr(Options.options, 'disable_python', False) if not conf.env.disable_python: -- 1.8.1.3 From 946d8e36638e6bdafcce0b6a8496dcbf485bff7c Mon Sep 17 00:00:00 2001 From: "Galen.Liu" Date: Mon, 4 Feb 2013 07:43:01 +0100 Subject: [PATCH 28/76] tevent: fix --disable-python cause configure fails (bug #8718) When we disable python, it will cause build fail! so, the patch will fix it. Reviewed-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/wscript | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tevent/wscript b/lib/tevent/wscript index c62c3de..71b1f61 100755 --- a/lib/tevent/wscript +++ b/lib/tevent/wscript @@ -109,7 +109,7 @@ def build(bld): pc_files='tevent.pc', private_library=private_library) - if not bld.CONFIG_SET('USING_SYSTEM_PYTEVENT'): + if not bld.CONFIG_SET('USING_SYSTEM_PYTEVENT') and not bld.env.disable_python: bld.SAMBA_PYTHON('pytevent', 'pytevent.c', deps='tevent', -- 1.8.1.3 From 2240b606053c0c0887fba3734be259f73e77792a Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Tue, 26 Feb 2013 15:51:38 +0100 Subject: [PATCH 29/76] tevent: fix a comment typo in tevent_epoll.c Signed-off-by: Michael Adam Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 8696215..61a8525 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -317,7 +317,7 @@ static int epoll_event_loop(struct epoll_event_context *epoll_ev, struct timeval if (epoll_ev->epoll_fd == -1) return -1; if (tvalp) { - /* it's better to trigger timed events a bit later than to early */ + /* it's better to trigger timed events a bit later than too early */ timeout = ((tvalp->tv_usec+999) / 1000) + (tvalp->tv_sec*1000); } -- 1.8.1.3 From 3cc1e66a0c2e40fd3192506e6284cb7f0ac70b67 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 22 Feb 2013 21:48:11 +0100 Subject: [PATCH 30/76] tevent: fix some compiler warnings in testsuite.c Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/testsuite.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/tevent/testsuite.c b/lib/tevent/testsuite.c index 3d2a79a..e053682 100644 --- a/lib/tevent/testsuite.c +++ b/lib/tevent/testsuite.c @@ -74,7 +74,9 @@ static bool test_event_context(struct torture_context *test, #ifdef SA_RESTART struct tevent_signal *se1 = NULL; #endif +#ifdef SA_RESETHAND struct tevent_signal *se2 = NULL; +#endif #ifdef SA_SIGINFO struct tevent_signal *se3 = NULL; #endif @@ -88,7 +90,8 @@ static bool test_event_context(struct torture_context *test, return true; } - torture_comment(test, "Testing event backend '%s'\n", backend); + torture_comment(test, "backend '%s' - %s\n", + backend, __FUNCTION__); /* reset globals */ fde_count = 0; @@ -105,12 +108,15 @@ static bool test_event_context(struct torture_context *test, #ifdef SA_RESTART se1 = tevent_add_signal(ev_ctx, ev_ctx, SIGALRM, SA_RESTART, count_handler, &alarm_count); + torture_assert(test, se1 != NULL, "failed to setup se1"); #endif #ifdef SA_RESETHAND se2 = tevent_add_signal(ev_ctx, ev_ctx, SIGALRM, SA_RESETHAND, count_handler, &alarm_count); + torture_assert(test, se2 != NULL, "failed to setup se2"); #endif #ifdef SA_SIGINFO se3 = tevent_add_signal(ev_ctx, ev_ctx, SIGUSR1, SA_SIGINFO, count_handler, &info_count); + torture_assert(test, se3 != NULL, "failed to setup se3"); #endif write(fd[1], &c, 1); @@ -141,6 +147,14 @@ static bool test_event_context(struct torture_context *test, torture_assert_int_equal(test, alarm_count, 1+fde_count, "alarm count mismatch"); +#ifdef SA_RESETHAND + /* + * we do not call talloc_free(se2) + * because it is already gone, + * after triggering the event handler. + */ +#endif + #ifdef SA_SIGINFO talloc_free(se3); torture_assert_int_equal(test, info_count, fde_count, "info count mismatch"); -- 1.8.1.3 From 86fe30bbc036f08d172c2b9e7c5afcbcba390313 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 17 Feb 2013 16:36:25 +0100 Subject: [PATCH 31/76] tevent: fix compiler warning in tevent_context_init_byname() Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tevent/tevent.c b/lib/tevent/tevent.c index aa758de..3b273d6 100644 --- a/lib/tevent/tevent.c +++ b/lib/tevent/tevent.c @@ -273,7 +273,7 @@ struct tevent_context *tevent_context_init_ops(TALLOC_CTX *mem_ctx, struct tevent_context *tevent_context_init_byname(TALLOC_CTX *mem_ctx, const char *name) { - struct tevent_ops *ops; + const struct tevent_ops *ops; ops = tevent_find_ops_byname(name); if (ops == NULL) { -- 1.8.1.3 From 4953509de64145c9901867c40094d8d1a3c02d39 Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Tue, 26 Feb 2013 15:54:57 +0100 Subject: [PATCH 32/76] tevent: add trace points BEFORE_LOOP_ONCE and AFTER_LOOP_ONCE The define TEVENT_HAS_LOOP_ONCE_TRACE_POINTS can be used to detect the new feature, without writing configure tests. Pair-Programmed-With: Stefan Metzmacher Signed-off-by: Michael Adam Signed-off-by: Stefan Metzmacher --- lib/tevent/testsuite.c | 3 +++ lib/tevent/tevent.c | 4 ++++ lib/tevent/tevent.h | 11 +++++++++++ 3 files changed, 18 insertions(+) diff --git a/lib/tevent/testsuite.c b/lib/tevent/testsuite.c index e053682..142ba4f 100644 --- a/lib/tevent/testsuite.c +++ b/lib/tevent/testsuite.c @@ -194,6 +194,9 @@ static void test_event_threaded_trace(enum tevent_trace_point point, case TEVENT_TRACE_AFTER_WAIT: test_event_threaded_lock(); break; + case TEVENT_TRACE_BEFORE_LOOP_ONCE: + case TEVENT_TRACE_AFTER_LOOP_ONCE: + break; } } diff --git a/lib/tevent/tevent.c b/lib/tevent/tevent.c index 3b273d6..63d5f15 100644 --- a/lib/tevent/tevent.c +++ b/lib/tevent/tevent.c @@ -525,7 +525,9 @@ int _tevent_loop_once(struct tevent_context *ev, const char *location) } } + tevent_trace_point_callback(ev, TEVENT_TRACE_BEFORE_LOOP_ONCE); ret = ev->ops->loop_once(ev, location); + tevent_trace_point_callback(ev, TEVENT_TRACE_AFTER_LOOP_ONCE); if (ev->nesting.level > 0) { if (ev->nesting.hook_fn) { @@ -585,7 +587,9 @@ int _tevent_loop_until(struct tevent_context *ev, } while (!finished(private_data)) { + tevent_trace_point_callback(ev, TEVENT_TRACE_BEFORE_LOOP_ONCE); ret = ev->ops->loop_once(ev, location); + tevent_trace_point_callback(ev, TEVENT_TRACE_AFTER_LOOP_ONCE); if (ret != 0) { break; } diff --git a/lib/tevent/tevent.h b/lib/tevent/tevent.h index dc61912..6b4d371 100644 --- a/lib/tevent/tevent.h +++ b/lib/tevent/tevent.h @@ -524,6 +524,17 @@ enum tevent_trace_point { * Corresponds to a trace point just after waiting */ TEVENT_TRACE_AFTER_WAIT, +#define TEVENT_HAS_LOOP_ONCE_TRACE_POINTS 1 + /** + * Corresponds to a trace point just before calling + * the loop_once() backend function. + */ + TEVENT_TRACE_BEFORE_LOOP_ONCE, + /** + * Corresponds to a trace point right after the + * loop_once() backend function has returned. + */ + TEVENT_TRACE_AFTER_LOOP_ONCE, }; typedef void (*tevent_trace_callback_t)(enum tevent_trace_point, -- 1.8.1.3 From 5a5253b2e13f95f957be8c0d1a3e778859151d78 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 15 Feb 2013 11:24:59 +0100 Subject: [PATCH 33/76] tevent: remember the errno from select(), poll() and epoll_wait() Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 6 ++++-- lib/tevent/tevent_poll.c | 4 +++- lib/tevent/tevent_select.c | 6 ++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 61a8525..99d4806 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -313,6 +313,7 @@ static int epoll_event_loop(struct epoll_event_context *epoll_ev, struct timeval #define MAXEVENTS 1 struct epoll_event events[MAXEVENTS]; int timeout = -1; + int wait_errno; if (epoll_ev->epoll_fd == -1) return -1; @@ -328,15 +329,16 @@ static int epoll_event_loop(struct epoll_event_context *epoll_ev, struct timeval tevent_trace_point_callback(epoll_ev->ev, TEVENT_TRACE_BEFORE_WAIT); ret = epoll_wait(epoll_ev->epoll_fd, events, MAXEVENTS, timeout); + wait_errno = errno; tevent_trace_point_callback(epoll_ev->ev, TEVENT_TRACE_AFTER_WAIT); - if (ret == -1 && errno == EINTR && epoll_ev->ev->signal_events) { + if (ret == -1 && wait_errno == EINTR && epoll_ev->ev->signal_events) { if (tevent_common_check_signal(epoll_ev->ev)) { return 0; } } - if (ret == -1 && errno != EINTR) { + if (ret == -1 && wait_errno != EINTR) { epoll_panic(epoll_ev, "epoll_wait() failed", true); return -1; } diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index 89b3bbc..81a7176 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -444,6 +444,7 @@ static int poll_event_loop_poll(struct tevent_context *ev, int timeout = -1; unsigned first_fd; unsigned i; + int poll_errno; if (ev->signal_events && tevent_common_check_signal(ev)) { return 0; @@ -462,9 +463,10 @@ static int poll_event_loop_poll(struct tevent_context *ev, tevent_trace_point_callback(poll_ev->ev, TEVENT_TRACE_BEFORE_WAIT); pollrtn = poll(poll_ev->fds, poll_ev->num_fds, timeout); + poll_errno = errno; tevent_trace_point_callback(poll_ev->ev, TEVENT_TRACE_AFTER_WAIT); - if (pollrtn == -1 && errno == EINTR && ev->signal_events) { + if (pollrtn == -1 && poll_errno == EINTR && ev->signal_events) { tevent_common_check_signal(ev); return 0; } diff --git a/lib/tevent/tevent_select.c b/lib/tevent/tevent_select.c index 7e0c927..ffb0d18 100644 --- a/lib/tevent/tevent_select.c +++ b/lib/tevent/tevent_select.c @@ -144,6 +144,7 @@ static int select_event_loop_select(struct select_event_context *select_ev, stru fd_set r_fds, w_fds; struct tevent_fd *fde; int selrtn; + int select_errno; /* we maybe need to recalculate the maxfd */ if (select_ev->maxfd == EVENT_INVALID_MAXFD) { @@ -175,15 +176,16 @@ static int select_event_loop_select(struct select_event_context *select_ev, stru tevent_trace_point_callback(select_ev->ev, TEVENT_TRACE_BEFORE_WAIT); selrtn = select(select_ev->maxfd+1, &r_fds, &w_fds, NULL, tvalp); + select_errno = errno; tevent_trace_point_callback(select_ev->ev, TEVENT_TRACE_AFTER_WAIT); - if (selrtn == -1 && errno == EINTR && + if (selrtn == -1 && select_errno == EINTR && select_ev->ev->signal_events) { tevent_common_check_signal(select_ev->ev); return 0; } - if (selrtn == -1 && errno == EBADF) { + if (selrtn == -1 && select_errno == EBADF) { /* the socket is dead! this should never happen as the socket should have first been made readable and that should have removed -- 1.8.1.3 From 493fd141da157d26c2020d3b7216dbd8f50a5dea Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 17 Feb 2013 23:21:28 +0100 Subject: [PATCH 34/76] tevent: don't skip a fd event if the previous one was deleted during poll() In a threaded environment it can happen that an tevent_fd is talloc_free'ed while the main thread sleeps in the poll() syscall. In such a case poll_event_fd_destructor() would set poll_ev->fdes[i] = NULL. We then skip the removed event, but before we also skipped the one that was located at the end of the array. We moved it to possition 'i', but the next loop uses 'i=i+1'. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index 81a7176..68885e9 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -443,7 +443,7 @@ static int poll_event_loop_poll(struct tevent_context *ev, int pollrtn; int timeout = -1; unsigned first_fd; - unsigned i; + unsigned i, next_i; int poll_errno; if (ev->signal_events && tevent_common_check_signal(ev)) { @@ -490,11 +490,13 @@ static int poll_event_loop_poll(struct tevent_context *ev, which ones and call the handler, being careful to allow the handler to remove itself when called */ - for (i=first_fd; inum_fds; i++) { + for (i=first_fd; inum_fds; i = next_i) { struct pollfd *pfd; struct tevent_fd *fde; uint16_t flags = 0; + next_i = i + 1; + fde = poll_ev->fdes[i]; if (fde == NULL) { /* @@ -502,11 +504,16 @@ static int poll_event_loop_poll(struct tevent_context *ev, * from the arrays */ poll_ev->num_fds -= 1; + if (poll_ev->num_fds == i) { + break; + } poll_ev->fds[i] = poll_ev->fds[poll_ev->num_fds]; poll_ev->fdes[i] = poll_ev->fdes[poll_ev->num_fds]; if (poll_ev->fdes[i] != NULL) { poll_ev->fdes[i]->additional_flags = i; } + /* we have to reprocess position 'i' */ + next_i = i; continue; } -- 1.8.1.3 From c1f043694830a6c13e04c4aa3110264ce954b413 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 21 Feb 2013 14:41:24 +0100 Subject: [PATCH 35/76] tevent: make sure we cleanup the array passed to poll() after deleting an event If we don't cleanup the array passed to poll after an event was deleted, we may pass a bad file descriptor to poll(). This was found by the following test failure in Samba's autobuild, while removing the epoll support from the "standard" backend. [48/1555 in 4m37s] samba3.smbtorture_s3.plain(s3dc).LOCK4 UNEXPECTED(failure): samba3.smbtorture_s3.plain(s3dc).LOCK4.smbtorture(s3dc) REASON: _StringException: _StringException: using seed 1361530718 host=127.0.0.2 share=tmp user=metze myname=sn-devel-104 Running LOCK4 starting locktest4 Failed to create file: NT_STATUS_INVALID_HANDLE finished locktest4 TEST LOCK4 FAILED! LOCK4 took 190.492 secs Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index 68885e9..0928cbd 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -38,6 +38,7 @@ struct poll_event_context { * picked up yet by poll_event_loop_once */ struct tevent_fd *fresh; + bool deleted; /* * These two arrays are maintained together. @@ -219,6 +220,7 @@ static int poll_event_fd_destructor(struct tevent_fd *fde) ev->additional_data, struct poll_event_context); poll_ev->fdes[del_idx] = NULL; + poll_ev->deleted = true; poll_event_wake_pollthread(poll_ev); done: return tevent_common_fd_destructor(fde); @@ -363,6 +365,34 @@ static bool poll_event_setup_fresh(struct tevent_context *ev, struct tevent_fd *fde, *next; unsigned num_fresh, num_fds; + if (poll_ev->deleted) { + unsigned first_fd = (poll_ev->signal_fd != -1) ? 1 : 0; + unsigned i; + + for (i=first_fd; i < poll_ev->num_fds;) { + fde = poll_ev->fdes[i]; + if (fde != NULL) { + i++; + continue; + } + + /* + * This fde was talloc_free()'ed. Delete it + * from the arrays + */ + poll_ev->num_fds -= 1; + if (poll_ev->num_fds == i) { + break; + } + poll_ev->fds[i] = poll_ev->fds[poll_ev->num_fds]; + poll_ev->fdes[i] = poll_ev->fdes[poll_ev->num_fds]; + if (poll_ev->fdes[i] != NULL) { + poll_ev->fdes[i]->additional_flags = i; + } + } + } + poll_ev->deleted = false; + if (poll_ev->fresh == NULL) { return true; } -- 1.8.1.3 From 11b570fcdad1f5a2fa3137a9d21ff5d02986abb7 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 28 Feb 2013 17:02:45 +0100 Subject: [PATCH 36/76] tevent: use tevent_poll_event_add_fd_internal() in poll_event_add_fd() Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index 0928cbd..6e1c5c3 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -308,8 +308,7 @@ static struct tevent_fd *poll_event_add_fd(struct tevent_context *ev, fde->additional_flags = UINT64_MAX; fde->additional_data = NULL; - DLIST_ADD(poll_ev->fresh, fde); - talloc_set_destructor(fde, poll_fresh_fde_destructor); + tevent_poll_event_add_fd_internal(ev, fde); poll_event_wake_pollthread(poll_ev); /* -- 1.8.1.3 From f4e5c9fd3694402da3181514da34525e9514d622 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 28 Feb 2013 17:04:00 +0100 Subject: [PATCH 37/76] tevent: merge poll_fresh_fde_destructor() into poll_event_fd_destructor() Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index 6e1c5c3..792abef 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -219,25 +219,15 @@ static int poll_event_fd_destructor(struct tevent_fd *fde) poll_ev = talloc_get_type_abort( ev->additional_data, struct poll_event_context); - poll_ev->fdes[del_idx] = NULL; - poll_ev->deleted = true; - poll_event_wake_pollthread(poll_ev); -done: - return tevent_common_fd_destructor(fde); -} - -static int poll_fresh_fde_destructor(struct tevent_fd *fde) -{ - struct tevent_context *ev = fde->event_ctx; - struct poll_event_context *poll_ev; + if (del_idx == UINT64_MAX) { - if (ev == NULL) { + DLIST_REMOVE(poll_ev->fresh, fde); goto done; } - poll_ev = talloc_get_type_abort( - ev->additional_data, struct poll_event_context); - DLIST_REMOVE(poll_ev->fresh, fde); + poll_ev->fdes[del_idx] = NULL; + poll_ev->deleted = true; + poll_event_wake_pollthread(poll_ev); done: return tevent_common_fd_destructor(fde); } @@ -270,7 +260,7 @@ _PRIVATE_ void tevent_poll_event_add_fd_internal(struct tevent_context *ev, fde->additional_flags = UINT64_MAX; fde->additional_data = NULL; DLIST_ADD(poll_ev->fresh, fde); - talloc_set_destructor(fde, poll_fresh_fde_destructor); + talloc_set_destructor(fde, poll_event_fd_destructor); } /* @@ -454,8 +444,6 @@ static bool poll_event_setup_fresh(struct tevent_context *ev, DLIST_REMOVE(poll_ev->fresh, fde); DLIST_ADD(ev->fd_events, fde); - talloc_set_destructor(fde, poll_event_fd_destructor); - poll_ev->num_fds += 1; } return true; -- 1.8.1.3 From 9a35dcf22fc0b7d45b5f4565604e289cf03114d0 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 28 Feb 2013 17:07:28 +0100 Subject: [PATCH 38/76] tevent: maintain a list of disabled events in the poll backend We need to avoid passing pollfd.events == 0, to poll(), as it will report POLLERR and POLLHUP events, but our caller does not expect the event handler to be called. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 55 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index 792abef..9e2cf1e 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -38,6 +38,13 @@ struct poll_event_context { * picked up yet by poll_event_loop_once */ struct tevent_fd *fresh; + /* + * A DLIST for disabled fde's. + */ + struct tevent_fd *disabled; + /* + * one or more events were deleted or disabled + */ bool deleted; /* @@ -66,6 +73,12 @@ static int poll_event_context_destructor(struct poll_event_context *poll_ev) DLIST_REMOVE(poll_ev->fresh, fd); } + for (fd = poll_ev->disabled; fd; fd = fn) { + fn = fd->next; + fd->event_ctx = NULL; + DLIST_REMOVE(poll_ev->disabled, fd); + } + if (poll_ev->signal_fd == -1) { /* * Non-threaded, no signal pipe @@ -220,8 +233,10 @@ static int poll_event_fd_destructor(struct tevent_fd *fde) ev->additional_data, struct poll_event_context); if (del_idx == UINT64_MAX) { + struct tevent_fd **listp = + (struct tevent_fd **)fde->additional_data; - DLIST_REMOVE(poll_ev->fresh, fde); + DLIST_REMOVE((*listp), fde); goto done; } @@ -256,10 +271,18 @@ _PRIVATE_ void tevent_poll_event_add_fd_internal(struct tevent_context *ev, { struct poll_event_context *poll_ev = talloc_get_type_abort( ev->additional_data, struct poll_event_context); + struct tevent_fd **listp; + + if (fde->flags != 0) { + listp = &poll_ev->fresh; + } else { + listp = &poll_ev->disabled; + } fde->additional_flags = UINT64_MAX; - fde->additional_data = NULL; - DLIST_ADD(poll_ev->fresh, fde); + fde->additional_data = listp; + + DLIST_ADD((*listp), fde); talloc_set_destructor(fde, poll_event_fd_destructor); } @@ -327,11 +350,28 @@ static void poll_event_set_fd_flags(struct tevent_fd *fde, uint16_t flags) fde->flags = flags; if (idx == UINT64_MAX) { + struct tevent_fd **listp = + (struct tevent_fd **)fde->additional_data; + + /* + * We move it between the fresh and disabled lists. + */ + DLIST_REMOVE((*listp), fde); + tevent_poll_event_add_fd_internal(ev, fde); + poll_event_wake_pollthread(poll_ev); + return; + } + + if (fde->flags == 0) { /* - * poll_event_setup_fresh not yet called after this fde was - * added. We don't have to do anything to transfer the changed - * flags to the array passed to poll(2) + * We need to remove it from the array + * and move it to the disabled list. */ + poll_ev->fdes[idx] = NULL; + poll_ev->deleted = true; + DLIST_REMOVE(ev->fd_events, fde); + tevent_poll_event_add_fd_internal(ev, fde); + poll_event_wake_pollthread(poll_ev); return; } @@ -602,7 +642,8 @@ static int poll_event_loop_wait(struct tevent_context *ev, ev->timer_events || ev->immediate_events || ev->signal_events || - poll_ev->fresh) { + poll_ev->fresh || + poll_ev->disabled) { int ret; ret = _tevent_loop_once(ev, location); if (ret != 0) { -- 1.8.1.3 From b8a1dd841f4e976aa78b5be9593985b35facb98e Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 21 Feb 2013 15:41:17 +0100 Subject: [PATCH 39/76] tevent: traverse the ev->fd_event list instead of the poll_ev->fds array That has the same complexity but simplifies the code. It also seems to fix a real problem, which was found by the following test failure in Samba's autobuild, while removing the epoll support from the "standard" backend. [402/1555 in 19m8s] samba3.raw.composite(plugin_s4_dc) smbtorture 4.1.0pre1-DEVELOPERBUILD Using seed 1361531675 Testing savefile Testing parallel fetchfile with 10 ops waiting for completion maximum runtime exceeded for smbtorture - terminating UNEXPECTED(error): samba3.raw.composite.fetchfile (subunit.RemotedTestCase)(plugin_s4_dc) REASON: _StringException: _StringException: was started but never finished! UNEXPECTED(error): samba3.raw.composite.fetchfile(plugin_s4_dc) (subunit.RemotedTestCase) REASON: was started but never finished! Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index 9e2cf1e..763a139 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -499,9 +499,8 @@ static int poll_event_loop_poll(struct tevent_context *ev, ev->additional_data, struct poll_event_context); int pollrtn; int timeout = -1; - unsigned first_fd; - unsigned i, next_i; int poll_errno; + struct tevent_fd *fde = NULL; if (ev->signal_events && tevent_common_check_signal(ev)) { return 0; @@ -541,40 +540,20 @@ static int poll_event_loop_poll(struct tevent_context *ev, return 0; } - first_fd = (poll_ev->signal_fd != -1) ? 1 : 0; - /* at least one file descriptor is ready - check which ones and call the handler, being careful to allow the handler to remove itself when called */ - for (i=first_fd; inum_fds; i = next_i) { + for (fde = ev->fd_events; fde; fde = fde->next) { + unsigned idx = fde->additional_flags; struct pollfd *pfd; - struct tevent_fd *fde; uint16_t flags = 0; - next_i = i + 1; - - fde = poll_ev->fdes[i]; - if (fde == NULL) { - /* - * This fde was talloc_free()'ed. Delete it - * from the arrays - */ - poll_ev->num_fds -= 1; - if (poll_ev->num_fds == i) { - break; - } - poll_ev->fds[i] = poll_ev->fds[poll_ev->num_fds]; - poll_ev->fdes[i] = poll_ev->fdes[poll_ev->num_fds]; - if (poll_ev->fdes[i] != NULL) { - poll_ev->fdes[i]->additional_flags = i; - } - /* we have to reprocess position 'i' */ - next_i = i; + if (idx == UINT64_MAX) { continue; } - pfd = &poll_ev->fds[i]; + pfd = &poll_ev->fds[idx]; if (pfd->revents & (POLLHUP|POLLERR)) { /* If we only wait for TEVENT_FD_WRITE, we -- 1.8.1.3 From 47cad2ef05367e4128319073b1fb04767aea9023 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 22 Feb 2013 21:45:11 +0100 Subject: [PATCH 40/76] tevent: ignore POLLNVAL from poll() and disable the event Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index 763a139..867d951 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -501,6 +501,7 @@ static int poll_event_loop_poll(struct tevent_context *ev, int timeout = -1; int poll_errno; struct tevent_fd *fde = NULL; + unsigned i; if (ev->signal_events && tevent_common_check_signal(ev)) { return 0; @@ -555,6 +556,26 @@ static int poll_event_loop_poll(struct tevent_context *ev, pfd = &poll_ev->fds[idx]; + if (pfd->revents & POLLNVAL) { + /* + * the socket is dead! this should never + * happen as the socket should have first been + * made readable and that should have removed + * the event, so this must be a bug. + * + * We ignore it here to match the epoll + * behavior. + */ + tevent_debug(ev, TEVENT_DEBUG_ERROR, + "POLLNVAL on fde[%p] fd[%d] - disabling\n", + fde, pfd->fd); + poll_ev->fdes[idx] = NULL; + poll_ev->deleted = true; + DLIST_REMOVE(ev->fd_events, fde); + fde->event_ctx = NULL; + continue; + } + if (pfd->revents & (POLLHUP|POLLERR)) { /* If we only wait for TEVENT_FD_WRITE, we should not tell the event handler about it, @@ -575,7 +596,29 @@ static int poll_event_loop_poll(struct tevent_context *ev, } if (flags != 0) { fde->handler(ev, fde, flags, fde->private_data); - break; + return 0; + } + } + + for (i = 0; i < poll_ev->num_fds; i++) { + if (poll_ev->fds[i].revents & POLLNVAL) { + /* + * the socket is dead! this should never + * happen as the socket should have first been + * made readable and that should have removed + * the event, so this must be a bug or + * a race in the poll_mt usage. + */ + fde = poll_ev->fdes[i]; + tevent_debug(ev, TEVENT_DEBUG_WARNING, + "POLLNVAL on dangling fd[%d] fde[%p] - disabling\n", + poll_ev->fds[i].fd, fde); + poll_ev->fdes[i] = NULL; + poll_ev->deleted = true; + if (fde != NULL) { + DLIST_REMOVE(ev->fd_events, fde); + fde->event_ctx = NULL; + } } } -- 1.8.1.3 From 63f6ee9c498425bb2cefbfe59c81bf8233315367 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 1 Mar 2013 16:43:35 +0100 Subject: [PATCH 41/76] tevent: revalidate fde->flags after poll() This is important to avoid races between threads if the poll_mt backend is used. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index 867d951..aa4c50c 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -594,6 +594,12 @@ static int poll_event_loop_poll(struct tevent_context *ev, if (pfd->revents & POLLOUT) { flags |= TEVENT_FD_WRITE; } + /* + * Note that fde->flags could be changed when using + * the poll_mt backend together with threads, + * that why we need to check pfd->revents and fde->flags + */ + flags &= fde->flags; if (flags != 0) { fde->handler(ev, fde, flags, fde->private_data); return 0; -- 1.8.1.3 From 85c14906f056d739535a0f799568c3c5b1509a6f Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Wed, 27 Feb 2013 10:18:44 -0800 Subject: [PATCH 42/76] tevent: Fix multiple handler on the same fd bug in the tevent select backend. When we're deciding what handlers to call in the select backend, we didn't take into account the fact that the same fd may have been added into the read FD_SET and the write FD_SET but with different handlers. We must match on both the file descriptor and the flags requested before calling the handler. Signed-off-by: Jeremy Allison Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent_select.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/tevent/tevent_select.c b/lib/tevent/tevent_select.c index ffb0d18..2403263 100644 --- a/lib/tevent/tevent_select.c +++ b/lib/tevent/tevent_select.c @@ -210,8 +210,12 @@ static int select_event_loop_select(struct select_event_context *select_ev, stru for (fde = select_ev->ev->fd_events; fde; fde = fde->next) { uint16_t flags = 0; - if (FD_ISSET(fde->fd, &r_fds)) flags |= TEVENT_FD_READ; - if (FD_ISSET(fde->fd, &w_fds)) flags |= TEVENT_FD_WRITE; + if (FD_ISSET(fde->fd, &r_fds) && (fde->flags & TEVENT_FD_READ)) { + flags |= TEVENT_FD_READ; + } + if (FD_ISSET(fde->fd, &w_fds) && (fde->flags & TEVENT_FD_WRITE)) { + flags |= TEVENT_FD_WRITE; + } if (flags) { fde->handler(select_ev->ev, fde, flags, fde->private_data); break; -- 1.8.1.3 From 9741a3f2dd684dba33242816a6aa5b1790db6695 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 28 Feb 2013 14:43:55 +0100 Subject: [PATCH 43/76] tevent: debug a FATAL message on EBADF from the select backend Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_select.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/tevent/tevent_select.c b/lib/tevent/tevent_select.c index 2403263..d8aeaa7 100644 --- a/lib/tevent/tevent_select.c +++ b/lib/tevent/tevent_select.c @@ -157,6 +157,10 @@ static int select_event_loop_select(struct select_event_context *select_ev, stru /* setup any fd events */ for (fde = select_ev->ev->fd_events; fde; fde = fde->next) { if (fde->fd < 0 || fde->fd >= FD_SETSIZE) { + tevent_debug(select_ev->ev, TEVENT_DEBUG_FATAL, + "ERROR: EBADF fd[%d] >= %d " + "select_event_loop_once\n", + fde->fd, FD_SETSIZE); errno = EBADF; return -1; } -- 1.8.1.3 From 55637d4257fee4e0e74b8fcfc70bbc8d5fd54c61 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 21 Feb 2013 15:44:30 +0100 Subject: [PATCH 44/76] tevent: use DLIST_DEMOTE() before calling on fde handler after select() and poll() This makes sure we don't preferr events which are at the beginning of the list. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_poll.c | 1 + lib/tevent/tevent_select.c | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index aa4c50c..0175cae 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -601,6 +601,7 @@ static int poll_event_loop_poll(struct tevent_context *ev, */ flags &= fde->flags; if (flags != 0) { + DLIST_DEMOTE(ev->fd_events, fde, struct tevent_fd); fde->handler(ev, fde, flags, fde->private_data); return 0; } diff --git a/lib/tevent/tevent_select.c b/lib/tevent/tevent_select.c index d8aeaa7..5e26569 100644 --- a/lib/tevent/tevent_select.c +++ b/lib/tevent/tevent_select.c @@ -221,6 +221,7 @@ static int select_event_loop_select(struct select_event_context *select_ev, stru flags |= TEVENT_FD_WRITE; } if (flags) { + DLIST_DEMOTE(select_ev->ev->fd_events, fde, struct tevent_fd); fde->handler(select_ev->ev, fde, flags, fde->private_data); break; } -- 1.8.1.3 From 425010ccf71b8eb0757656d7641daef26ee47a8e Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 15 Feb 2013 12:10:26 +0100 Subject: [PATCH 45/76] tevent: add epoll_{create,ctl}_panic_fallback() for testing This makes sure we only do random panics if a fallback handler is registered. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 54 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 99d4806..4c16aec 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -44,11 +44,50 @@ struct epoll_event_context { }; #ifdef TEST_PANIC_FALLBACK -static int epoll_wait_panic_fallback(int epfd, - struct epoll_event *events, - int maxevents, - int timeout) + +static int epoll_create_panic_fallback(struct epoll_event_context *epoll_ev, + int size) +{ + if (epoll_ev->panic_fallback == NULL) { + return epoll_create(size); + } + + /* 50% of the time, fail... */ + if ((random() % 2) == 0) { + errno = EINVAL; + return -1; + } + + return epoll_create(size); +} + +static int epoll_ctl_panic_fallback(struct epoll_event_context *epoll_ev, + int epfd, int op, int fd, + struct epoll_event *event) +{ + if (epoll_ev->panic_fallback == NULL) { + return epoll_ctl(epfd, op, fd, event); + } + + /* 50% of the time, fail... */ + if ((random() % 2) == 0) { + errno = EINVAL; + return -1; + } + + return epoll_ctl(epfd, op, fd, event); +} + +static int epoll_wait_panic_fallback(struct epoll_event_context *epoll_ev, + int epfd, + struct epoll_event *events, + int maxevents, + int timeout) { + if (epoll_ev->panic_fallback == NULL) { + return epoll_wait(epfd, events, maxevents, timeout); + } + /* 50% of the time, fail... */ if ((random() % 2) == 0) { errno = EINVAL; @@ -58,7 +97,12 @@ static int epoll_wait_panic_fallback(int epfd, return epoll_wait(epfd, events, maxevents, timeout); } -#define epoll_wait epoll_wait_panic_fallback +#define epoll_create(_size) \ + epoll_create_panic_fallback(epoll_ev, _size) +#define epoll_ctl(_epfd, _op, _fd, _event) \ + epoll_ctl_panic_fallback(epoll_ev,_epfd, _op, _fd, _event) +#define epoll_wait(_epfd, _events, _maxevents, _timeout) \ + epoll_wait_panic_fallback(epoll_ev, _epfd, _events, _maxevents, _timeout) #endif /* -- 1.8.1.3 From 356c6d871019bcc471b8a0fea0c242ec8e5d0b5c Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 15 Feb 2013 16:33:56 +0100 Subject: [PATCH 46/76] tevent: avoid any operation on epoll_ev after a epoll_panic() This calls TALLOC_FREE(ev->additional_data), which is epoll_ev within epoll_panic() before calling the fallback handler. In order to notice that a epoll_panic() happened, a caller can register a pointer to a bool variable under epoll_ev->panic_state. As epoll_check_reopen() can fail due to a epoll_panic(), we need to force the replay flag if we have called any event handler. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 85 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 74 insertions(+), 11 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 4c16aec..6ad7b5b 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -40,6 +40,8 @@ struct epoll_event_context { pid_t pid; + bool panic_force_replay; + bool *panic_state; bool (*panic_fallback)(struct tevent_context *ev, bool replay); }; @@ -134,8 +136,21 @@ static void epoll_panic(struct epoll_event_context *epoll_ev, const char *reason, bool replay) { struct tevent_context *ev = epoll_ev->ev; + bool (*panic_fallback)(struct tevent_context *ev, bool replay); - if (epoll_ev->panic_fallback == NULL) { + panic_fallback = epoll_ev->panic_fallback; + + if (epoll_ev->panic_state != NULL) { + *epoll_ev->panic_state = true; + } + + if (epoll_ev->panic_force_replay) { + replay = true; + } + + TALLOC_FREE(ev->additional_data); + + if (panic_fallback == NULL) { tevent_debug(ev, TEVENT_DEBUG_FATAL, "%s (%s) replay[%u] - calling abort()\n", reason, strerror(errno), (unsigned)replay); @@ -146,7 +161,7 @@ static void epoll_panic(struct epoll_event_context *epoll_ev, "%s (%s) replay[%u] - calling panic_fallback\n", reason, strerror(errno), (unsigned)replay); - if (!epoll_ev->panic_fallback(ev, replay)) { + if (!panic_fallback(ev, replay)) { /* Fallback failed. */ tevent_debug(ev, TEVENT_DEBUG_FATAL, "%s (%s) replay[%u] - calling abort()\n", @@ -209,6 +224,8 @@ static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_ static void epoll_check_reopen(struct epoll_event_context *epoll_ev) { struct tevent_fd *fde; + bool *caller_panic_state = epoll_ev->panic_state; + bool panic_triggered = false; if (epoll_ev->pid == getpid()) { return; @@ -217,8 +234,7 @@ static void epoll_check_reopen(struct epoll_event_context *epoll_ev) close(epoll_ev->epoll_fd); epoll_ev->epoll_fd = epoll_create(64); if (epoll_ev->epoll_fd == -1) { - tevent_debug(epoll_ev->ev, TEVENT_DEBUG_FATAL, - "Failed to recreate epoll handle after fork\n"); + epoll_panic(epoll_ev, "epoll_create() failed", false); return; } @@ -228,9 +244,17 @@ static void epoll_check_reopen(struct epoll_event_context *epoll_ev) } epoll_ev->pid = getpid(); + epoll_ev->panic_state = &panic_triggered; for (fde=epoll_ev->ev->fd_events;fde;fde=fde->next) { epoll_add_event(epoll_ev, fde); + if (panic_triggered) { + if (caller_panic_state != NULL) { + *caller_panic_state = true; + } + return; + } } + epoll_ev->panic_state = NULL; } #define EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT (1<<0) @@ -463,15 +487,33 @@ static int epoll_event_fd_destructor(struct tevent_fd *fde) { struct tevent_context *ev = fde->event_ctx; struct epoll_event_context *epoll_ev = NULL; + bool panic_triggered = false; - if (ev) { - epoll_ev = talloc_get_type(ev->additional_data, - struct epoll_event_context); + if (ev == NULL) { + return tevent_common_fd_destructor(fde); + } - epoll_check_reopen(epoll_ev); + epoll_ev = talloc_get_type_abort(ev->additional_data, + struct epoll_event_context); - epoll_del_event(epoll_ev, fde); + /* + * we must remove the event from the list + * otherwise a panic fallback handler may + * reuse invalid memory + */ + DLIST_REMOVE(ev->fd_events, fde); + + epoll_ev->panic_state = &panic_triggered; + epoll_check_reopen(epoll_ev); + if (panic_triggered) { + return tevent_common_fd_destructor(fde); + } + + epoll_del_event(epoll_ev, fde); + if (panic_triggered) { + return tevent_common_fd_destructor(fde); } + epoll_ev->panic_state = NULL; return tevent_common_fd_destructor(fde); } @@ -490,8 +532,7 @@ static struct tevent_fd *epoll_event_add_fd(struct tevent_context *ev, TALLOC_CT struct epoll_event_context *epoll_ev = talloc_get_type(ev->additional_data, struct epoll_event_context); struct tevent_fd *fde; - - epoll_check_reopen(epoll_ev); + bool panic_triggered = false; fde = tevent_common_add_fd(ev, mem_ctx, fd, flags, handler, private_data, @@ -500,6 +541,13 @@ static struct tevent_fd *epoll_event_add_fd(struct tevent_context *ev, TALLOC_CT talloc_set_destructor(fde, epoll_event_fd_destructor); + epoll_ev->panic_state = &panic_triggered; + epoll_check_reopen(epoll_ev); + if (panic_triggered) { + return fde; + } + epoll_ev->panic_state = NULL; + epoll_add_event(epoll_ev, fde); return fde; @@ -512,6 +560,7 @@ static void epoll_event_set_fd_flags(struct tevent_fd *fde, uint16_t flags) { struct tevent_context *ev; struct epoll_event_context *epoll_ev; + bool panic_triggered = false; if (fde->flags == flags) return; @@ -520,7 +569,12 @@ static void epoll_event_set_fd_flags(struct tevent_fd *fde, uint16_t flags) fde->flags = flags; + epoll_ev->panic_state = &panic_triggered; epoll_check_reopen(epoll_ev); + if (panic_triggered) { + return; + } + epoll_ev->panic_state = NULL; epoll_change_event(epoll_ev, fde); } @@ -533,6 +587,7 @@ static int epoll_event_loop_once(struct tevent_context *ev, const char *location struct epoll_event_context *epoll_ev = talloc_get_type(ev->additional_data, struct epoll_event_context); struct timeval tval; + bool panic_triggered = false; if (ev->signal_events && tevent_common_check_signal(ev)) { @@ -549,7 +604,15 @@ static int epoll_event_loop_once(struct tevent_context *ev, const char *location return 0; } + epoll_ev->panic_state = &panic_triggered; + epoll_ev->panic_force_replay = true; epoll_check_reopen(epoll_ev); + if (panic_triggered) { + errno = EINVAL; + return -1; + } + epoll_ev->panic_force_replay = false; + epoll_ev->panic_state = NULL; return epoll_event_loop(epoll_ev, &tval); } -- 1.8.1.3 From 781c5824443227b2a9f342e9610cbada1064d665 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 17 Feb 2013 16:40:20 +0100 Subject: [PATCH 47/76] tevent: don't call TALLOC_FREE(ev->additional_data) in std_fallback_to_poll() The epoll backend has done the cleanup already. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_standard.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/tevent/tevent_standard.c b/lib/tevent/tevent_standard.c index d7a5bd7..a9c6170 100644 --- a/lib/tevent/tevent_standard.c +++ b/lib/tevent/tevent_standard.c @@ -68,7 +68,6 @@ static bool std_fallback_to_poll(struct tevent_context *ev, bool replay) /* First switch all the ops to poll. */ glue->epoll_ops = NULL; - TALLOC_FREE(ev->additional_data); /* * Set custom_ops the same as poll. -- 1.8.1.3 From 1fd66cbce61f1441bffb3746ceadb9dc8785f032 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 17 Feb 2013 16:41:41 +0100 Subject: [PATCH 48/76] tevent: add std_event_loop_wait() We also need to fallback in tevent_loop_wait() otherwise we might miss events in the poll->fresh list. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_standard.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/tevent/tevent_standard.c b/lib/tevent/tevent_standard.c index a9c6170..2584994 100644 --- a/lib/tevent/tevent_standard.c +++ b/lib/tevent/tevent_standard.c @@ -127,6 +127,22 @@ static int std_event_loop_once(struct tevent_context *ev, const char *location) return glue->poll_ops->loop_once(ev, location); } +static int std_event_loop_wait(struct tevent_context *ev, const char *location) +{ + void *glue_ptr = talloc_parent(ev->ops); + struct std_event_glue *glue = + talloc_get_type_abort(glue_ptr, + struct std_event_glue); + int ret; + + ret = glue->epoll_ops->loop_wait(ev, location); + if (glue->epoll_ops != NULL) { + /* No fallback */ + return ret; + } + + return glue->poll_ops->loop_wait(ev, location); +} /* Initialize the epoll backend and allow it to call a switch function if epoll fails at runtime. @@ -183,6 +199,7 @@ static int std_event_context_init(struct tevent_context *ev) *glue->glue_ops = *glue->epoll_ops; glue->glue_ops->context_init = std_event_context_init; glue->glue_ops->loop_once = std_event_loop_once; + glue->glue_ops->loop_wait = std_event_loop_wait; ret = glue->epoll_ops->context_init(ev); if (ret == -1) { -- 1.8.1.3 From 2403e3eb163b1792bd5989dd419c36ad59af81ae Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 20 Feb 2013 13:07:34 +0100 Subject: [PATCH 49/76] tevent: remove unused if (epoll_ev->epoll_fd == -1) return; checks We'll never leave epoll_check_reopen() with epoll_fd == -1. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 6ad7b5b..bdf5735 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -268,8 +268,6 @@ static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_ { struct epoll_event event; - if (epoll_ev->epoll_fd == -1) return; - fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; /* if we don't want events yet, don't add an epoll_event */ @@ -297,8 +295,6 @@ static void epoll_del_event(struct epoll_event_context *epoll_ev, struct tevent_ { struct epoll_event event; - if (epoll_ev->epoll_fd == -1) return; - fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; /* if there's no epoll_event, we don't need to delete it */ @@ -321,7 +317,6 @@ static void epoll_del_event(struct epoll_event_context *epoll_ev, struct tevent_ static void epoll_mod_event(struct epoll_event_context *epoll_ev, struct tevent_fd *fde) { struct epoll_event event; - if (epoll_ev->epoll_fd == -1) return; fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; @@ -345,8 +340,6 @@ static void epoll_change_event(struct epoll_event_context *epoll_ev, struct teve bool want_read = (fde->flags & TEVENT_FD_READ); bool want_write= (fde->flags & TEVENT_FD_WRITE); - if (epoll_ev->epoll_fd == -1) return; - fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; /* there's already an event */ @@ -383,8 +376,6 @@ static int epoll_event_loop(struct epoll_event_context *epoll_ev, struct timeval int timeout = -1; int wait_errno; - if (epoll_ev->epoll_fd == -1) return -1; - if (tvalp) { /* it's better to trigger timed events a bit later than too early */ timeout = ((tvalp->tv_usec+999) / 1000) + (tvalp->tv_sec*1000); -- 1.8.1.3 From 374386c523edea495e59f86ffef5415e6edcd3dd Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 20 Feb 2013 13:12:27 +0100 Subject: [PATCH 50/76] tevent: use helper variables to hold the epoll_ctl() result Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index bdf5735..a662fee 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -267,6 +267,7 @@ static void epoll_check_reopen(struct epoll_event_context *epoll_ev) static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_fd *fde) { struct epoll_event event; + int ret; fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; @@ -276,7 +277,8 @@ static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_ ZERO_STRUCT(event); event.events = epoll_map_flags(fde->flags); event.data.ptr = fde; - if (epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_ADD, fde->fd, &event) != 0) { + ret = epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_ADD, fde->fd, &event); + if (ret != 0) { epoll_panic(epoll_ev, "EPOLL_CTL_ADD failed", false); return; } @@ -294,6 +296,7 @@ static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_ static void epoll_del_event(struct epoll_event_context *epoll_ev, struct tevent_fd *fde) { struct epoll_event event; + int ret; fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; @@ -303,7 +306,8 @@ static void epoll_del_event(struct epoll_event_context *epoll_ev, struct tevent_ ZERO_STRUCT(event); event.events = epoll_map_flags(fde->flags); event.data.ptr = fde; - if (epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_DEL, fde->fd, &event) != 0) { + ret = epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_DEL, fde->fd, &event); + if (ret != 0) { tevent_debug(epoll_ev->ev, TEVENT_DEBUG_FATAL, "epoll_del_event failed! probable early close bug (%s)\n", strerror(errno)); @@ -317,13 +321,15 @@ static void epoll_del_event(struct epoll_event_context *epoll_ev, struct tevent_ static void epoll_mod_event(struct epoll_event_context *epoll_ev, struct tevent_fd *fde) { struct epoll_event event; + int ret; fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; ZERO_STRUCT(event); event.events = epoll_map_flags(fde->flags); event.data.ptr = fde; - if (epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_MOD, fde->fd, &event) != 0) { + ret = epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_MOD, fde->fd, &event); + if (ret != 0) { epoll_panic(epoll_ev, "EPOLL_CTL_MOD failed", false); return; } -- 1.8.1.3 From a3810a156adb2d04ee01311b5f909307206d5c2c Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 15 Feb 2013 10:31:36 +0100 Subject: [PATCH 51/76] tevent: call epoll_panic() if EPOLL_CTL_DEL failed Only ENOENT is ignored as it is no error. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index a662fee..c2b3114 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -307,10 +307,17 @@ static void epoll_del_event(struct epoll_event_context *epoll_ev, struct tevent_ event.events = epoll_map_flags(fde->flags); event.data.ptr = fde; ret = epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_DEL, fde->fd, &event); - if (ret != 0) { - tevent_debug(epoll_ev->ev, TEVENT_DEBUG_FATAL, - "epoll_del_event failed! probable early close bug (%s)\n", - strerror(errno)); + if (ret != 0 && errno == ENOENT) { + /* + * This can happen after a epoll_check_reopen + * within epoll_event_fd_destructor. + */ + tevent_debug(epoll_ev->ev, TEVENT_DEBUG_TRACE, + "EPOLL_CTL_DEL ignoring ENOENT for fd[%d]\n", + fde->fd); + } else if (ret != 0) { + epoll_panic(epoll_ev, "EPOLL_CTL_DEL failed", false); + return; } fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; } -- 1.8.1.3 From 14cd69d8d46e6278fbebaf3830e9a1383e1f753c Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 20 Feb 2013 17:15:52 +0100 Subject: [PATCH 52/76] tevent: the content of event is ignored by EPOLL_CTL_DEL Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index c2b3114..f219cba 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -304,8 +304,6 @@ static void epoll_del_event(struct epoll_event_context *epoll_ev, struct tevent_ if (!(fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT)) return; ZERO_STRUCT(event); - event.events = epoll_map_flags(fde->flags); - event.data.ptr = fde; ret = epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_DEL, fde->fd, &event); if (ret != 0 && errno == ENOENT) { /* -- 1.8.1.3 From 8681a77ec0c61d78c0246f9a88258c531903b05c Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 20 Feb 2013 12:55:11 +0100 Subject: [PATCH 53/76] tevent: s/epoll_change_event/epoll_update_event Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index f219cba..f8b151d 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -345,7 +345,7 @@ static void epoll_mod_event(struct epoll_event_context *epoll_ev, struct tevent_ } } -static void epoll_change_event(struct epoll_event_context *epoll_ev, struct tevent_fd *fde) +static void epoll_update_event(struct epoll_event_context *epoll_ev, struct tevent_fd *fde) { bool got_error = (fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_GOT_ERROR); bool want_read = (fde->flags & TEVENT_FD_READ); @@ -578,7 +578,7 @@ static void epoll_event_set_fd_flags(struct tevent_fd *fde, uint16_t flags) } epoll_ev->panic_state = NULL; - epoll_change_event(epoll_ev, fde); + epoll_update_event(epoll_ev, fde); } /* -- 1.8.1.3 From f92b095f05bd37fbe35c77a6b26bfc17850eea01 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 20 Feb 2013 11:26:00 +0100 Subject: [PATCH 54/76] tevent: always go through epoll_update_event() Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index f8b151d..e68e37b 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -45,6 +45,10 @@ struct epoll_event_context { bool (*panic_fallback)(struct tevent_context *ev, bool replay); }; +#define EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT (1<<0) +#define EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR (1<<1) +#define EPOLL_ADDITIONAL_FD_FLAG_GOT_ERROR (1<<2) + #ifdef TEST_PANIC_FALLBACK static int epoll_create_panic_fallback(struct epoll_event_context *epoll_ev, @@ -214,7 +218,7 @@ static int epoll_init_ctx(struct epoll_event_context *epoll_ev) return 0; } -static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_fd *fde); +static void epoll_update_event(struct epoll_event_context *epoll_ev, struct tevent_fd *fde); /* reopen the epoll handle when our pid changes @@ -246,7 +250,9 @@ static void epoll_check_reopen(struct epoll_event_context *epoll_ev) epoll_ev->pid = getpid(); epoll_ev->panic_state = &panic_triggered; for (fde=epoll_ev->ev->fd_events;fde;fde=fde->next) { - epoll_add_event(epoll_ev, fde); + fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; + epoll_update_event(epoll_ev, fde); + if (panic_triggered) { if (caller_panic_state != NULL) { *caller_panic_state = true; @@ -257,10 +263,6 @@ static void epoll_check_reopen(struct epoll_event_context *epoll_ev) epoll_ev->panic_state = NULL; } -#define EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT (1<<0) -#define EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR (1<<1) -#define EPOLL_ADDITIONAL_FD_FLAG_GOT_ERROR (1<<2) - /* add the epoll event to the given fd_event */ @@ -437,7 +439,7 @@ static int epoll_event_loop(struct epoll_event_context *epoll_ev, struct timeval * to match the select() behavior */ if (!(fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR)) { - epoll_del_event(epoll_ev, fde); + epoll_update_event(epoll_ev, fde); continue; } flags |= TEVENT_FD_READ; @@ -490,6 +492,7 @@ static int epoll_event_fd_destructor(struct tevent_fd *fde) struct tevent_context *ev = fde->event_ctx; struct epoll_event_context *epoll_ev = NULL; bool panic_triggered = false; + int flags = fde->flags; if (ev == NULL) { return tevent_common_fd_destructor(fde); @@ -511,7 +514,9 @@ static int epoll_event_fd_destructor(struct tevent_fd *fde) return tevent_common_fd_destructor(fde); } - epoll_del_event(epoll_ev, fde); + fde->flags = 0; + epoll_update_event(epoll_ev, fde); + fde->flags = flags; if (panic_triggered) { return tevent_common_fd_destructor(fde); } @@ -550,7 +555,7 @@ static struct tevent_fd *epoll_event_add_fd(struct tevent_context *ev, TALLOC_CT } epoll_ev->panic_state = NULL; - epoll_add_event(epoll_ev, fde); + epoll_update_event(epoll_ev, fde); return fde; } -- 1.8.1.3 From 7dacd54eadc970a9cf1ac59c0cbec3cf79ec7dc9 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 20 Feb 2013 17:38:10 +0100 Subject: [PATCH 55/76] tevent: unify handling of HAS_EVENT and REPORT_ERROR in epoll_{add,mod,del}_event() epoll_{add,mod,del}_event() are only called via epoll_update_event() and epoll_update_event() should not remove REPORT_ERROR itself. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index e68e37b..0f7080c 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -271,11 +271,9 @@ static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_ struct epoll_event event; int ret; + fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; - /* if we don't want events yet, don't add an epoll_event */ - if (fde->flags == 0) return; - ZERO_STRUCT(event); event.events = epoll_map_flags(fde->flags); event.data.ptr = fde; @@ -284,8 +282,8 @@ static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_ epoll_panic(epoll_ev, "EPOLL_CTL_ADD failed", false); return; } - fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; + fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; /* only if we want to read we want to tell the event handler about errors */ if (fde->flags & TEVENT_FD_READ) { fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; @@ -300,11 +298,9 @@ static void epoll_del_event(struct epoll_event_context *epoll_ev, struct tevent_ struct epoll_event event; int ret; + fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; - /* if there's no epoll_event, we don't need to delete it */ - if (!(fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT)) return; - ZERO_STRUCT(event); ret = epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_DEL, fde->fd, &event); if (ret != 0 && errno == ENOENT) { @@ -315,11 +311,11 @@ static void epoll_del_event(struct epoll_event_context *epoll_ev, struct tevent_ tevent_debug(epoll_ev->ev, TEVENT_DEBUG_TRACE, "EPOLL_CTL_DEL ignoring ENOENT for fd[%d]\n", fde->fd); + return; } else if (ret != 0) { epoll_panic(epoll_ev, "EPOLL_CTL_DEL failed", false); return; } - fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; } /* @@ -330,6 +326,7 @@ static void epoll_mod_event(struct epoll_event_context *epoll_ev, struct tevent_ struct epoll_event event; int ret; + fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; ZERO_STRUCT(event); @@ -341,6 +338,7 @@ static void epoll_mod_event(struct epoll_event_context *epoll_ev, struct tevent_ return; } + fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; /* only if we want to read we want to tell the event handler about errors */ if (fde->flags & TEVENT_FD_READ) { fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; @@ -353,8 +351,6 @@ static void epoll_update_event(struct epoll_event_context *epoll_ev, struct teve bool want_read = (fde->flags & TEVENT_FD_READ); bool want_write= (fde->flags & TEVENT_FD_WRITE); - fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; - /* there's already an event */ if (fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT) { if (want_read || (want_write && !got_error)) { -- 1.8.1.3 From 143e18fb9103ef12efe10af23b1435254407428c Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 20 Feb 2013 13:33:19 +0100 Subject: [PATCH 56/76] tevent: ignore EBADF from epoll_ctl() and disable the event Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 0f7080c..adea677 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -278,7 +278,15 @@ static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_ event.events = epoll_map_flags(fde->flags); event.data.ptr = fde; ret = epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_ADD, fde->fd, &event); - if (ret != 0) { + if (ret != 0 && errno == EBADF) { + tevent_debug(epoll_ev->ev, TEVENT_DEBUG_ERROR, + "EPOLL_CTL_ADD EBADF for " + "fde[%p] fd[%d] - disabling\n", + fde, fde->fd); + DLIST_REMOVE(epoll_ev->ev->fd_events, fde); + fde->event_ctx = NULL; + return; + } else if (ret != 0) { epoll_panic(epoll_ev, "EPOLL_CTL_ADD failed", false); return; } @@ -312,6 +320,14 @@ static void epoll_del_event(struct epoll_event_context *epoll_ev, struct tevent_ "EPOLL_CTL_DEL ignoring ENOENT for fd[%d]\n", fde->fd); return; + } else if (ret != 0 && errno == EBADF) { + tevent_debug(epoll_ev->ev, TEVENT_DEBUG_WARNING, + "EPOLL_CTL_DEL EBADF for " + "fde[%p] fd[%d] - disabling\n", + fde, fde->fd); + DLIST_REMOVE(epoll_ev->ev->fd_events, fde); + fde->event_ctx = NULL; + return; } else if (ret != 0) { epoll_panic(epoll_ev, "EPOLL_CTL_DEL failed", false); return; @@ -333,7 +349,15 @@ static void epoll_mod_event(struct epoll_event_context *epoll_ev, struct tevent_ event.events = epoll_map_flags(fde->flags); event.data.ptr = fde; ret = epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_MOD, fde->fd, &event); - if (ret != 0) { + if (ret != 0 && errno == EBADF) { + tevent_debug(epoll_ev->ev, TEVENT_DEBUG_ERROR, + "EPOLL_CTL_MOD EBADF for " + "fde[%p] fd[%d] - disabling\n", + fde, fde->fd); + DLIST_REMOVE(epoll_ev->ev->fd_events, fde); + fde->event_ctx = NULL; + return; + } else if (ret != 0) { epoll_panic(epoll_ev, "EPOLL_CTL_MOD failed", false); return; } -- 1.8.1.3 From 1b74985a23675a2857af8929bd5dd2c14ec19712 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Thu, 14 Feb 2013 13:50:56 -0800 Subject: [PATCH 57/76] tevent: Start to fix the epoll backend to support 2 fd events on the same fd correctly. Add a utility function epoll_add_multiplex_fd() and a new flag EPOLL_ADDITIONAL_FD_FLAG_HAS_MPX. This will be called by epoll_add_event() to merge two fde events with the same file descriptor. Signed-off-by: Jeremy Allison Signed-off-by: Stefan Metzmacher Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index adea677..c2b743d 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -4,7 +4,8 @@ main select loop and event handling - epoll implementation Copyright (C) Andrew Tridgell 2003-2005 - Copyright (C) Stefan Metzmacher 2005-2009 + Copyright (C) Stefan Metzmacher 2005-2013 + Copyright (C) Jeremy Allison 2013 ** NOTE! The following LGPL license applies to the tevent ** library. This does NOT imply that all of Samba is released @@ -48,6 +49,7 @@ struct epoll_event_context { #define EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT (1<<0) #define EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR (1<<1) #define EPOLL_ADDITIONAL_FD_FLAG_GOT_ERROR (1<<2) +#define EPOLL_ADDITIONAL_FD_FLAG_HAS_MPX (1<<3) #ifdef TEST_PANIC_FALLBACK @@ -264,6 +266,99 @@ static void epoll_check_reopen(struct epoll_event_context *epoll_ev) } /* + epoll cannot add the same file descriptor twice, once + with read, once with write which is allowed by the + tevent backend. Multiplex the existing fde, flag it + as such so we can search for the correct fde on + event triggering. +*/ + +static int epoll_add_multiplex_fd(struct epoll_event_context *epoll_ev, + struct tevent_fd *add_fde) +{ + struct epoll_event event; + struct tevent_fd *mpx_fde; + int ret; + + /* Find the existing fde that caused the EEXIST error. */ + for (mpx_fde = epoll_ev->ev->fd_events; mpx_fde; mpx_fde = mpx_fde->next) { + if (mpx_fde->fd != add_fde->fd) { + continue; + } + + if (mpx_fde == add_fde) { + continue; + } + + break; + } + if (mpx_fde == NULL) { + tevent_debug(epoll_ev->ev, TEVENT_DEBUG_FATAL, + "can't find multiplex fde for fd[%d]", + add_fde->fd); + return -1; + } + + if (mpx_fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_HAS_MPX) { + /* Logic error. Can't have more than 2 multiplexed fde's. */ + tevent_debug(epoll_ev->ev, TEVENT_DEBUG_FATAL, + "multiplex fde for fd[%d] is already multiplexed\n", + mpx_fde->fd); + return -1; + } + + /* + * The multiplex fde must have the same fd, and also + * already have an epoll event attached. + */ + if (!(mpx_fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT)) { + /* Logic error. Can't have more than 2 multiplexed fde's. */ + tevent_debug(epoll_ev->ev, TEVENT_DEBUG_FATAL, + "multiplex fde for fd[%d] has no event\n", + mpx_fde->fd); + return -1; + } + + /* Modify the mpx_fde to add in the new flags. */ + ZERO_STRUCT(event); + event.events = epoll_map_flags(mpx_fde->flags); + event.events |= epoll_map_flags(add_fde->flags); + event.data.ptr = mpx_fde; + ret = epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_MOD, mpx_fde->fd, &event); + if (ret != 0 && errno == EBADF) { + tevent_debug(epoll_ev->ev, TEVENT_DEBUG_ERROR, + "EPOLL_CTL_MOD EBADF for " + "add_fde[%p] mpx_fde[%p] fd[%d] - disabling\n", + add_fde, mpx_fde, add_fde->fd); + DLIST_REMOVE(epoll_ev->ev->fd_events, mpx_fde); + mpx_fde->event_ctx = NULL; + DLIST_REMOVE(epoll_ev->ev->fd_events, add_fde); + add_fde->event_ctx = NULL; + return 0; + } else if (ret != 0) { + return ret; + } + + /* + * Make each fde->additional_data pointers point at each other + * so we can look them up from each other. They are now paired. + */ + mpx_fde->additional_data = (struct tevent_fd *)add_fde; + add_fde->additional_data = (struct tevent_fd *)mpx_fde; + + /* Now flag both fde's as being multiplexed. */ + mpx_fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_HAS_MPX; + add_fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_HAS_MPX; + + /* we need to keep the GOT_ERROR flag */ + if (mpx_fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_GOT_ERROR) { + add_fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_GOT_ERROR; + } + + return 0; +} + +/* add the epoll event to the given fd_event */ static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_fd *fde) -- 1.8.1.3 From 44fdfc3da08ffb046eeb2c6c48d4aefcfbeb9107 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 20 Feb 2013 17:14:52 +0100 Subject: [PATCH 58/76] tevent: handle multiplexed fde's in epoll_add_event() Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index c2b743d..987748c 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -365,21 +365,41 @@ static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_ { struct epoll_event event; int ret; + struct tevent_fd *mpx_fde = NULL; fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; + if (fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_HAS_MPX) { + /* + * This is a multiplexed fde, we need to include both + * flags in the modified event. + */ + mpx_fde = talloc_get_type_abort(fde->additional_data, + struct tevent_fd); + + mpx_fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; + mpx_fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; + } + ZERO_STRUCT(event); event.events = epoll_map_flags(fde->flags); + if (mpx_fde != NULL) { + event.events |= epoll_map_flags(mpx_fde->flags); + } event.data.ptr = fde; ret = epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_ADD, fde->fd, &event); if (ret != 0 && errno == EBADF) { tevent_debug(epoll_ev->ev, TEVENT_DEBUG_ERROR, "EPOLL_CTL_ADD EBADF for " - "fde[%p] fd[%d] - disabling\n", - fde, fde->fd); + "fde[%p] mpx_fde[%p] fd[%d] - disabling\n", + fde, mpx_fde, fde->fd); DLIST_REMOVE(epoll_ev->ev->fd_events, fde); fde->event_ctx = NULL; + if (mpx_fde != NULL) { + DLIST_REMOVE(epoll_ev->ev->fd_events, mpx_fde); + mpx_fde->event_ctx = NULL; + } return; } else if (ret != 0) { epoll_panic(epoll_ev, "EPOLL_CTL_ADD failed", false); @@ -391,6 +411,16 @@ static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_ if (fde->flags & TEVENT_FD_READ) { fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; } + + if (mpx_fde == NULL) { + return; + } + + mpx_fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; + /* only if we want to read we want to tell the event handler about errors */ + if (mpx_fde->flags & TEVENT_FD_READ) { + mpx_fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; + } } /* -- 1.8.1.3 From 13053fb8a6790274596e7f97b65587d4403b4c87 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Fri, 15 Feb 2013 10:16:44 -0800 Subject: [PATCH 59/76] tevent: Fix up epoll_del_event to cope with deleting a multiplexed fde event. Signed-off-by: Jeremy Allison Signed-off-by: Stefan Metzmacher Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 987748c..97130db 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -430,10 +430,22 @@ static void epoll_del_event(struct epoll_event_context *epoll_ev, struct tevent_ { struct epoll_event event; int ret; + struct tevent_fd *mpx_fde = NULL; fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; + if (fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_HAS_MPX) { + /* + * This is a multiplexed fde, we need to modify both events. + */ + mpx_fde = talloc_get_type_abort(fde->additional_data, + struct tevent_fd); + + mpx_fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; + mpx_fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; + } + ZERO_STRUCT(event); ret = epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_DEL, fde->fd, &event); if (ret != 0 && errno == ENOENT) { @@ -448,10 +460,14 @@ static void epoll_del_event(struct epoll_event_context *epoll_ev, struct tevent_ } else if (ret != 0 && errno == EBADF) { tevent_debug(epoll_ev->ev, TEVENT_DEBUG_WARNING, "EPOLL_CTL_DEL EBADF for " - "fde[%p] fd[%d] - disabling\n", - fde, fde->fd); + "fde[%p] mpx_fde[%p] fd[%d] - disabling\n", + fde, mpx_fde, fde->fd); DLIST_REMOVE(epoll_ev->ev->fd_events, fde); fde->event_ctx = NULL; + if (mpx_fde != NULL) { + DLIST_REMOVE(epoll_ev->ev->fd_events, mpx_fde); + mpx_fde->event_ctx = NULL; + } return; } else if (ret != 0) { epoll_panic(epoll_ev, "EPOLL_CTL_DEL failed", false); -- 1.8.1.3 From f9c3085c51a0490bb336f2b15f48dc8b51822bde Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 1 Mar 2013 13:57:05 +0100 Subject: [PATCH 60/76] tevent: Fix epoll_mod_event() to cope with modifying a multiplexed fde event. Signed-off-by: Jeremy Allison Signed-off-by: Stefan Metzmacher Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 97130db..d8c1050 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -480,23 +480,43 @@ static void epoll_del_event(struct epoll_event_context *epoll_ev, struct tevent_ */ static void epoll_mod_event(struct epoll_event_context *epoll_ev, struct tevent_fd *fde) { + struct tevent_fd *mpx_fde = NULL; struct epoll_event event; int ret; fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; + if (fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_HAS_MPX) { + /* + * This is a multiplexed fde, we need to include both + * flags in the modified event. + */ + mpx_fde = talloc_get_type_abort(fde->additional_data, + struct tevent_fd); + + mpx_fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; + mpx_fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; + } + ZERO_STRUCT(event); event.events = epoll_map_flags(fde->flags); + if (mpx_fde != NULL) { + event.events |= epoll_map_flags(mpx_fde->flags); + } event.data.ptr = fde; ret = epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_MOD, fde->fd, &event); if (ret != 0 && errno == EBADF) { tevent_debug(epoll_ev->ev, TEVENT_DEBUG_ERROR, "EPOLL_CTL_MOD EBADF for " - "fde[%p] fd[%d] - disabling\n", - fde, fde->fd); + "fde[%p] mpx_fde[%p] fd[%d] - disabling\n", + fde, mpx_fde, fde->fd); DLIST_REMOVE(epoll_ev->ev->fd_events, fde); fde->event_ctx = NULL; + if (mpx_fde != NULL) { + DLIST_REMOVE(epoll_ev->ev->fd_events, mpx_fde); + mpx_fde->event_ctx = NULL; + } return; } else if (ret != 0) { epoll_panic(epoll_ev, "EPOLL_CTL_MOD failed", false); @@ -508,6 +528,16 @@ static void epoll_mod_event(struct epoll_event_context *epoll_ev, struct tevent_ if (fde->flags & TEVENT_FD_READ) { fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; } + + if (mpx_fde == NULL) { + return; + } + + mpx_fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; + /* only if we want to read we want to tell the event handler about errors */ + if (mpx_fde->flags & TEVENT_FD_READ) { + mpx_fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR; + } } static void epoll_update_event(struct epoll_event_context *epoll_ev, struct tevent_fd *fde) -- 1.8.1.3 From 4ac9c49b2da759dfa65fb7306590ee06ac80814d Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 20 Feb 2013 18:01:57 +0100 Subject: [PATCH 61/76] tevent: handle multiplexed fde's in epoll_event_fd_destructor() Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index d8c1050..b1927d3 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -683,6 +683,7 @@ static int epoll_event_fd_destructor(struct tevent_fd *fde) struct tevent_context *ev = fde->event_ctx; struct epoll_event_context *epoll_ev = NULL; bool panic_triggered = false; + struct tevent_fd *mpx_fde = NULL; int flags = fde->flags; if (ev == NULL) { @@ -699,12 +700,32 @@ static int epoll_event_fd_destructor(struct tevent_fd *fde) */ DLIST_REMOVE(ev->fd_events, fde); + if (fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_HAS_MPX) { + mpx_fde = talloc_get_type_abort(fde->additional_data, + struct tevent_fd); + + fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_MPX; + mpx_fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_MPX; + + fde->additional_data = NULL; + mpx_fde->additional_data = NULL; + + fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; + } + epoll_ev->panic_state = &panic_triggered; epoll_check_reopen(epoll_ev); if (panic_triggered) { return tevent_common_fd_destructor(fde); } + if (mpx_fde != NULL) { + epoll_update_event(epoll_ev, mpx_fde); + if (panic_triggered) { + return tevent_common_fd_destructor(fde); + } + } + fde->flags = 0; epoll_update_event(epoll_ev, fde); fde->flags = flags; -- 1.8.1.3 From 82dfea642a794148a0757bdfc9019399a8a9598d Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 20 Feb 2013 17:17:31 +0100 Subject: [PATCH 62/76] tevent: handle EPOLL_ADDITIONAL_FD_FLAG_HAS_MPX in epoll_update_event() Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index b1927d3..3608d4b 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -545,6 +545,23 @@ static void epoll_update_event(struct epoll_event_context *epoll_ev, struct teve bool got_error = (fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_GOT_ERROR); bool want_read = (fde->flags & TEVENT_FD_READ); bool want_write= (fde->flags & TEVENT_FD_WRITE); + struct tevent_fd *mpx_fde = NULL; + + if (fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_HAS_MPX) { + /* + * work out what the multiplexed fde wants. + */ + mpx_fde = talloc_get_type_abort(fde->additional_data, + struct tevent_fd); + + if (mpx_fde->flags & TEVENT_FD_READ) { + want_read = true; + } + + if (mpx_fde->flags & TEVENT_FD_WRITE) { + want_write = true; + } + } /* there's already an event */ if (fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT) { -- 1.8.1.3 From 09147c3f7ef2cc2b398895c0632edd5918951017 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Thu, 14 Feb 2013 15:53:38 -0800 Subject: [PATCH 63/76] tevent: Add utility function epoll_handle_hup_or_err() We'll use this to handle the EPOLL_ADDITIONAL_FD_FLAG_GOT_ERROR and EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR flags with multiplexed events in the event loop. Signed-off-by: Jeremy Allison Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 3608d4b..3e3ddf9 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -587,6 +587,38 @@ static void epoll_update_event(struct epoll_event_context *epoll_ev, struct teve } /* + Cope with epoll returning EPOLLHUP|EPOLLERR on an event. + Return true if there's nothing else to do, false if + this event needs further handling. +*/ +static bool epoll_handle_hup_or_err(struct epoll_event_context *epoll_ev, + struct tevent_fd *fde) +{ + if (fde == NULL) { + /* Nothing to do if no event. */ + return true; + } + + fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_GOT_ERROR; + /* + * if we only wait for TEVENT_FD_WRITE, we should not tell the + * event handler about it, and remove the epoll_event, + * as we only report errors when waiting for read events, + * to match the select() behavior + */ + if (!(fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR)) { + /* + * Do the same as the poll backend and + * remove the writeable flag. + */ + fde->flags &= ~TEVENT_FD_WRITE; + return true; + } + /* This has TEVENT_FD_READ set, we're not finished. */ + return false; +} + +/* event loop handling using epoll */ static int epoll_event_loop(struct epoll_event_context *epoll_ev, struct timeval *tvalp) -- 1.8.1.3 From 0c871a4e1166b6f0187cf9127df6b7e8b440e675 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 20 Feb 2013 14:49:17 +0100 Subject: [PATCH 64/76] tevent: In epoll_event_loop() ensure we trigger the right handler for a multiplexed fde event. Signed-off-by: Jeremy Allison Signed-off-by: Stefan Metzmacher Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 3e3ddf9..41415b0 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -665,27 +665,56 @@ static int epoll_event_loop(struct epoll_event_context *epoll_ev, struct timeval struct tevent_fd *fde = talloc_get_type(events[i].data.ptr, struct tevent_fd); uint16_t flags = 0; + struct tevent_fd *mpx_fde = NULL; if (fde == NULL) { epoll_panic(epoll_ev, "epoll_wait() gave bad data", true); return -1; } - if (events[i].events & (EPOLLHUP|EPOLLERR)) { - fde->additional_flags |= EPOLL_ADDITIONAL_FD_FLAG_GOT_ERROR; + if (fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_HAS_MPX) { /* - * if we only wait for TEVENT_FD_WRITE, we should not tell the - * event handler about it, and remove the epoll_event, - * as we only report errors when waiting for read events, - * to match the select() behavior + * Save off the multiplexed event in case we need + * to use it to call the handler function. */ - if (!(fde->additional_flags & EPOLL_ADDITIONAL_FD_FLAG_REPORT_ERROR)) { + mpx_fde = talloc_get_type_abort(fde->additional_data, + struct tevent_fd); + } + if (events[i].events & (EPOLLHUP|EPOLLERR)) { + bool handled_fde = epoll_handle_hup_or_err(epoll_ev, fde); + bool handled_mpx = epoll_handle_hup_or_err(epoll_ev, mpx_fde); + + if (handled_fde && handled_mpx) { epoll_update_event(epoll_ev, fde); continue; } + + if (!handled_mpx) { + /* + * If the mpx event was the one that needs + * further handling, it's the TEVENT_FD_READ + * event so switch over and call that handler. + */ + fde = mpx_fde; + mpx_fde = NULL; + } flags |= TEVENT_FD_READ; } if (events[i].events & EPOLLIN) flags |= TEVENT_FD_READ; if (events[i].events & EPOLLOUT) flags |= TEVENT_FD_WRITE; + + if (mpx_fde) { + /* Ensure we got the right fde. */ + if ((flags & fde->flags) == 0) { + fde = mpx_fde; + mpx_fde = NULL; + } + } + + /* + * make sure we only pass the flags + * the handler is expecting. + */ + flags &= fde->flags; if (flags) { fde->handler(epoll_ev->ev, fde, flags, fde->private_data); break; -- 1.8.1.3 From 3812bb0ecbd23f14ee09413f6d824e03e334c0b2 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 20 Feb 2013 11:50:56 +0100 Subject: [PATCH 65/76] tevent: preferr the write handler if there're two possible handlers registered with epoll Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 41415b0..19f6809 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -702,6 +702,16 @@ static int epoll_event_loop(struct epoll_event_context *epoll_ev, struct timeval if (events[i].events & EPOLLIN) flags |= TEVENT_FD_READ; if (events[i].events & EPOLLOUT) flags |= TEVENT_FD_WRITE; + if (flags & TEVENT_FD_WRITE) { + if (fde->flags & TEVENT_FD_WRITE) { + mpx_fde = NULL; + } + if (mpx_fde && mpx_fde->flags & TEVENT_FD_WRITE) { + fde = mpx_fde; + mpx_fde = NULL; + } + } + if (mpx_fde) { /* Ensure we got the right fde. */ if ((flags & fde->flags) == 0) { -- 1.8.1.3 From 282c75ba0702837a5c8bbc8cae8efc0cb7fde4de Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Thu, 14 Feb 2013 13:52:41 -0800 Subject: [PATCH 66/76] tevent: If epoll_ctl(..EPOLL_CTL_ADD,..) failes with EEXIST, merge the two fde's into one epoll event. Signed-off-by: Jeremy Allison Signed-off-by: Stefan Metzmacher Reviewed-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 19f6809..3a3c01c 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -401,6 +401,13 @@ static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_ mpx_fde->event_ctx = NULL; } return; + } else if (ret != 0 && errno == EEXIST && mpx_fde == NULL) { + ret = epoll_add_multiplex_fd(epoll_ev, fde); + if (ret != 0) { + epoll_panic(epoll_ev, "epoll_add_multiplex_fd failed", + false); + return; + } } else if (ret != 0) { epoll_panic(epoll_ev, "EPOLL_CTL_ADD failed", false); return; -- 1.8.1.3 From 2095e82d821a052ecbbb50e044e34d479cd1fd03 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 1 Mar 2013 14:30:54 +0100 Subject: [PATCH 67/76] tevent: use DEBUG_ERROR for the fallback message in epoll_panic() A Samba autobuild passed without a fallback, so this is really an error. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_epoll.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 3a3c01c..1ec97c5 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -163,7 +163,7 @@ static void epoll_panic(struct epoll_event_context *epoll_ev, abort(); } - tevent_debug(ev, TEVENT_DEBUG_WARNING, + tevent_debug(ev, TEVENT_DEBUG_ERROR, "%s (%s) replay[%u] - calling panic_fallback\n", reason, strerror(errno), (unsigned)replay); -- 1.8.1.3 From 37f34b1dca72e89bed1261de97d102313de19817 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Thu, 14 Feb 2013 14:16:31 -0800 Subject: [PATCH 68/76] tevent: Regression test to ensure that a tevent backend can cope with separate read/write events on a single fd. This tests the multiplex fd changes to the epoll backend to ensure they work correctly. Signed-off-by: Jeremy Allison Reviewed-by: Stefan Metzmacher --- lib/tevent/testsuite.c | 61 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/lib/tevent/testsuite.c b/lib/tevent/testsuite.c index 142ba4f..0f164aa 100644 --- a/lib/tevent/testsuite.c +++ b/lib/tevent/testsuite.c @@ -4,6 +4,7 @@ testing of the events subsystem Copyright (C) Stefan Metzmacher 2006-2009 + Copyright (C) Jeremy Allison 2013 ** NOTE! The following LGPL license applies to the tevent ** library. This does NOT imply that all of Samba is released @@ -35,7 +36,7 @@ static int fde_count; -static void fde_handler(struct tevent_context *ev_ctx, struct tevent_fd *f, +static void fde_handler_read(struct tevent_context *ev_ctx, struct tevent_fd *f, uint16_t flags, void *private_data) { int *fd = (int *)private_data; @@ -44,11 +45,38 @@ static void fde_handler(struct tevent_context *ev_ctx, struct tevent_fd *f, kill(getpid(), SIGUSR1); #endif kill(getpid(), SIGALRM); + read(fd[0], &c, 1); - write(fd[1], &c, 1); fde_count++; } +static void fde_handler_write(struct tevent_context *ev_ctx, struct tevent_fd *f, + uint16_t flags, void *private_data) +{ + int *fd = (int *)private_data; + char c = 0; + write(fd[1], &c, 1); +} + + +/* These should never fire... */ +static void fde_handler_read_1(struct tevent_context *ev_ctx, struct tevent_fd *f, + uint16_t flags, void *private_data) +{ + struct torture_context *test = (struct torture_context *)private_data; + torture_comment(test, "fde_handler_read_1 should never fire !\n"); + abort(); +} + +/* These should never fire... */ +static void fde_handler_write_1(struct tevent_context *ev_ctx, struct tevent_fd *f, + uint16_t flags, void *private_data) +{ + struct torture_context *test = (struct torture_context *)private_data; + torture_comment(test, "fde_handler_write_1 should never fire !\n"); + abort(); +} + static void finished_handler(struct tevent_context *ev_ctx, struct tevent_timer *te, struct timeval tval, void *private_data) { @@ -70,7 +98,10 @@ static bool test_event_context(struct torture_context *test, int fd[2] = { -1, -1 }; const char *backend = (const char *)test_data; int alarm_count=0, info_count=0; - struct tevent_fd *fde; + struct tevent_fd *fde_read; + struct tevent_fd *fde_read_1; + struct tevent_fd *fde_write; + struct tevent_fd *fde_write_1; #ifdef SA_RESTART struct tevent_signal *se1 = NULL; #endif @@ -82,7 +113,6 @@ static bool test_event_context(struct torture_context *test, #endif int finished=0; struct timeval t; - char c = 0; ev_ctx = tevent_context_init_byname(test, backend); if (ev_ctx == NULL) { @@ -99,9 +129,18 @@ static bool test_event_context(struct torture_context *test, /* create a pipe */ pipe(fd); - fde = tevent_add_fd(ev_ctx, ev_ctx, fd[0], TEVENT_FD_READ, - fde_handler, fd); - tevent_fd_set_auto_close(fde); + fde_read = tevent_add_fd(ev_ctx, ev_ctx, fd[0], TEVENT_FD_READ, + fde_handler_read, fd); + fde_write_1 = tevent_add_fd(ev_ctx, ev_ctx, fd[0], TEVENT_FD_WRITE, + fde_handler_write_1, test); + + fde_write = tevent_add_fd(ev_ctx, ev_ctx, fd[1], TEVENT_FD_WRITE, + fde_handler_write, fd); + fde_read_1 = tevent_add_fd(ev_ctx, ev_ctx, fd[1], TEVENT_FD_READ, + fde_handler_read_1, test); + + tevent_fd_set_auto_close(fde_read); + tevent_fd_set_auto_close(fde_write); tevent_add_timer(ev_ctx, ev_ctx, timeval_current_ofs(2,0), finished_handler, &finished); @@ -119,8 +158,6 @@ static bool test_event_context(struct torture_context *test, torture_assert(test, se3 != NULL, "failed to setup se3"); #endif - write(fd[1], &c, 1); - t = timeval_current(); while (!finished) { errno = 0; @@ -130,8 +167,10 @@ static bool test_event_context(struct torture_context *test, } } - talloc_free(fde); - close(fd[1]); + talloc_free(fde_read); + talloc_free(fde_write); + talloc_free(fde_read_1); + talloc_free(fde_write_1); while (alarm_count < fde_count+1) { if (tevent_loop_once(ev_ctx) == -1) { -- 1.8.1.3 From ceb58bc435e3a25f5c2ebef84233565ffddd02ac Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 28 Feb 2013 10:12:42 +0100 Subject: [PATCH 69/76] tevent: use better names for the subtests Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/testsuite.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/tevent/testsuite.c b/lib/tevent/testsuite.c index 0f164aa..5c2f695 100644 --- a/lib/tevent/testsuite.c +++ b/lib/tevent/testsuite.c @@ -348,13 +348,20 @@ struct torture_suite *torture_local_event(TALLOC_CTX *mem_ctx) int i; for (i=0;list && list[i];i++) { - torture_suite_add_simple_tcase_const(suite, list[i], + struct torture_suite *backend_suite; + + backend_suite = torture_suite_create(mem_ctx, list[i]); + + torture_suite_add_simple_tcase_const(backend_suite, + "context", test_event_context, (const void *)list[i]); + + torture_suite_add_suite(suite, backend_suite); } #ifdef HAVE_PTHREAD - torture_suite_add_simple_tcase_const(suite, "poll_mt_threaded", + torture_suite_add_simple_tcase_const(suite, "threaded_poll_mt", test_event_context_threaded, NULL); #endif -- 1.8.1.3 From 9eda05864bf5670c9ae2376ff8ce8c419b8c8b66 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 22 Feb 2013 21:48:44 +0100 Subject: [PATCH 70/76] tevent: add test_event_fd1() This test verifies that TEVENT_FD_* flags are handled correctly. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/testsuite.c | 221 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) diff --git a/lib/tevent/testsuite.c b/lib/tevent/testsuite.c index 5c2f695..cca7e9a 100644 --- a/lib/tevent/testsuite.c +++ b/lib/tevent/testsuite.c @@ -28,6 +28,7 @@ #include "lib/tevent/tevent.h" #include "system/filesys.h" #include "system/select.h" +#include "system/network.h" #include "torture/torture.h" #ifdef HAVE_PTHREAD #include @@ -204,6 +205,222 @@ static bool test_event_context(struct torture_context *test, return true; } +struct test_event_fd1_state { + struct torture_context *tctx; + const char *backend; + struct tevent_context *ev; + int sock[2]; + struct tevent_timer *te; + struct tevent_fd *fde0; + struct tevent_fd *fde1; + bool got_write; + bool got_read; + bool drain; + bool drain_done; + unsigned loop_count; + bool finished; + const char *error; +}; + +static void test_event_fd1_fde_handler(struct tevent_context *ev_ctx, + struct tevent_fd *fde, + uint16_t flags, + void *private_data) +{ + struct test_event_fd1_state *state = + (struct test_event_fd1_state *)private_data; + + if (state->drain_done) { + state->finished = true; + state->error = __location__; + return; + } + + if (state->drain) { + ssize_t ret; + uint8_t c = 0; + + if (!(flags & TEVENT_FD_READ)) { + state->finished = true; + state->error = __location__; + return; + } + + ret = read(state->sock[0], &c, 1); + if (ret == 1) { + return; + } + + /* + * end of test... + */ + tevent_fd_set_flags(fde, 0); + state->drain_done = true; + return; + } + + if (!state->got_write) { + uint8_t c = 0; + + if (flags != TEVENT_FD_WRITE) { + state->finished = true; + state->error = __location__; + return; + } + state->got_write = true; + + /* + * we write to the other socket... + */ + write(state->sock[1], &c, 1); + TEVENT_FD_NOT_WRITEABLE(fde); + TEVENT_FD_READABLE(fde); + return; + } + + if (!state->got_read) { + if (flags != TEVENT_FD_READ) { + state->finished = true; + state->error = __location__; + return; + } + state->got_read = true; + + TEVENT_FD_NOT_READABLE(fde); + return; + } + + state->finished = true; + state->error = __location__; + return; +} + +static void test_event_fd1_finished(struct tevent_context *ev_ctx, + struct tevent_timer *te, + struct timeval tval, + void *private_data) +{ + struct test_event_fd1_state *state = + (struct test_event_fd1_state *)private_data; + + if (state->drain_done) { + state->finished = true; + return; + } + + if (!state->got_write) { + state->finished = true; + state->error = __location__; + return; + } + + if (!state->got_read) { + state->finished = true; + state->error = __location__; + return; + } + + state->loop_count++; + if (state->loop_count > 3) { + state->finished = true; + state->error = __location__; + return; + } + + state->got_write = false; + state->got_read = false; + + tevent_fd_set_flags(state->fde0, TEVENT_FD_WRITE); + + if (state->loop_count > 2) { + state->drain = true; + TALLOC_FREE(state->fde1); + TEVENT_FD_READABLE(state->fde0); + } + + state->te = tevent_add_timer(state->ev, state->ev, + timeval_current_ofs(0,2000), + test_event_fd1_finished, state); +} + +static bool test_event_fd1(struct torture_context *tctx, + const void *test_data) +{ + struct test_event_fd1_state state; + + ZERO_STRUCT(state); + state.tctx = tctx; + state.backend = (const char *)test_data; + + state.ev = tevent_context_init_byname(tctx, state.backend); + if (state.ev == NULL) { + torture_skip(tctx, talloc_asprintf(tctx, + "event backend '%s' not supported\n", + state.backend)); + return true; + } + + tevent_set_debug_stderr(state.ev); + torture_comment(tctx, "backend '%s' - %s\n", + state.backend, __FUNCTION__); + + /* + * This tests the following: + * + * It monitors the state of state.sock[0] + * with tevent_fd, but we never read/write on state.sock[0] + * while state.sock[1] * is only used to write a few bytes. + * + * We have a loop: + * - we wait only for TEVENT_FD_WRITE on state.sock[0] + * - we write 1 byte to state.sock[1] + * - we wait only for TEVENT_FD_READ on state.sock[0] + * - we disable events on state.sock[0] + * - the timer event restarts the loop + * Then we close state.sock[1] + * We have a loop: + * - we wait for TEVENT_FD_READ/WRITE on state.sock[0] + * - we try to read 1 byte + * - if the read gets an error of returns 0 + * we disable the event handler + * - the timer finishes the test + */ + state.sock[0] = -1; + state.sock[1] = -1; + socketpair(AF_UNIX, SOCK_STREAM, 0, state.sock); + + state.te = tevent_add_timer(state.ev, state.ev, + timeval_current_ofs(0,1000), + test_event_fd1_finished, &state); + state.fde0 = tevent_add_fd(state.ev, state.ev, + state.sock[0], TEVENT_FD_WRITE, + test_event_fd1_fde_handler, &state); + /* state.fde1 is only used to auto close */ + state.fde1 = tevent_add_fd(state.ev, state.ev, + state.sock[1], 0, + test_event_fd1_fde_handler, &state); + + tevent_fd_set_auto_close(state.fde0); + tevent_fd_set_auto_close(state.fde1); + + while (!state.finished) { + errno = 0; + if (tevent_loop_once(state.ev) == -1) { + talloc_free(state.ev); + torture_fail(tctx, talloc_asprintf(tctx, + "Failed event loop %s\n", + strerror(errno))); + } + } + + talloc_free(state.ev); + + torture_assert(tctx, state.error == NULL, talloc_asprintf(tctx, + "%s", state.error)); + + return true; +} + #ifdef HAVE_PTHREAD static pthread_mutex_t threaded_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -356,6 +573,10 @@ struct torture_suite *torture_local_event(TALLOC_CTX *mem_ctx) "context", test_event_context, (const void *)list[i]); + torture_suite_add_simple_tcase_const(backend_suite, + "fd1", + test_event_fd1, + (const void *)list[i]); torture_suite_add_suite(suite, backend_suite); } -- 1.8.1.3 From b397b04a527a30ad515d81a4b038589bfe31469d Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 27 Feb 2013 16:43:44 +0100 Subject: [PATCH 71/76] tevent: add test_event_fd2() This test fills the socket kernel buffers and verifies that we don't report TEVENT_FD_WRITE if the buffer is full. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/testsuite.c | 242 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) diff --git a/lib/tevent/testsuite.c b/lib/tevent/testsuite.c index cca7e9a..8e3f4af 100644 --- a/lib/tevent/testsuite.c +++ b/lib/tevent/testsuite.c @@ -421,6 +421,244 @@ static bool test_event_fd1(struct torture_context *tctx, return true; } +struct test_event_fd2_state { + struct torture_context *tctx; + const char *backend; + struct tevent_context *ev; + struct tevent_timer *te; + struct test_event_fd2_sock { + struct test_event_fd2_state *state; + int fd; + struct tevent_fd *fde; + size_t num_written; + size_t num_read; + bool got_full; + } sock0, sock1; + bool finished; + const char *error; +}; + +static void test_event_fd2_sock_handler(struct tevent_context *ev_ctx, + struct tevent_fd *fde, + uint16_t flags, + void *private_data) +{ + struct test_event_fd2_sock *cur_sock = + (struct test_event_fd2_sock *)private_data; + struct test_event_fd2_state *state = cur_sock->state; + struct test_event_fd2_sock *oth_sock = NULL; + uint8_t v = 0, c; + ssize_t ret; + + if (cur_sock == &state->sock0) { + oth_sock = &state->sock1; + } else { + oth_sock = &state->sock0; + } + + if (oth_sock->num_written == 1) { + if (flags != (TEVENT_FD_READ | TEVENT_FD_WRITE)) { + state->finished = true; + state->error = __location__; + return; + } + } + + if (cur_sock->num_read == oth_sock->num_written) { + state->finished = true; + state->error = __location__; + return; + } + + if (!(flags & TEVENT_FD_READ)) { + state->finished = true; + state->error = __location__; + return; + } + + if (oth_sock->num_read > 0) { + /* + * There should be room to write a byte again + */ + if (!(flags & TEVENT_FD_WRITE)) { + state->finished = true; + state->error = __location__; + return; + } + } + + if ((flags & TEVENT_FD_WRITE) && !cur_sock->got_full) { + v = (uint8_t)cur_sock->num_written; + ret = write(cur_sock->fd, &v, 1); + if (ret != 1) { + state->finished = true; + state->error = __location__; + return; + } + cur_sock->num_written++; + if (cur_sock->num_written > 0x80000000) { + state->finished = true; + state->error = __location__; + return; + } + return; + } + + if (!cur_sock->got_full) { + cur_sock->got_full = true; + + if (!oth_sock->got_full) { + /* + * cur_sock is full, + * lets wait for oth_sock + * to be filled + */ + tevent_fd_set_flags(cur_sock->fde, 0); + return; + } + + /* + * oth_sock waited for cur_sock, + * lets restart it + */ + tevent_fd_set_flags(oth_sock->fde, + TEVENT_FD_READ|TEVENT_FD_WRITE); + } + + ret = read(cur_sock->fd, &v, 1); + if (ret != 1) { + state->finished = true; + state->error = __location__; + return; + } + c = (uint8_t)cur_sock->num_read; + if (c != v) { + state->finished = true; + state->error = __location__; + return; + } + cur_sock->num_read++; + + if (cur_sock->num_read < oth_sock->num_written) { + /* there is more to read */ + return; + } + /* + * we read everything, we need to remove TEVENT_FD_WRITE + * to avoid spinning + */ + TEVENT_FD_NOT_WRITEABLE(cur_sock->fde); + + if (oth_sock->num_read == cur_sock->num_written) { + /* + * both directions are finished + */ + state->finished = true; + } + + return; +} + +static void test_event_fd2_finished(struct tevent_context *ev_ctx, + struct tevent_timer *te, + struct timeval tval, + void *private_data) +{ + struct test_event_fd2_state *state = + (struct test_event_fd2_state *)private_data; + + /* + * this should never be triggered + */ + state->finished = true; + state->error = __location__; +} + +static bool test_event_fd2(struct torture_context *tctx, + const void *test_data) +{ + struct test_event_fd2_state state; + int sock[2]; + uint8_t c = 0; + + ZERO_STRUCT(state); + state.tctx = tctx; + state.backend = (const char *)test_data; + + state.ev = tevent_context_init_byname(tctx, state.backend); + if (state.ev == NULL) { + torture_skip(tctx, talloc_asprintf(tctx, + "event backend '%s' not supported\n", + state.backend)); + return true; + } + + tevent_set_debug_stderr(state.ev); + torture_comment(tctx, "backend '%s' - %s\n", + state.backend, __FUNCTION__); + + /* + * This tests the following + * + * - We write 1 byte to each socket + * - We wait for TEVENT_FD_READ/WRITE on both sockets + * - When we get TEVENT_FD_WRITE we write 1 byte + * until both socket buffers are full, which + * means both sockets only get TEVENT_FD_READ. + * - Then we read 1 byte until we have consumed + * all bytes the other end has written. + */ + sock[0] = -1; + sock[1] = -1; + socketpair(AF_UNIX, SOCK_STREAM, 0, sock); + + /* + * the timer should never expire + */ + state.te = tevent_add_timer(state.ev, state.ev, + timeval_current_ofs(600, 0), + test_event_fd2_finished, &state); + state.sock0.state = &state; + state.sock0.fd = sock[0]; + state.sock0.fde = tevent_add_fd(state.ev, state.ev, + state.sock0.fd, + TEVENT_FD_READ | TEVENT_FD_WRITE, + test_event_fd2_sock_handler, + &state.sock0); + state.sock1.state = &state; + state.sock1.fd = sock[1]; + state.sock1.fde = tevent_add_fd(state.ev, state.ev, + state.sock1.fd, + TEVENT_FD_READ | TEVENT_FD_WRITE, + test_event_fd2_sock_handler, + &state.sock1); + + tevent_fd_set_auto_close(state.sock0.fde); + tevent_fd_set_auto_close(state.sock1.fde); + + write(state.sock0.fd, &c, 1); + state.sock0.num_written++; + write(state.sock1.fd, &c, 1); + state.sock1.num_written++; + + while (!state.finished) { + errno = 0; + if (tevent_loop_once(state.ev) == -1) { + talloc_free(state.ev); + torture_fail(tctx, talloc_asprintf(tctx, + "Failed event loop %s\n", + strerror(errno))); + } + } + + talloc_free(state.ev); + + torture_assert(tctx, state.error == NULL, talloc_asprintf(tctx, + "%s", state.error)); + + return true; +} + #ifdef HAVE_PTHREAD static pthread_mutex_t threaded_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -577,6 +815,10 @@ struct torture_suite *torture_local_event(TALLOC_CTX *mem_ctx) "fd1", test_event_fd1, (const void *)list[i]); + torture_suite_add_simple_tcase_const(backend_suite, + "fd2", + test_event_fd2, + (const void *)list[i]); torture_suite_add_suite(suite, backend_suite); } -- 1.8.1.3 From 2ded6f4975a166558dbacaa17c635ac228e9e91b Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 1 Mar 2013 09:25:32 +0100 Subject: [PATCH 72/76] tevent: add a debug message in tevent_common_loop_timer_delay() We should debug a message before and after running the handler. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_timed.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/tevent/tevent_timed.c b/lib/tevent/tevent_timed.c index f7c3969..d3ec8f5 100644 --- a/lib/tevent/tevent_timed.c +++ b/lib/tevent/tevent_timed.c @@ -244,6 +244,10 @@ struct timeval tevent_common_loop_timer_delay(struct tevent_context *ev) * handler we don't want to come across this event again -- vl */ DLIST_REMOVE(ev->timer_events, te); + tevent_debug(te->event_ctx, TEVENT_DEBUG_TRACE, + "Running timer event %p \"%s\"\n", + te, te->handler_name); + /* * If the timed event was registered for a zero current_time, * then we pass a zero timeval here too! To avoid the -- 1.8.1.3 From f5acd8238d49cac756931ba9621f59288985750b Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 22 Feb 2013 12:45:39 +0100 Subject: [PATCH 73/76] tevent: optimize adding new timer events As new timestamps typically get higher:-) it's better to traverse the existing list from the tail. This is not completely optimal, but it should be better than before. A second optimization could be done for zero timestamps, we would just remember the last_zero_timer, but that would change the internal ABI. Normally thatshould not be a poblem, but the Samba's source3/lib/events.c abuses tevent_internal.h from the current source tree, even if an external tevent.h is used. The other problem is that it makes use of tevent_common_add_timer() without using tevent_common_loop_timer_delay(). Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent_timed.c | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/lib/tevent/tevent_timed.c b/lib/tevent/tevent_timed.c index d3ec8f5..fd954f4 100644 --- a/lib/tevent/tevent_timed.c +++ b/lib/tevent/tevent_timed.c @@ -160,7 +160,7 @@ struct tevent_timer *tevent_common_add_timer(struct tevent_context *ev, TALLOC_C const char *handler_name, const char *location) { - struct tevent_timer *te, *last_te, *cur_te; + struct tevent_timer *te, *prev_te, *cur_te; te = talloc(mem_ctx?mem_ctx:ev, struct tevent_timer); if (te == NULL) return NULL; @@ -174,17 +174,33 @@ struct tevent_timer *tevent_common_add_timer(struct tevent_context *ev, TALLOC_C te->additional_data = NULL; /* keep the list ordered */ - last_te = NULL; - for (cur_te = ev->timer_events; cur_te; cur_te = cur_te->next) { - /* if the new event comes before the current one break */ - if (tevent_timeval_compare(&te->next_event, &cur_te->next_event) < 0) { - break; + prev_te = NULL; + /* + * we traverse the list from the tail + * because it's much more likely that + * timers are added at the end of the list + */ + for (cur_te = DLIST_TAIL(ev->timer_events); + cur_te != NULL; + cur_te = DLIST_PREV(cur_te)) + { + int ret; + + /* + * if the new event comes before the current + * we continue searching + */ + ret = tevent_timeval_compare(&te->next_event, + &cur_te->next_event); + if (ret < 0) { + continue; } - last_te = cur_te; + break; } + prev_te = cur_te; - DLIST_ADD_AFTER(ev->timer_events, te, last_te); + DLIST_ADD_AFTER(ev->timer_events, te, prev_te); talloc_set_destructor(te, tevent_common_timed_destructor); -- 1.8.1.3 From 414a527248744db791a66aa480005733f1a479dd Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 22 Feb 2013 12:45:39 +0100 Subject: [PATCH 74/76] tevent: optimize adding new zero timer events Such events were used before we had immediate events. It's likely that there're a lot of this events and we need to add new ones in fifo order. The tricky part is that tevent_common_add_timer() should not use the optimization as it's used by broken Samba versions, which don't use tevent_common_loop_timer_delay() in source3/lib/events.c. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/tevent.c | 1 + lib/tevent/tevent_epoll.c | 2 +- lib/tevent/tevent_internal.h | 14 +++++ lib/tevent/tevent_poll.c | 4 +- lib/tevent/tevent_select.c | 2 +- lib/tevent/tevent_timed.c | 121 +++++++++++++++++++++++++++++++++---------- 6 files changed, 113 insertions(+), 31 deletions(-) diff --git a/lib/tevent/tevent.c b/lib/tevent/tevent.c index 63d5f15..be0afd4 100644 --- a/lib/tevent/tevent.c +++ b/lib/tevent/tevent.c @@ -190,6 +190,7 @@ int tevent_common_context_destructor(struct tevent_context *ev) DLIST_REMOVE(ev->fd_events, fd); } + ev->last_zero_timer = NULL; for (te = ev->timer_events; te; te = tn) { tn = te->next; te->event_ctx = NULL; diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 1ec97c5..599c190 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -937,7 +937,7 @@ static const struct tevent_ops epoll_event_ops = { .set_fd_close_fn = tevent_common_fd_set_close_fn, .get_fd_flags = tevent_common_fd_get_flags, .set_fd_flags = epoll_event_set_fd_flags, - .add_timer = tevent_common_add_timer, + .add_timer = tevent_common_add_timer_v2, .schedule_immediate = tevent_common_schedule_immediate, .add_signal = tevent_common_add_signal, .loop_once = epoll_event_loop_once, diff --git a/lib/tevent/tevent_internal.h b/lib/tevent/tevent_internal.h index 8433333..b239e74 100644 --- a/lib/tevent/tevent_internal.h +++ b/lib/tevent/tevent_internal.h @@ -263,6 +263,13 @@ struct tevent_context { tevent_trace_callback_t callback; void *private_data; } tracing; + + /* + * an optimization pointer into timer_events + * used by used by common code via + * tevent_common_add_timer_v2() + */ + struct tevent_timer *last_zero_timer; }; const struct tevent_ops *tevent_find_ops_byname(const char *name); @@ -292,6 +299,13 @@ struct tevent_timer *tevent_common_add_timer(struct tevent_context *ev, void *private_data, const char *handler_name, const char *location); +struct tevent_timer *tevent_common_add_timer_v2(struct tevent_context *ev, + TALLOC_CTX *mem_ctx, + struct timeval next_event, + tevent_timer_handler_t handler, + void *private_data, + const char *handler_name, + const char *location); struct timeval tevent_common_loop_timer_delay(struct tevent_context *); void tevent_common_schedule_immediate(struct tevent_immediate *im, diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index 0175cae..92fcc44 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -694,7 +694,7 @@ static const struct tevent_ops poll_event_ops = { .set_fd_close_fn = tevent_common_fd_set_close_fn, .get_fd_flags = tevent_common_fd_get_flags, .set_fd_flags = poll_event_set_fd_flags, - .add_timer = tevent_common_add_timer, + .add_timer = tevent_common_add_timer_v2, .schedule_immediate = tevent_common_schedule_immediate, .add_signal = tevent_common_add_signal, .loop_once = poll_event_loop_once, @@ -712,7 +712,7 @@ static const struct tevent_ops poll_event_mt_ops = { .set_fd_close_fn = tevent_common_fd_set_close_fn, .get_fd_flags = tevent_common_fd_get_flags, .set_fd_flags = poll_event_set_fd_flags, - .add_timer = tevent_common_add_timer, + .add_timer = tevent_common_add_timer_v2, .schedule_immediate = poll_event_schedule_immediate, .add_signal = tevent_common_add_signal, .loop_once = poll_event_loop_once, diff --git a/lib/tevent/tevent_select.c b/lib/tevent/tevent_select.c index 5e26569..bfce246 100644 --- a/lib/tevent/tevent_select.c +++ b/lib/tevent/tevent_select.c @@ -264,7 +264,7 @@ static const struct tevent_ops select_event_ops = { .set_fd_close_fn = tevent_common_fd_set_close_fn, .get_fd_flags = tevent_common_fd_get_flags, .set_fd_flags = tevent_common_fd_set_flags, - .add_timer = tevent_common_add_timer, + .add_timer = tevent_common_add_timer_v2, .schedule_immediate = tevent_common_schedule_immediate, .add_signal = tevent_common_add_signal, .loop_once = select_event_loop_once, diff --git a/lib/tevent/tevent_timed.c b/lib/tevent/tevent_timed.c index fd954f4..920d39f 100644 --- a/lib/tevent/tevent_timed.c +++ b/lib/tevent/tevent_timed.c @@ -133,13 +133,18 @@ struct timeval tevent_timeval_current_ofs(uint32_t secs, uint32_t usecs) */ static int tevent_common_timed_destructor(struct tevent_timer *te) { + if (te->event_ctx == NULL) { + return 0; + } + tevent_debug(te->event_ctx, TEVENT_DEBUG_TRACE, "Destroying timer event %p \"%s\"\n", te, te->handler_name); - if (te->event_ctx) { - DLIST_REMOVE(te->event_ctx->timer_events, te); + if (te->event_ctx->last_zero_timer == te) { + te->event_ctx->last_zero_timer = DLIST_PREV(te); } + DLIST_REMOVE(te->event_ctx->timer_events, te); return 0; } @@ -153,12 +158,15 @@ static int tevent_common_timed_deny_destructor(struct tevent_timer *te) add a timed event return NULL on failure (memory allocation error) */ -struct tevent_timer *tevent_common_add_timer(struct tevent_context *ev, TALLOC_CTX *mem_ctx, - struct timeval next_event, - tevent_timer_handler_t handler, - void *private_data, - const char *handler_name, - const char *location) +static struct tevent_timer *tevent_common_add_timer_internal( + struct tevent_context *ev, + TALLOC_CTX *mem_ctx, + struct timeval next_event, + tevent_timer_handler_t handler, + void *private_data, + const char *handler_name, + const char *location, + bool optimize_zero) { struct tevent_timer *te, *prev_te, *cur_te; @@ -173,32 +181,50 @@ struct tevent_timer *tevent_common_add_timer(struct tevent_context *ev, TALLOC_C te->location = location; te->additional_data = NULL; + if (ev->timer_events == NULL) { + ev->last_zero_timer = NULL; + } + /* keep the list ordered */ prev_te = NULL; - /* - * we traverse the list from the tail - * because it's much more likely that - * timers are added at the end of the list - */ - for (cur_te = DLIST_TAIL(ev->timer_events); - cur_te != NULL; - cur_te = DLIST_PREV(cur_te)) - { - int ret; - + if (optimize_zero && tevent_timeval_is_zero(&te->next_event)) { /* - * if the new event comes before the current - * we continue searching + * Some callers use zero tevent_timer + * instead of tevent_immediate events. + * + * As these can happen very often, + * we remember the last zero timer + * in the list. */ - ret = tevent_timeval_compare(&te->next_event, - &cur_te->next_event); - if (ret < 0) { - continue; + prev_te = ev->last_zero_timer; + ev->last_zero_timer = te; + } else { + /* + * we traverse the list from the tail + * because it's much more likely that + * timers are added at the end of the list + */ + for (cur_te = DLIST_TAIL(ev->timer_events); + cur_te != NULL; + cur_te = DLIST_PREV(cur_te)) + { + int ret; + + /* + * if the new event comes before the current + * we continue searching + */ + ret = tevent_timeval_compare(&te->next_event, + &cur_te->next_event); + if (ret < 0) { + continue; + } + + break; } - break; + prev_te = cur_te; } - prev_te = cur_te; DLIST_ADD_AFTER(ev->timer_events, te, prev_te); @@ -210,6 +236,44 @@ struct tevent_timer *tevent_common_add_timer(struct tevent_context *ev, TALLOC_C return te; } +struct tevent_timer *tevent_common_add_timer(struct tevent_context *ev, + TALLOC_CTX *mem_ctx, + struct timeval next_event, + tevent_timer_handler_t handler, + void *private_data, + const char *handler_name, + const char *location) +{ + /* + * do not use optimization, there are broken Samba + * versions which use tevent_common_add_timer() + * without using tevent_common_loop_timer_delay(), + * it just uses DLIST_REMOVE(ev->timer_events, te) + * and would leave ev->last_zero_timer behind. + */ + return tevent_common_add_timer_internal(ev, mem_ctx, next_event, + handler, private_data, + handler_name, location, + false); +} + +struct tevent_timer *tevent_common_add_timer_v2(struct tevent_context *ev, + TALLOC_CTX *mem_ctx, + struct timeval next_event, + tevent_timer_handler_t handler, + void *private_data, + const char *handler_name, + const char *location) +{ + /* + * Here we turn on last_zero_timer optimization + */ + return tevent_common_add_timer_internal(ev, mem_ctx, next_event, + handler, private_data, + handler_name, location, + true); +} + /* do a single event loop using the events defined in ev @@ -258,6 +322,9 @@ struct timeval tevent_common_loop_timer_delay(struct tevent_context *ev) /* We need to remove the timer from the list before calling the * handler because in a semi-async inner event loop called from the * handler we don't want to come across this event again -- vl */ + if (ev->last_zero_timer == te) { + ev->last_zero_timer = DLIST_PREV(te); + } DLIST_REMOVE(ev->timer_events, te); tevent_debug(te->event_ctx, TEVENT_DEBUG_TRACE, -- 1.8.1.3 From 76a493e3bbe7e18ea79e0b3e06da0d3f38a1f329 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 30 Nov 2012 09:43:23 +0100 Subject: [PATCH 75/76] tevent: change version to 0.9.18 This release contains a lot of fixes: - Adding new timer events is now faster, if there's a large number of timer events. - sigprocmask does not work on FreeBSD to stop further signals in a signal handler. - TEVENT_NUM_SIGNALS is calculated by configure in order to support realtime signals on freebsd. - ./configure --disable-python was fixed for the standalone build. - Several crash bugs in the poll backend are fixed. - The poll backend removes deleted events from the cached pollfd array now. - The poll doesn't pass pollfd.events == 0 to poll() and maintains a list of disabled events, instead of consuming 100% cpu and/or triggering the callers handler. - The poll backend detects POLLNVAL and reports EBADF instead of consuming 100% cpu. - The select backend supports separate handlers for TEVENT_FD_READ and TEVENT_FD_WRITE. - The poll and select backends are now doing fair queuing of fd events. - The epoll has better error checking and supports separate handlers for TEVENT_FD_READ and TEVENT_FD_WRITE. - The standard backend was rewritten to be a tiny wrapper on top of epoll with a fallback to poll, which means that it doesn't use select directly anymore. - TEVENT_TRACE_BEFORE_LOOP_ONCE and TEVENT_TRACE_AFTER_LOOP_ONCE are added in order to allow the application to hook in before and after the loop_once() backend function is called. The TEVENT_HAS_LOOP_ONCE_TRACE_POINTS define can be used to detect the new feature. Signed-off-by: Stefan Metzmacher Reviewed-by: Jeremy Allison --- lib/tevent/ABI/tevent-0.9.18.sigs | 83 +++++++++++++++++++++++++++++++++++++++ lib/tevent/wscript | 2 +- 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 lib/tevent/ABI/tevent-0.9.18.sigs diff --git a/lib/tevent/ABI/tevent-0.9.18.sigs b/lib/tevent/ABI/tevent-0.9.18.sigs new file mode 100644 index 0000000..70d20b6 --- /dev/null +++ b/lib/tevent/ABI/tevent-0.9.18.sigs @@ -0,0 +1,83 @@ +_tevent_add_fd: struct tevent_fd *(struct tevent_context *, TALLOC_CTX *, int, uint16_t, tevent_fd_handler_t, void *, const char *, const char *) +_tevent_add_signal: struct tevent_signal *(struct tevent_context *, TALLOC_CTX *, int, int, tevent_signal_handler_t, void *, const char *, const char *) +_tevent_add_timer: struct tevent_timer *(struct tevent_context *, TALLOC_CTX *, struct timeval, tevent_timer_handler_t, void *, const char *, const char *) +_tevent_create_immediate: struct tevent_immediate *(TALLOC_CTX *, const char *) +_tevent_loop_once: int (struct tevent_context *, const char *) +_tevent_loop_until: int (struct tevent_context *, bool (*)(void *), void *, const char *) +_tevent_loop_wait: int (struct tevent_context *, const char *) +_tevent_queue_create: struct tevent_queue *(TALLOC_CTX *, const char *, const char *) +_tevent_req_callback_data: void *(struct tevent_req *) +_tevent_req_cancel: bool (struct tevent_req *, const char *) +_tevent_req_create: struct tevent_req *(TALLOC_CTX *, void *, size_t, const char *, const char *) +_tevent_req_data: void *(struct tevent_req *) +_tevent_req_done: void (struct tevent_req *, const char *) +_tevent_req_error: bool (struct tevent_req *, uint64_t, const char *) +_tevent_req_nomem: bool (const void *, struct tevent_req *, const char *) +_tevent_req_notify_callback: void (struct tevent_req *, const char *) +_tevent_req_oom: void (struct tevent_req *, const char *) +_tevent_schedule_immediate: void (struct tevent_immediate *, struct tevent_context *, tevent_immediate_handler_t, void *, const char *, const char *) +tevent_backend_list: const char **(TALLOC_CTX *) +tevent_cleanup_pending_signal_handlers: void (struct tevent_signal *) +tevent_common_add_fd: struct tevent_fd *(struct tevent_context *, TALLOC_CTX *, int, uint16_t, tevent_fd_handler_t, void *, const char *, const char *) +tevent_common_add_signal: struct tevent_signal *(struct tevent_context *, TALLOC_CTX *, int, int, tevent_signal_handler_t, void *, const char *, const char *) +tevent_common_add_timer: struct tevent_timer *(struct tevent_context *, TALLOC_CTX *, struct timeval, tevent_timer_handler_t, void *, const char *, const char *) +tevent_common_add_timer_v2: struct tevent_timer *(struct tevent_context *, TALLOC_CTX *, struct timeval, tevent_timer_handler_t, void *, const char *, const char *) +tevent_common_check_signal: int (struct tevent_context *) +tevent_common_context_destructor: int (struct tevent_context *) +tevent_common_fd_destructor: int (struct tevent_fd *) +tevent_common_fd_get_flags: uint16_t (struct tevent_fd *) +tevent_common_fd_set_close_fn: void (struct tevent_fd *, tevent_fd_close_fn_t) +tevent_common_fd_set_flags: void (struct tevent_fd *, uint16_t) +tevent_common_loop_immediate: bool (struct tevent_context *) +tevent_common_loop_timer_delay: struct timeval (struct tevent_context *) +tevent_common_loop_wait: int (struct tevent_context *, const char *) +tevent_common_schedule_immediate: void (struct tevent_immediate *, struct tevent_context *, tevent_immediate_handler_t, void *, const char *, const char *) +tevent_context_init: struct tevent_context *(TALLOC_CTX *) +tevent_context_init_byname: struct tevent_context *(TALLOC_CTX *, const char *) +tevent_context_init_ops: struct tevent_context *(TALLOC_CTX *, const struct tevent_ops *, void *) +tevent_debug: void (struct tevent_context *, enum tevent_debug_level, const char *, ...) +tevent_fd_get_flags: uint16_t (struct tevent_fd *) +tevent_fd_set_auto_close: void (struct tevent_fd *) +tevent_fd_set_close_fn: void (struct tevent_fd *, tevent_fd_close_fn_t) +tevent_fd_set_flags: void (struct tevent_fd *, uint16_t) +tevent_get_trace_callback: void (struct tevent_context *, tevent_trace_callback_t *, void *) +tevent_loop_allow_nesting: void (struct tevent_context *) +tevent_loop_set_nesting_hook: void (struct tevent_context *, tevent_nesting_hook, void *) +tevent_queue_add: bool (struct tevent_queue *, struct tevent_context *, struct tevent_req *, tevent_queue_trigger_fn_t, void *) +tevent_queue_add_entry: struct tevent_queue_entry *(struct tevent_queue *, struct tevent_context *, struct tevent_req *, tevent_queue_trigger_fn_t, void *) +tevent_queue_add_optimize_empty: struct tevent_queue_entry *(struct tevent_queue *, struct tevent_context *, struct tevent_req *, tevent_queue_trigger_fn_t, void *) +tevent_queue_length: size_t (struct tevent_queue *) +tevent_queue_running: bool (struct tevent_queue *) +tevent_queue_start: void (struct tevent_queue *) +tevent_queue_stop: void (struct tevent_queue *) +tevent_re_initialise: int (struct tevent_context *) +tevent_register_backend: bool (const char *, const struct tevent_ops *) +tevent_req_default_print: char *(struct tevent_req *, TALLOC_CTX *) +tevent_req_defer_callback: void (struct tevent_req *, struct tevent_context *) +tevent_req_is_error: bool (struct tevent_req *, enum tevent_req_state *, uint64_t *) +tevent_req_is_in_progress: bool (struct tevent_req *) +tevent_req_poll: bool (struct tevent_req *, struct tevent_context *) +tevent_req_post: struct tevent_req *(struct tevent_req *, struct tevent_context *) +tevent_req_print: char *(TALLOC_CTX *, struct tevent_req *) +tevent_req_received: void (struct tevent_req *) +tevent_req_set_callback: void (struct tevent_req *, tevent_req_fn, void *) +tevent_req_set_cancel_fn: void (struct tevent_req *, tevent_req_cancel_fn) +tevent_req_set_endtime: bool (struct tevent_req *, struct tevent_context *, struct timeval) +tevent_req_set_print_fn: void (struct tevent_req *, tevent_req_print_fn) +tevent_set_abort_fn: void (void (*)(const char *)) +tevent_set_debug: int (struct tevent_context *, void (*)(void *, enum tevent_debug_level, const char *, va_list), void *) +tevent_set_debug_stderr: int (struct tevent_context *) +tevent_set_default_backend: void (const char *) +tevent_set_trace_callback: void (struct tevent_context *, tevent_trace_callback_t, void *) +tevent_signal_support: bool (struct tevent_context *) +tevent_timeval_add: struct timeval (const struct timeval *, uint32_t, uint32_t) +tevent_timeval_compare: int (const struct timeval *, const struct timeval *) +tevent_timeval_current: struct timeval (void) +tevent_timeval_current_ofs: struct timeval (uint32_t, uint32_t) +tevent_timeval_is_zero: bool (const struct timeval *) +tevent_timeval_set: struct timeval (uint32_t, uint32_t) +tevent_timeval_until: struct timeval (const struct timeval *, const struct timeval *) +tevent_timeval_zero: struct timeval (void) +tevent_trace_point_callback: void (struct tevent_context *, enum tevent_trace_point) +tevent_wakeup_recv: bool (struct tevent_req *) +tevent_wakeup_send: struct tevent_req *(TALLOC_CTX *, struct tevent_context *, struct timeval) diff --git a/lib/tevent/wscript b/lib/tevent/wscript index 71b1f61..02bddb8 100755 --- a/lib/tevent/wscript +++ b/lib/tevent/wscript @@ -1,7 +1,7 @@ #!/usr/bin/env python APPNAME = 'tevent' -VERSION = '0.9.17' +VERSION = '0.9.18' blddir = 'bin' -- 1.8.1.3 From 5e40c99e3ec26e22c425d6a4a78ed348a7ee7728 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Fri, 1 Mar 2013 16:10:47 -0800 Subject: [PATCH 76/76] Fix typo to make master/lib/tevent identical to 4.0.x/lib/tevent This was missed in the patch that was applied to fix bug 9550. Signed-off-by: Jeremy Allison --- lib/tevent/tevent_signal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tevent/tevent_signal.c b/lib/tevent/tevent_signal.c index 0b637dc..b5a56ef 100644 --- a/lib/tevent/tevent_signal.c +++ b/lib/tevent/tevent_signal.c @@ -142,7 +142,7 @@ static void tevent_common_signal_handler_info(int signum, siginfo_t *info, * By default signum is blocked inside this * signal handler, but calling sigprocmask * modifies the temporary signal mask being - * used *inside* this hander, which will be + * used *inside* this handler, which will be * replaced by the callers signal mask once * we return from here. See Samba * bug #9550 for details. -- 1.8.1.3