Fixing potential AsyncInvoker deadlock that occurs for "reentrant" invocations.

webrtc/base/thread_unittest.cc

Index: webrtc/base/thread_unittest.cc
diff --git a/webrtc/base/thread_unittest.cc b/webrtc/base/thread_unittest.cc
index c143120fbcceec8d7611b42156761989102213ff..c8f9e353210fcfa615012bbb1c09584edc84d063 100644
--- a/webrtc/base/thread_unittest.cc
+++ b/webrtc/base/thread_unittest.cc
@@ -104,7 +104,7 @@ class MessageClient : public MessageHandler, public TestGenerator {
   Socket* socket_;
 };
 
-class CustomThread : public rtc::Thread {
+class CustomThread : public Thread {
  public:
   CustomThread() {}
   virtual ~CustomThread() { Stop(); }
@@ -150,7 +150,7 @@ class SignalWhenDestroyedThread : public Thread {
 
 // Using std::atomic<bool> or std::atomic_flag in C++11 is probably
 // the right thing to do, but those features are not yet allowed. Or
-// rtc::AtomicInt, if/when that is added. Since the use isn't
+// AtomicInt, if/when that is added. Since the use isn't
 // performance critical, use a plain critical section for the time
 // being.
 
@@ -451,27 +451,23 @@ TEST_F(AsyncInvokeTest, KillInvokerDuringExecute) {
   // executing, and then to wait for it to finish, ensuring the "EXPECT_FALSE"
   // is run.
   Event functor_started(false, false);
-  Event functor_continue(false, false);
   Event functor_finished(false, false);
 
   Thread thread;
   thread.Start();
   volatile bool invoker_destroyed = false;
   {
+    auto functor = [&functor_started, &functor_finished, &invoker_destroyed] {
+      functor_started.Set();
+      Thread::Current()->SleepMs(kWaitTimeout);
+      EXPECT_FALSE(invoker_destroyed);
+      functor_finished.Set();
+    };
     AsyncInvoker invoker;
-    invoker.AsyncInvoke<void>(RTC_FROM_HERE, &thread,
-                              [&functor_started, &functor_continue,
-                               &functor_finished, &invoker_destroyed] {
-                                functor_started.Set();
-                                functor_continue.Wait(Event::kForever);
-                                rtc::Thread::Current()->SleepMs(kWaitTimeout);
-                                EXPECT_FALSE(invoker_destroyed);
-                                functor_finished.Set();
-                              });
+    invoker.AsyncInvoke<void>(RTC_FROM_HERE, &thread, functor);
     functor_started.Wait(Event::kForever);
-
-    // Allow the functor to continue and immediately destroy the invoker.
-    functor_continue.Set();
+    // Destroy the invoker while the functor is still executing (doing
+    // SleepMs).
   }
 
   // If the destructor DIDN'T wait for the functor to finish executing, it will
@@ -481,6 +477,35 @@ TEST_F(AsyncInvokeTest, KillInvokerDuringExecute) {
   functor_finished.Wait(Event::kForever);
 }
 
+// Variant of the above test where the async-invoked task calls AsyncInvoke
+// again, for the thread on which the AsyncInvoker is currently being
+// destroyed. This shouldn't deadlock or crash; this second invocation should
+// just be ignored.
+TEST_F(AsyncInvokeTest, KillInvokerDuringExecuteWithReentrantInvoke) {
+  Event functor_started(false, false);
+  bool reentrant_functor_run = false;
+
+  Thread* main = Thread::Current();
+  Thread thread;
+  thread.Start();
+  {
+    AsyncInvoker invoker;
+    auto reentrant_functor = [&reentrant_functor_run] {
+      reentrant_functor_run = true;
+    };
+    auto functor = [&functor_started, &invoker, main, reentrant_functor] {
+      functor_started.Set();
+      Thread::Current()->SleepMs(kWaitTimeout);
+      invoker.AsyncInvoke<void>(RTC_FROM_HERE, main, reentrant_functor);
+    };
+    // This queues a task on |thread| to sleep for |kWaitTimeout| then queue a
+    // task on |main|. But this second queued task should never run.
+    invoker.AsyncInvoke<void>(RTC_FROM_HERE, &thread, functor);
+    functor_started.Wait(Event::kForever);
+  }

[nisse-webrtc, 2017/05/18 08:04:55]

I'm trying to understand what happens here... Scope ends, so AsyncInvoker is
destroyed. And its destructor is going to block until |functor| has returned,
which takes about 1s?

That |reentrant_functor| never runs, is that due to the |destroying_| flag
(seems a bit scary, since it's set by the destructor running on main thread, and
read by the AsyncInvoke call on the other thread, with no synchronization), or
due to the MessageQueueManager::Clear call?

To me, it seems the synchronization primitive needed to make things easy is to
atomically {check that destroying_ is false and if so, increment the reference
count}. But seems difficult to implement without locks.

+  EXPECT_FALSE(reentrant_functor_run);
+}
+
 TEST_F(AsyncInvokeTest, Flush) {
   AsyncInvoker invoker;
   AtomicBool flag1;