SystemLayerImplSelect: add libev support when CHIP_SYSTEM_CONFIG_USE_LIBEV=1

When CHIP_SYSTEM_CONFIG_USE_LIBEV is set, SystemLayerImplSelect expects a *libev* mainloop to be present to schedule timers and socket watches (similar to CHIP_SYSTEM_CONFIG_USE_DISPATCH for Darwin).

A libev mainloop must be passed to SystemLayer using `SetLibEvLoop()` before any timers or socket watches are used - otherwise, `chipDie()` is invoked.

# Usage
The entire project needs to be build with `CHIP_SYSTEM_CONFIG_USE_LIBEV=1` (this can be done via invoking a project-specific extra config via the `default_configs_extra` argument in args.gni)

Setting up the libev mainloop and handing it over to SystemLayer must be done in application specific code, outside the code provided by chip examples. Also adding libev as a dependency must be done in the application's BUILD.gn.

# Background
*libev* is a multi-platform event library often used in embedded linux context to handle events, and builds the basis for various libraries with non-blocking APIs. This changeset allows using the *connectedhomeip* stack with libev based applications.

# Example
The opensource bridge project p44mbrd (https://github.com/plan44/p44mbrd) is based on libev and makes use of this changeset.

Author: plan44

src/platform/Darwin/PlatformManagerImpl.cpp

@@ -52,8 +52,10 @@ CHIP_ERROR PlatformManagerImpl::_InitChipStack()
     SuccessOrExit(err);
 #endif // CHIP_DISABLE_PLATFORM_KVS
 
+#if !CHIP_SYSTEM_CONFIG_USE_LIBEV
     // Ensure there is a dispatch queue available
     static_cast<System::LayerSocketsLoop &>(DeviceLayer::SystemLayer()).SetDispatchQueue(GetWorkQueue());
+#endif
 
     // Call _InitChipStack() on the generic implementation base class
     // to finish the initialization process.

src/platform/Darwin/SystemPlatformConfig.h

@@ -34,9 +34,14 @@ struct ChipDeviceEvent;
 
 // ==================== Platform Adaptations ====================
 
+#if !CHIP_SYSTEM_CONFIG_USE_LIBEV
+// FIXME: these should not be hardcoded here, it is set via build config
+// Need to exclude these for now in libev case
 #define CHIP_SYSTEM_CONFIG_POSIX_LOCKING 0
 #define CHIP_SYSTEM_CONFIG_FREERTOS_LOCKING 0
 #define CHIP_SYSTEM_CONFIG_NO_LOCKING 1
+#endif
+
 #define CHIP_SYSTEM_CONFIG_PLATFORM_PROVIDES_TIME 1
 #define CHIP_SYSTEM_CONFIG_USE_POSIX_TIME_FUNCTS 1
 #define CHIP_SYSTEM_CONFIG_POOL_USE_HEAP 1

src/system/BUILD.gn

@@ -77,6 +77,7 @@ buildconfig_header("system_buildconfig") {
     "CHIP_SYSTEM_CONFIG_TEST=${chip_build_tests}",
     "CHIP_WITH_NLFAULTINJECTION=${chip_with_nlfaultinjection}",
     "CHIP_SYSTEM_CONFIG_USE_DISPATCH=${chip_system_config_use_dispatch}",
+    "CHIP_SYSTEM_CONFIG_USE_LIBEV=${chip_system_config_use_libev}",
     "CHIP_SYSTEM_CONFIG_USE_LWIP=${chip_system_config_use_lwip}",
     "CHIP_SYSTEM_CONFIG_USE_OPEN_THREAD_ENDPOINT=${chip_system_config_use_open_thread_inet_endpoints}",
     "CHIP_SYSTEM_CONFIG_USE_SOCKETS=${chip_system_config_use_sockets}",

src/system/SystemLayer.h

@@ -43,7 +43,9 @@
 
 #if CHIP_SYSTEM_CONFIG_USE_DISPATCH
 #include <dispatch/dispatch.h>
-#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH
+#elif CHIP_SYSTEM_CONFIG_USE_LIBEV
+#include <ev.h>
+#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH/LIBEV
 
 #include <utility>
 
@@ -250,7 +252,9 @@ class LayerSocketsLoop : public LayerSockets
 #if CHIP_SYSTEM_CONFIG_USE_DISPATCH
     virtual void SetDispatchQueue(dispatch_queue_t dispatchQueue) = 0;
     virtual dispatch_queue_t GetDispatchQueue()                   = 0;
-#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH
+#elif CHIP_SYSTEM_CONFIG_USE_LIBEV
+    virtual void SetLibEvLoop(struct ev_loop * aLibEvLoopP) = 0;
+#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH/LIBEV
 };
 
 #endif // CHIP_SYSTEM_CONFIG_USE_SOCKETS

src/system/SystemLayerImplSelect.cpp

@@ -35,6 +35,17 @@
 #define PTHREAD_NULL 0
 #endif // CHIP_SYSTEM_CONFIG_POSIX_LOCKING && !defined(PTHREAD_NULL)
 
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+// older libev do not yet have ev_io_modify
+#ifndef ev_io_modify
+#define ev_io_modify(ev, events_)                                                                                                  \
+    do                                                                                                                             \
+    {                                                                                                                              \
+        (ev)->events = ((ev)->events & EV__IOFDSET) | (events_);                                                                   \
+    } while (0)
+#endif // ev_io_modify
+#endif // CHIP_SYSTEM_CONFIG_USE_LIBEV
+
 namespace chip {
 namespace System {
 
@@ -82,11 +93,25 @@ void LayerImplSelect::Shutdown()
     {
         w.DisableAndClear();
     }
+#elif CHIP_SYSTEM_CONFIG_USE_LIBEV
+    TimerList::Node * timer;
+    while ((timer = mTimerList.PopEarliest()) != nullptr)
+    {
+        if (ev_is_active(&timer->mLibEvTimer))
+        {
+            ev_timer_stop(mLibEvLoopP, &timer->mLibEvTimer);
+        }
+    }
+    mTimerPool.ReleaseAll();
 
-#else  // CHIP_SYSTEM_CONFIG_USE_DISPATCH
+    for (auto & w : mSocketWatchPool)
+    {
+        w.DisableAndClear();
+    }
+#else
     mTimerList.Clear();
     mTimerPool.ReleaseAll();
-#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH
+#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH/LIBEV
 
     mWakeEvent.Close(*this);
 
@@ -155,14 +180,28 @@ CHIP_ERROR LayerImplSelect::StartTimer(Clock::Timeout delay, TimerCompleteCallba
         dispatch_resume(timerSource);
         return CHIP_NO_ERROR;
     }
-#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH
-
+#elif CHIP_SYSTEM_CONFIG_USE_LIBEV
+    if (mLibEvLoopP == nullptr)
+    {
+        chipDie();
+    }
+    ev_timer_init(&timer->mLibEvTimer, &LayerImplSelect::HandleLibEvTimer, 1, 0);
+    timer->mLibEvTimer.data = timer;
+    auto t                  = Clock::Milliseconds64(delay).count();
+    ev_timer_set(&timer->mLibEvTimer, static_cast<double>(t) / 1E3, 0.);
+    (void) mTimerList.Add(timer);
+    ev_timer_start(mLibEvLoopP, &timer->mLibEvTimer);
+    return CHIP_NO_ERROR;
+#endif
+#if !CHIP_SYSTEM_CONFIG_USE_LIBEV
+    // Note: dispatch based implementation needs this as fallback, but not LIBEV (and dead code is not allowed with -Werror)
     if (mTimerList.Add(timer) == timer)
     {
         // The new timer is the earliest, so the time until the next event has probably changed.
         Signal();
     }
     return CHIP_NO_ERROR;
+#endif // !CHIP_SYSTEM_CONFIG_USE_LIBEV
 }
 
 void LayerImplSelect::CancelTimer(TimerCompleteCallback onComplete, void * appState)
@@ -186,7 +225,13 @@ void LayerImplSelect::CancelTimer(TimerCompleteCallback onComplete, void * appSt
         dispatch_source_cancel(timer->mTimerSource);
         dispatch_release(timer->mTimerSource);
     }
-#endif
+#elif CHIP_SYSTEM_CONFIG_USE_LIBEV
+    if (mLibEvLoopP == nullptr)
+    {
+        chipDie();
+    }
+    ev_timer_stop(mLibEvLoopP, &timer->mLibEvTimer);
+#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH/LIBEV
 
     mTimerPool.Release(timer);
     Signal();
@@ -205,7 +250,12 @@ CHIP_ERROR LayerImplSelect::ScheduleWork(TimerCompleteCallback onComplete, void
         });
         return CHIP_NO_ERROR;
     }
-#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH
+#elif CHIP_SYSTEM_CONFIG_USE_LIBEV
+    // just a timer with no delay
+    return StartTimer(Clock::Timeout(0), onComplete, appState);
+#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH/LIBEV
+#if !CHIP_SYSTEM_CONFIG_USE_LIBEV
+    // Note: dispatch based implementation needs this as fallback, but not LIBEV (and dead code is not allowed with -Werror)
 
     // Ideally we would not use a timer here at all, but if we try to just
     // ScheduleLambda the lambda needs to capture the following:
@@ -241,6 +291,7 @@ CHIP_ERROR LayerImplSelect::ScheduleWork(TimerCompleteCallback onComplete, void
         Signal();
     }
     return CHIP_NO_ERROR;
+#endif // !CHIP_SYSTEM_CONFIG_USE_LIBEV
 }
 
 CHIP_ERROR LayerImplSelect::StartWatchingSocket(int fd, SocketWatchToken * tokenOut)
@@ -262,6 +313,11 @@ CHIP_ERROR LayerImplSelect::StartWatchingSocket(int fd, SocketWatchToken * token
     VerifyOrReturnError(watch != nullptr, CHIP_ERROR_ENDPOINT_POOL_FULL);
 
     watch->mFD = fd;
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+    ev_io_init(&watch->mIoWatcher, &LayerImplSelect::HandleLibEvIoWatcher, 0, 0);
+    watch->mIoWatcher.data   = watch;
+    watch->mLayerImplSelectP = this;
+#endif
 
     *tokenOut = reinterpret_cast<SocketWatchToken>(watch);
     return CHIP_NO_ERROR;
@@ -314,6 +370,27 @@ CHIP_ERROR LayerImplSelect::RequestCallbackOnPendingRead(SocketWatchToken token)
             dispatch_activate(watch->mRdSource);
         }
     }
+#elif CHIP_SYSTEM_CONFIG_USE_LIBEV
+    if (mLibEvLoopP == nullptr)
+    {
+        chipDie();
+    }
+    int evs = (watch->mPendingIO.Has(SocketEventFlags::kRead) ? EV_READ : 0) |
+        (watch->mPendingIO.Has(SocketEventFlags::kWrite) ? EV_WRITE : 0);
+    if (!ev_is_active(&watch->mIoWatcher))
+    {
+        // First time actually using that watch
+        ev_io_set(&watch->mIoWatcher, watch->mFD, evs);
+        ev_io_start(mLibEvLoopP, &watch->mIoWatcher);
+    }
+    else
+    {
+        // already active, just change flags
+        // Note: changing flags only reliably works when the watcher is stopped
+        ev_io_stop(mLibEvLoopP, &watch->mIoWatcher);
+        ev_io_modify(&watch->mIoWatcher, evs);
+        ev_io_start(mLibEvLoopP, &watch->mIoWatcher);
+    }
 #endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH
 
     return CHIP_NO_ERROR;
@@ -353,10 +430,30 @@ CHIP_ERROR LayerImplSelect::RequestCallbackOnPendingWrite(SocketWatchToken token
                 }
             });
             // only now we are sure the source exists and can become active
-            watch->mPendingIO.Set(SocketEventFlags::kWrite);
             dispatch_activate(watch->mWrSource);
         }
     }
+#elif CHIP_SYSTEM_CONFIG_USE_LIBEV
+    if (mLibEvLoopP == nullptr)
+    {
+        chipDie();
+    }
+    int evs = (watch->mPendingIO.Has(SocketEventFlags::kRead) ? EV_READ : 0) |
+        (watch->mPendingIO.Has(SocketEventFlags::kWrite) ? EV_WRITE : 0);
+    if (!ev_is_active(&watch->mIoWatcher))
+    {
+        // First time actually using that watch
+        ev_io_set(&watch->mIoWatcher, watch->mFD, evs);
+        ev_io_start(mLibEvLoopP, &watch->mIoWatcher);
+    }
+    else
+    {
+        // already active, just change flags
+        // Note: changing flags only reliably works when the watcher is stopped
+        ev_io_stop(mLibEvLoopP, &watch->mIoWatcher);
+        ev_io_modify(&watch->mIoWatcher, evs);
+        ev_io_start(mLibEvLoopP, &watch->mIoWatcher);
+    }
 #endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH
 
     return CHIP_NO_ERROR;
@@ -369,6 +466,14 @@ CHIP_ERROR LayerImplSelect::ClearCallbackOnPendingRead(SocketWatchToken token)
 
     watch->mPendingIO.Clear(SocketEventFlags::kRead);
 
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+    if (ev_is_active(&watch->mIoWatcher) && watch->mPendingIO.Raw() == 0)
+    {
+        // all flags cleared now, stop watching
+        ev_io_stop(mLibEvLoopP, &watch->mIoWatcher);
+    }
+#endif
+
     return CHIP_NO_ERROR;
 }
 
@@ -379,6 +484,14 @@ CHIP_ERROR LayerImplSelect::ClearCallbackOnPendingWrite(SocketWatchToken token)
 
     watch->mPendingIO.Clear(SocketEventFlags::kWrite);
 
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+    if (ev_is_active(&watch->mIoWatcher) && watch->mPendingIO.Raw() == 0)
+    {
+        // all flags cleared now, stop watching
+        ev_io_stop(mLibEvLoopP, &watch->mIoWatcher);
+    }
+#endif
+
     return CHIP_NO_ERROR;
 }
 
@@ -390,7 +503,7 @@ CHIP_ERROR LayerImplSelect::StopWatchingSocket(SocketWatchToken * tokenInOut)
     VerifyOrReturnError(watch != nullptr, CHIP_ERROR_INVALID_ARGUMENT);
     VerifyOrReturnError(watch->mFD >= 0, CHIP_ERROR_INCORRECT_STATE);
 
-#if CHIP_SYSTEM_CONFIG_USE_DISPATCH
+#if CHIP_SYSTEM_CONFIG_USE_DISPATCH || CHIP_SYSTEM_CONFIG_USE_LIBEV
     watch->DisableAndClear();
 #else
     watch->Clear();
@@ -526,12 +639,47 @@ void LayerImplSelect::HandleEvents()
 }
 
 #if CHIP_SYSTEM_CONFIG_USE_DISPATCH
+
 void LayerImplSelect::HandleTimerComplete(TimerList::Node * timer)
 {
     mTimerList.Remove(timer);
     mTimerPool.Invoke(timer);
 }
-#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH
+
+#elif CHIP_SYSTEM_CONFIG_USE_LIBEV
+
+void LayerImplSelect::HandleLibEvTimer(EV_P_ struct ev_timer * t, int revents)
+{
+    TimerList::Node * timer = static_cast<TimerList::Node *>(t->data);
+    VerifyOrDie(timer);
+    LayerImplSelect * layerP = dynamic_cast<LayerImplSelect *>(timer->mCallback.mSystemLayer);
+    VerifyOrDie(layerP);
+    layerP->mTimerList.Remove(timer);
+    layerP->mTimerPool.Invoke(timer);
+}
+
+void LayerImplSelect::HandleLibEvIoWatcher(EV_P_ struct ev_io * i, int revents)
+{
+    SocketWatch * watch = static_cast<SocketWatch *>(i->data);
+    if (watch != nullptr && watch->mCallback != nullptr && watch->mLayerImplSelectP != nullptr)
+    {
+        SocketEvents events;
+        if (revents & EV_READ)
+        {
+            events.Set(SocketEventFlags::kRead);
+        }
+        if (revents & EV_WRITE)
+        {
+            events.Set(SocketEventFlags::kWrite);
+        }
+        if (events.HasAny())
+        {
+            watch->mCallback(events, watch->mCallbackData);
+        }
+    }
+}
+
+#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH/LIBEV
 
 void LayerImplSelect::SocketWatch::Clear()
 {
@@ -542,6 +690,8 @@ void LayerImplSelect::SocketWatch::Clear()
 #if CHIP_SYSTEM_CONFIG_USE_DISPATCH
     mRdSource = nullptr;
     mWrSource = nullptr;
+#elif CHIP_SYSTEM_CONFIG_USE_LIBEV
+    mLayerImplSelectP = nullptr;
 #endif
 }
 
@@ -560,7 +710,16 @@ void LayerImplSelect::SocketWatch::DisableAndClear()
     }
     Clear();
 }
-#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH
+#elif CHIP_SYSTEM_CONFIG_USE_LIBEV
+void LayerImplSelect::SocketWatch::DisableAndClear()
+{
+    if (mLayerImplSelectP != nullptr && mLayerImplSelectP->mLibEvLoopP != nullptr)
+    {
+        ev_io_stop(mLayerImplSelectP->mLibEvLoopP, &mIoWatcher);
+    }
+    Clear();
+}
+#endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH/LIBEV
 
 } // namespace System
 } // namespace chip