app and poller initialization is now properly synchronized

apps/pingpong/main.c

@@ -176,7 +176,6 @@ int main(int argc, char *argv[])
         goto pingpong_end;
         return -1;
     }
-    sleep(2);
     printf("App: UDPDK Intialized\n");
 
     // Parse app-specific arguments

apps/pktgen/main.c

@@ -300,7 +300,6 @@ int main(int argc, char *argv[])
         goto pktgen_end;
         return -1;
     }
-    sleep(2);
     printf("App: UDPDK Intialized\n");
 
     // Parse app-specific arguments

udpdk/Makefile

@@ -46,6 +46,7 @@ UDPDK_CORE_SRCS+=    \
 	udpdk_monitor.c  \
 	udpdk_poller.c   \
 	udpdk_syscall.c  \
+    udpdk_sync.c     \
 
 UDPDK_LIST_SRCS+=    \
     list/udpdk_list.c      \

udpdk/udpdk_globals.c

@@ -29,3 +29,9 @@ struct rte_mempool *tx_pktmbuf_indirect_pool = NULL;
 struct exch_zone_info *exch_zone_desc = NULL;
 
 struct exch_slot *exch_slots = NULL;
+
+struct rte_ring *ipc_app_to_pol = NULL;
+
+struct rte_ring *ipc_pol_to_app = NULL;
+
+struct rte_mempool *ipc_msg_pool = NULL;

udpdk/udpdk_init.c

@@ -28,6 +28,7 @@
 #include "udpdk_bind_table.h"
 #include "udpdk_monitor.h"
 #include "udpdk_poller.h"
+#include "udpdk_sync.h"
 #include "udpdk_types.h"
 
 #define RTE_LOGTYPE_INIT RTE_LOGTYPE_USER1
@@ -46,6 +47,9 @@ extern int primary_argc;
 extern int secondary_argc;
 extern char *primary_argv[MAX_ARGC];
 extern char *secondary_argv[MAX_ARGC];
+extern struct rte_ring *ipc_app_to_pol;
+extern struct rte_ring *ipc_pol_to_app;
+extern struct rte_mempool *ipc_msg_pool;
 static pid_t poller_pid;
 
 
@@ -311,6 +315,13 @@ int udpdk_init(int argc, char *argv[])
             RTE_LOG(INFO, INIT, "Using the same port for RX and TX\n");
         }
 
+        // Initialize IPC channel to synchronize with the poller
+        retval = init_ipc_channel();
+        if (retval < 0) {
+            RTE_LOG(ERR, INIT, "Cannot initialize IPC channel for app-poller synchronization\n");
+            return -1;
+        }
+
         // Initialize memzone for exchange
         retval = init_exch_memzone();
         if (retval < 0) {
@@ -329,8 +340,14 @@ int udpdk_init(int argc, char *argv[])
             RTE_LOG(ERR, INIT, "Cannot initialize exchange slots\n");
             return -1;
         }
+
+        // Let the poller process resume initialization
+        ipc_notify_to_poller();
+        
+        // Wait for the poller to be fully initialized
+        RTE_LOG(INFO, INIT, "Waiting for the poller to complete its inialization...\n");
+        ipc_wait_for_poller();
     } else {  // child -> packet poller
-        sleep(1);   // TODO use some synchronization mechanism between primary and secondary
         if (poller_init(secondary_argc, (char **)secondary_argv) < 0) {
             RTE_LOG(INFO, INIT, "Poller initialization failed\n");
             return -1;

udpdk/udpdk_poller.c

@@ -27,6 +27,7 @@
 #include "udpdk_constants.h"
 #include "udpdk_bind_table.h"
 #include "udpdk_shmalloc.h"
+#include "udpdk_sync.h"
 #include "udpdk_types.h"
 
 #define RTE_LOGTYPE_POLLBODY RTE_LOGTYPE_USER1
@@ -39,6 +40,9 @@ extern struct exch_zone_info *exch_zone_desc;
 extern struct exch_slot *exch_slots;
 extern udpdk_list_t **sock_bind_table;
 extern const void *bind_info_alloc;
+extern struct rte_ring *ipc_app_to_pol;
+extern struct rte_ring *ipc_pol_to_app;
+extern struct rte_mempool *ipc_msg_pool;
 
 /* Descriptor of a RX queue */
 struct rx_queue {
@@ -214,6 +218,16 @@ int poller_init(int argc, char *argv[])
         return -1;
     }
 
+    // Initialize the IPC channel to synchronize with the app
+    while (retrieve_ipc_channel() < 0) {
+        RTE_LOG(INFO, POLLINIT, "Waiting to initialize IPC...\n");
+        sleep(1);
+    }
+    RTE_LOG(INFO, POLLINIT, "IPC initialized\n");
+
+    // Wait for a synchronization signal from the application process before proceeding
+    ipc_wait_for_app();
+
     // Setup memory allocators
     retval = setup_allocators();
     if (retval < 0) {
@@ -246,6 +260,9 @@ int poller_init(int argc, char *argv[])
     signal(SIGINT, poller_sighandler);
     signal(SIGTERM, poller_sighandler);
 
+    // Notify the primary about the successful initialization
+    ipc_notify_to_app();
+
     return 0;
 }
 

udpdk/udpdk_sync.c

+//
+// Created by leoll2 on 11/13/20.
+// Copyright (c) 2020 Leonardo Lai. All rights reserved.
+//
+
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_lcore.h>
+#include <rte_log.h>
+#include <rte_mempool.h>
+
+#include "udpdk_sync.h"
+
+#define WAIT_MAX_CYCLES  100
+extern struct rte_ring *ipc_app_to_pol;
+extern struct rte_ring *ipc_pol_to_app;
+extern struct rte_mempool *ipc_msg_pool;
+
+
+/* Initialize IPC channel for the synchonization between app and poller processes */
+int init_ipc_channel(void)
+{
+    ipc_app_to_pol = rte_ring_create("IPC_channel_app_to_pol", 1, rte_socket_id(), RING_F_SP_ENQ | RING_F_SC_DEQ);
+    if (ipc_app_to_pol == NULL) {
+        return -1;
+    }
+    ipc_pol_to_app = rte_ring_create("IPC_channel_pol_to_app", 1, rte_socket_id(), RING_F_SP_ENQ | RING_F_SC_DEQ);
+    if (ipc_pol_to_app == NULL) {
+        return -1;
+    }
+    ipc_msg_pool = rte_mempool_create("IPC_msg_pool", 5, 64, 0, 0, NULL, NULL, NULL, NULL, rte_socket_id(), 0);
+    return 0;
+}
+
+/* Retrieve IPC channel for the synchonization between app and poller processes */
+int retrieve_ipc_channel(void)
+{
+    ipc_app_to_pol = rte_ring_lookup("IPC_channel_app_to_pol");
+    if (ipc_app_to_pol == NULL) {
+        return -1;
+    }
+    ipc_pol_to_app = rte_ring_lookup("IPC_channel_pol_to_app");
+    if (ipc_pol_to_app == NULL) {
+        return -1;
+    }
+    ipc_msg_pool = rte_mempool_lookup("IPC_msg_pool");
+    if (ipc_msg_pool == NULL) {
+        return -1;
+    }
+    return 0;
+}
+
+/* Wait for a signal from the application to the poller */
+int ipc_wait_for_app(void)
+{
+    void *sync_msg;
+    int c = 0;
+
+    while (rte_ring_dequeue(ipc_app_to_pol, &sync_msg) < 0) {
+        usleep(50000);
+        c++;
+        if (c > WAIT_MAX_CYCLES) {
+            return -1;
+        }
+    }
+    rte_mempool_put(ipc_msg_pool, sync_msg);
+    return 0;
+}
+
+/* Wait for a signal from the poller to the application */
+int ipc_wait_for_poller(void)
+{
+    void *sync_msg;
+    int c = 0;
+
+    while (rte_ring_dequeue(ipc_pol_to_app, &sync_msg) < 0) {
+        usleep(50000);
+        c++;
+        if (c > WAIT_MAX_CYCLES) {
+            return -1;
+        }
+    }
+    rte_mempool_put(ipc_msg_pool, sync_msg);
+    return 0;
+}
+
+/* Send a notification signal from the poller to the application */
+void ipc_notify_to_app(void)
+{
+    void *sync_msg;
+
+    if (rte_mempool_get(ipc_msg_pool, &sync_msg) < 0) {
+        rte_panic("Failed to get a sync message from the pool\n");
+    }
+    // NOTE: no need to write anything in the message
+    if (rte_ring_enqueue(ipc_pol_to_app, sync_msg) < 0) {
+        rte_mempool_put(ipc_msg_pool, sync_msg);
+    }
+}
+
+/* Send a notification signal from the application to the poller */
+void ipc_notify_to_poller(void)
+{
+    void *sync_msg;
+
+    if (rte_mempool_get(ipc_msg_pool, &sync_msg) < 0) {
+        rte_panic("Failed to get a sync message from the pool\n");
+    }
+    // NOTE: no need to write anything in the message
+    if (rte_ring_enqueue(ipc_app_to_pol, sync_msg) < 0) {
+        rte_mempool_put(ipc_msg_pool, sync_msg);
+    }
+}

udpdk/udpdk_sync.h

+//
+// Created by leoll2 on 11/13/20.
+// Copyright (c) 2020 Leonardo Lai. All rights reserved.
+//
+
+#ifndef UDPDK_SYNC_H
+#define UDPDK_SYNC_H
+
+int init_ipc_channel(void);
+
+int retrieve_ipc_channel(void);
+
+int ipc_wait_for_app(void);
+
+int ipc_wait_for_poller(void);
+
+void ipc_notify_to_app(void);
+
+void ipc_notify_to_poller(void);
+
+#endif //UDPDK_SYNC_H