gpdb: send dummy packet failed, sendto failed

#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <netdb.h>
#include <netinet/in.h>   // Needed for sockets stuff
#include <arpa/inet.h>    // Needed for sockets stuff
#include <fcntl.h>        // Needed for sockets stuff
#include <stdbool.h>

#define closesocket close
#define PGINVALID_SOCKET (-1)

typedef int pgsocket;
typedef uint32_t uint32;
static struct sockaddr_storage udp_dummy_packet_sockaddr;
static char *interconnect_address = "127.0.0.1";

int pg_getaddrinfo_all(const char *hostname, const char *servname,
                       const struct addrinfo *hintp, struct addrinfo **result)
{
    int			rc;

    /* not all versions of getaddrinfo() zero *result on failure */
    *result = NULL;

#ifdef HAVE_UNIX_SOCKETS
    if (hintp->ai_family == AF_UNIX)
		return getaddrinfo_unix(servname, hintp, result);
#endif

    /* NULL has special meaning to getaddrinfo(). */
    rc = getaddrinfo((!hostname || hostname[0] == '\0') ? NULL : hostname,
                     servname, hintp, result);

    return rc;
}

bool pg_set_noblock(pgsocket sock)
{
#if !defined(WIN32)
    int			flags;

    flags = fcntl(sock, F_GETFL);
    if (flags < 0)
        return false;
    if (fcntl(sock, F_SETFL, (flags | O_NONBLOCK)) == -1)
        return false;
    return true;
#else
    unsigned long ioctlsocket_ret = 1;

	/* Returns non-0 on failure, while fcntl() returns -1 on failure */
	return (ioctlsocket(sock, FIONBIO, &ioctlsocket_ret) == 0);
#endif
}


void setupUDPListeningSocket(int *listenerSocketFd, uint16_t *listenerPort, int *txFamily, struct sockaddr_storage *listenerSockaddr)
{
    struct addrinfo 		*addrs = NULL;
    struct addrinfo 		*addr;
    struct addrinfo 		hints;
    int						ret;
    int 					ic_socket = PGINVALID_SOCKET;
    struct sockaddr_storage ic_socket_addr;
    int 					tries = 0;
    struct sockaddr_storage listenerAddr;
    socklen_t 				listenerAddrlen = sizeof(ic_socket_addr);
    uint32					socketSendBufferSize;
    uint32					socketRecvBufferSize;

    memset(&hints, 0, sizeof(struct addrinfo));
    hints.ai_family = AF_UNSPEC;	/* Allow IPv4 or IPv6 */
    hints.ai_socktype = SOCK_DGRAM; /* Datagram socket */
    hints.ai_protocol = 0;
    hints.ai_addrlen = 0;
    hints.ai_addr = NULL;
    hints.ai_canonname = NULL;
    hints.ai_next = NULL;
    hints.ai_flags |= AI_NUMERICHOST;

#ifdef USE_ASSERT_CHECKING
    if (gp_udpic_network_disable_ipv6)
		hints.ai_family = AF_INET;
#endif

    hints.ai_flags |= AI_NUMERICHOST;

    /*
     * Restrict what IP address we will listen on to just the one that was
     * used to create this QE session.
     */
    ret = pg_getaddrinfo_all(interconnect_address, NULL, &hints, &addrs);
    if (ret || !addrs)
    {
        printf("could not resolve address for UDP IC socket %s: %s \n", interconnect_address, ret);
        exit(1);
    }

    /*
     * On some platforms, pg_getaddrinfo_all() may return multiple addresses
     * only one of which will actually work (eg, both IPv6 and IPv4 addresses
     * when kernel will reject IPv6).  Worse, the failure may occur at the
     * bind() or perhaps even connect() stage.  So we must loop through the
     * results till we find a working combination. We will generate DEBUG
     * messages, but no error, for bogus combinations.
     */
    for (addr = addrs; addr != NULL; addr = addr->ai_next)
    {

#ifdef HAVE_UNIX_SOCKETS
        /* Ignore AF_UNIX sockets, if any are returned. */
		if (addr->ai_family == AF_UNIX)
			continue;
#endif

        ic_socket = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
        if (ic_socket == PGINVALID_SOCKET)
        {
            continue;
        }

        /*
         * Bind the socket to a kernel assigned ephemeral port on the
         * interconnect_address.
         */
        if (bind(ic_socket, addr->ai_addr, addr->ai_addrlen) < 0)
        {
            closesocket(ic_socket);
            ic_socket = PGINVALID_SOCKET;
            continue;
        }

        /* Call getsockname() to eventually obtain the assigned ephemeral port */
        if (getsockname(ic_socket, (struct sockaddr *) &listenerAddr, &listenerAddrlen) < 0)
        {
            closesocket(ic_socket);
            ic_socket = PGINVALID_SOCKET;
            continue;
        }

        /* If we get here, we have a working socket */
        break;
    }

    if (!addr || ic_socket == PGINVALID_SOCKET)
    {
        printf("can get an valid ic socket \n");
        exit(1);
    }


    /* Memorize the socket fd, kernel assigned port and address family */
    *listenerSocketFd = ic_socket;
    if (listenerAddr.ss_family == AF_INET6)
    {
        *listenerPort = ntohs(((struct sockaddr_in6 *) &listenerAddr)->sin6_port);
        *txFamily = AF_INET6;
    }
    else
    {
        *listenerPort = ntohs(((struct sockaddr_in *) &listenerAddr)->sin_port);
        *txFamily = AF_INET;
    }

    /*
     * cache the successful sockaddr of the listening socket, so
     * we can use this information to connect to the listening socket.
     */
    if (listenerSockaddr != NULL)
        memcpy(listenerSockaddr, &listenerAddr, sizeof(struct sockaddr_storage));

    /* Set up socket non-blocking mode */
    if (!pg_set_noblock(ic_socket))
    {
        printf("could not set UDP interconnect socket to nonblocking mode \n");
        exit(1);
    }
}

int main() {
    static int	ICSenderSocket = -1;
    static uint16_t ICSenderPort = 0;
    static int	ICSenderFamily = 0;
    int					ret;
    char				*dummy_pkt = "stop it";
    struct sockaddr_storage dest;
    socklen_t	dest_len;

    // do it 20 times
    for (int i = 0; i < 20; i++) {

        setupUDPListeningSocket(&ICSenderSocket, &ICSenderPort, &ICSenderFamily, &udp_dummy_packet_sockaddr);

        dest = udp_dummy_packet_sockaddr;
        dest_len = (ICSenderFamily == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);

        // do it 100 times
        for (int j = 0; j < 100; j++) {
            ret = sendto(ICSenderSocket, dummy_pkt, strlen(dummy_pkt), 0, (struct sockaddr *) &dest, dest_len);
            if (ret < 0) {
                if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)
                    exit(0);
                else {
                    printf("send dummy packet failed, sendto failed \n");
                    exit(1);
                }
            } else
                printf("send dummy packet success! \n");
        }
    }
}

I wrote another test script to test local network,Its logic is basically the same as greenplum, you can compile and test it on your environment, for example:

gcc test_network.c -o test_network
./test_network >> test_log 2>&1
grep "send dummy packet failed" test_log

If “send dummy packet failed” appears during the test, please check the machine’s network settings.