Fix IPv6 on Windows by listening on serveral sockets.

The server networklayer listens on several sockets for
available networks and IP versions. IPv4 connections
are still supported.

The OPC Foundation ANSI C Stack before 2016 does not fully
support IPv6. On Windows, the 'localhost' target is resolved
to IPv6 by default. Old applications (e.g. the Conformance
Testing Tools) need to connect to 127.0.0.1 instead of
'localhost' to force IPv4.

CHANGELOG

@@ -1,6 +1,18 @@
 The changelog tracks changes to the public API.
 Internal refactorings and bug fixes are not reported here.
 
+2017-06-26 janitza-thbe
+
+    * Enable IPv6 in the networklayer plugin
+
+      The server networklayer listens on several sockets for available
+      networks and IP versions. IPv4 connections are still supported.
+
+      The OPC Foundation ANSI C Stack before 2016 does not fully support IPv6.
+      On Windows, the 'localhost' target is resolved to IPv6 by default. Old
+      applications (e.g. the Conformance Testing Tools) need to connect to
+      127.0.0.1 instead of 'localhost' to force IPv4.
+
 2017-06-16 jpfr <julius.pfrommer at web.de>
 
     * Require the AccessLevel bit UA_ACCESSLEVELMASK_READ for reading

include/ua_plugin_network.h

@@ -101,9 +101,9 @@ struct UA_Connection {
     /* Release the buffer of a received message */
     void (*releaseRecvBuffer)(UA_Connection *connection, UA_ByteString *buf);
 
-    /* Close the connection. The network layer closes the socket, removes
-     * internal pointers to the connection and calls
-     * UA_Server_removeConnection. */
+    /* Close the connection. The network layer closes the socket. This is picked
+     * up during the next 'listen' and the connection is freed in the network
+     * layer. */
     void (*close)(UA_Connection *connection);
 
     /* To be called only from within the server (and not the network layer).
@@ -157,8 +157,9 @@ struct UA_ServerNetworkLayer {
     UA_StatusCode (*start)(UA_ServerNetworkLayer *nl);
 
     /* Listen for new and closed connections and arriving packets. Calls
-     * UA_Server_processBinaryMessage for the arriving packets. Calls
-     * UA_Server_removeConnection for closed connections.
+     * UA_Server_processBinaryMessage for the arriving packets. Closed
+     * connections are picked up here and forwarded to
+     * UA_Server_removeConnection where they are cleaned up and freed.
      *
      * @param nl The network layer
      * @param server The server for processing the incoming packets and for

plugins/ua_network_tcp.c

@@ -19,15 +19,8 @@
 #include <stdio.h> // snprintf
 #include <string.h> // memset
 #include <errno.h>
-#if UNDER_CE
-#define errno WSAGetLastError()
-#endif
+
 #ifdef _WIN32
-# ifndef __clang__
-#  include <malloc.h>
-# endif
-/* inet_ntoa is deprecated on MSVC but used for compatibility */
-# define _WINSOCK_DEPRECATED_NO_WARNINGS
 # include <winsock2.h>
 # include <ws2tcpip.h>
 # define CLOSESOCKET(S) closesocket((SOCKET)S)
@@ -71,8 +64,8 @@
 # define UA_fd_isset(fd, fds) FD_ISSET(fd, fds)
 #endif
 
-#ifdef UA_ENABLE_MULTITHREADING
-# include <urcu/uatomic.h>
+#if UNDER_CE
+# define errno WSAGetLastError()
 #endif
 
 #ifdef _WIN32
@@ -91,40 +84,70 @@
 /* Generic Socket Functions */
 /****************************/
 
+/* This performs only 'shutdown'. 'close' is called after the next recv on the
+ * socket. */
 static void
-socket_close(UA_Connection *connection) {
+connection_close(UA_Connection *connection) {
+    shutdown((SOCKET)connection->sockfd, 2);
     connection->state = UA_CONNECTION_CLOSED;
-    shutdown((SOCKET)connection->sockfd,2);
-    CLOSESOCKET(connection->sockfd);
 }
 
 static UA_StatusCode
-socket_write(UA_Connection *connection, UA_ByteString *buf) {
+connection_getsendbuffer(UA_Connection *connection,
+                         size_t length, UA_ByteString *buf) {
+    if(length > connection->remoteConf.recvBufferSize)
+        return UA_STATUSCODE_BADCOMMUNICATIONERROR;
+    return UA_ByteString_allocBuffer(buf, length);
+}
+
+static void
+connection_releasesendbuffer(UA_Connection *connection,
+                             UA_ByteString *buf) {
+    UA_ByteString_deleteMembers(buf);
+}
+
+static void
+connection_releaserecvbuffer(UA_Connection *connection,
+                             UA_ByteString *buf) {
+    UA_ByteString_deleteMembers(buf);
+}
+
+static UA_StatusCode
+connection_write(UA_Connection *connection, UA_ByteString *buf) {
+    /* Prevent OS signals when sending to a closed socket */
+    int flags = 0;
+#ifdef MSG_NOSIGNAL
+    flags |= MSG_NOSIGNAL;
+#endif
+
+    /* Send the full buffer. This may require several calls to send */
     size_t nWritten = 0;
     do {
         ssize_t n = 0;
         do {
-        /* If the OS throws EMSGSIZE, force a smaller packet size:
-         * size_t bytes_to_send = buf->length - nWritten >  1024 ? 1024 : buf->length - nWritten; */
             size_t bytes_to_send = buf->length - nWritten;
-            n = send((SOCKET)connection->sockfd, (const char*)buf->data + nWritten,
-                     WIN32_INT bytes_to_send, 0);
+            n = send((SOCKET)connection->sockfd,
+                     (const char*)buf->data + nWritten,
+                     WIN32_INT bytes_to_send, flags);
             if(n < 0 && errno__ != INTERRUPTED && errno__ != AGAIN) {
-                connection->close(connection);
-                socket_close(connection);
+                connection_close(connection);
                 UA_ByteString_deleteMembers(buf);
                 return UA_STATUSCODE_BADCONNECTIONCLOSED;
             }
         } while(n < 0);
         nWritten += (size_t)n;
     } while(nWritten < buf->length);
+
+    /* Free the buffer */
     UA_ByteString_deleteMembers(buf);
     return UA_STATUSCODE_GOOD;
 }
 
 static UA_StatusCode
-socket_recv(UA_Connection *connection, UA_ByteString *response, UA_UInt32 timeout) {
-    response->data = (UA_Byte *)malloc(connection->localConf.recvBufferSize);
+connection_recv(UA_Connection *connection, UA_ByteString *response,
+                UA_UInt32 timeout) {
+    response->data =
+        (UA_Byte*)malloc(connection->localConf.recvBufferSize);
     if(!response->data) {
         response->length = 0;
         return UA_STATUSCODE_BADOUTOFMEMORY; /* not enough memory retry */
@@ -148,7 +171,6 @@ socket_recv(UA_Connection *connection, UA_ByteString *response, UA_UInt32 timeou
 #endif
         if(0 != ret) {
             UA_ByteString_deleteMembers(response);
-            socket_close(connection);
             return UA_STATUSCODE_BADCONNECTIONCLOSED;
         }
     }
@@ -179,10 +201,9 @@ socket_recv(UA_Connection *connection, UA_ByteString *response, UA_UInt32 timeou
                        connection->localConf.recvBufferSize, 0);
 #endif
 
-    /* server has closed the connection */
+    /* The socket is shutdown */
     if(ret == 0) {
         UA_ByteString_deleteMembers(response);
-        socket_close(connection);
         return UA_STATUSCODE_BADCONNECTIONCLOSED;
     }
 
@@ -192,10 +213,10 @@ socket_recv(UA_Connection *connection, UA_ByteString *response, UA_UInt32 timeou
         if(errno__ == INTERRUPTED || (timeout > 0) ?
            false : (errno__ == EAGAIN || errno__ == WOULDBLOCK))
             return UA_STATUSCODE_GOOD; /* statuscode_good but no data -> retry */
-        socket_close(connection);
+        connection_close(connection);
         return UA_STATUSCODE_BADCONNECTIONCLOSED;
     }
-    
+
     /* default case */
     response->length = (size_t)ret;
     return UA_STATUSCODE_GOOD;
@@ -218,43 +239,8 @@ static UA_StatusCode socket_set_nonblocking(SOCKET sockfd) {
 /* Server NetworkLayer TCP */
 /***************************/
 
-/**
- * For the multithreaded mode, assume a single thread that periodically "gets
- * work" from the network layer. In addition, several worker threads are
- * asynchronously calling into the callbacks of the UA_Connection that holds a
- * single connection.
- *
- * Creating a connection: When "GetJobs" encounters a new connection, it creates
- * a UA_Connection with the socket information. This is added to the mappings
- * array that links sockets to UA_Connection structs.
- *
- * Reading data: In "GetJobs", we listen on the sockets in the mappings array.
- * If data arrives (or the connection closes), a WorkItem is created that
- * carries the work and a pointer to the connection.
- *
- * Closing a connection: Closing can happen in two ways. Either it is triggered
- * by the server in an asynchronous callback. Or the connection is close by the
- * client and this is detected in "GetJobs". The server needs to do some
- * internal cleanups (close attached securechannels, etc.). So even when a
- * closed connection is detected in "GetJobs", we trigger the server to close
- * the connection (with a WorkItem) and continue from the callback.
- *
- * - Server calls close-callback: We close the socket, set the connection-state
- *   to closed and add the connection to a linked list from which it is deleted
- *   later. The connection cannot be freed right away since other threads might
- *   still be using it.
- *
- * - GetJobs: We remove the connection from the mappings array. In the
- *   non-multithreaded case, the connection is freed. For multithreading, we
- *   return a workitem that is delayed, i.e. that is called only after all
- *   workitems created before are finished in all threads. This workitems
- *   contains a callback that goes through the linked list of connections to be
- *   freed. */
-
 #define MAXBACKLOG 100
 
-/* UA_Connection is the first element. We can exchange pointers to the first
- * element and free the entire structure. */
 typedef struct ConnectionEntry {
     UA_Connection connection;
     LIST_ENTRY(ConnectionEntry) pointers;
@@ -263,64 +249,31 @@ typedef struct ConnectionEntry {
 typedef struct {
     UA_ConnectionConfig conf;
     UA_UInt16 port;
-
-    /* open sockets and connections */
-    UA_Int32 serversockfd;
+    UA_Int32 serverSockets[FD_SETSIZE];
+    UA_UInt16 serverSocketsSize;
     LIST_HEAD(, ConnectionEntry) connections;
 } ServerNetworkLayerTCP;
 
-static UA_StatusCode
-ServerNetworkLayerGetSendBuffer(UA_Connection *connection,
-                                size_t length, UA_ByteString *buf) {
-    if(length > connection->remoteConf.recvBufferSize)
-        return UA_STATUSCODE_BADCOMMUNICATIONERROR;
-    return UA_ByteString_allocBuffer(buf, length);
-}
-
-static void
-ServerNetworkLayerReleaseSendBuffer(UA_Connection *connection,
-                                    UA_ByteString *buf) {
-    UA_ByteString_deleteMembers(buf);
-}
-
-static void
-ServerNetworkLayerReleaseRecvBuffer(UA_Connection *connection,
-                                    UA_ByteString *buf) {
-    UA_ByteString_deleteMembers(buf);
-}
-
 static void
 ServerNetworkLayerTCP_freeConnection(UA_Connection *connection) {
     UA_Connection_deleteMembers(connection);
     free(connection);
- }
-
-/* Callback triggered from the server (any thread). Only "shutdown" here. This
- * triggers the select, where the connection is closed and freed in the server's
- * mainloop. */
-static void
-ServerNetworkLayerTCP_closeConnection(UA_Connection *connection) {
-    connection->state = UA_CONNECTION_CLOSED;
-    UA_LOG_INFO(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
-                "Connection %i | Force closing the connection",
-                connection->sockfd);
-    shutdown(connection->sockfd, 2);
 }
 
 static UA_StatusCode
-ServerNetworkLayerTCP_add(ServerNetworkLayerTCP *layer, UA_Int32 newsockfd) {
-    /* Look up the peer name for logging */
-    struct sockaddr_in addr;
-    socklen_t addrlen = sizeof(struct sockaddr_in);
-    int res = getpeername(newsockfd, (struct sockaddr*)&addr, &addrlen);
-    if(res == 0) {
+ServerNetworkLayerTCP_add(ServerNetworkLayerTCP *layer, UA_Int32 newsockfd,
+                          struct sockaddr_storage *remote) {
+    /* Get the peer name for logging */
+    char remote_name[100];
+    if(getnameinfo((struct sockaddr*)remote, sizeof(struct sockaddr_storage), remote_name,
+                   sizeof(remote_name), NULL, 0, 0) == 0) {
         UA_LOG_INFO(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
-                    "Connection %i | New connection over TCP from %s:%d",
-                    newsockfd, inet_ntoa(addr.sin_addr), ntohs(addr.sin_port));
+                    "Connection %i | New connection over TCP from %s",
+                    newsockfd, remote_name);
     } else {
         UA_LOG_WARNING(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
                        "Connection %i | New connection over TCP, "
-                       "getpeername failed with errno %i", newsockfd, errno);
+                       "getnameinfo failed with errno %i", newsockfd, errno__);
     }
 
     /* Allocate and initialize the connection */
@@ -333,12 +286,12 @@ ServerNetworkLayerTCP_add(ServerNetworkLayerTCP *layer, UA_Int32 newsockfd) {
     c->handle = layer;
     c->localConf = layer->conf;
     c->remoteConf = layer->conf;
-    c->send = socket_write;
-    c->close = ServerNetworkLayerTCP_closeConnection;
+    c->send = connection_write;
+    c->close = connection_close;
     c->free = ServerNetworkLayerTCP_freeConnection;
-    c->getSendBuffer = ServerNetworkLayerGetSendBuffer;
-    c->releaseSendBuffer = ServerNetworkLayerReleaseSendBuffer;
-    c->releaseRecvBuffer = ServerNetworkLayerReleaseRecvBuffer;
+    c->getSendBuffer = connection_getsendbuffer;
+    c->releaseSendBuffer = connection_releasesendbuffer;
+    c->releaseRecvBuffer = connection_releaserecvbuffer;
     c->state = UA_CONNECTION_OPENING;
 
     /* Add to the linked list */
@@ -346,6 +299,76 @@ ServerNetworkLayerTCP_add(ServerNetworkLayerTCP *layer, UA_Int32 newsockfd) {
     return UA_STATUSCODE_GOOD;
 }
 
+static void
+addServerSockets(ServerNetworkLayerTCP *layer, struct addrinfo *ai) {
+    /* There might be serveral addrinfos (for different network cards,
+     * IPv4/IPv6). Add a server socket for all of them. */
+    for(layer->serverSocketsSize = 0;
+        layer->serverSocketsSize < FD_SETSIZE && ai != NULL;
+        ai = ai->ai_next) {
+
+        /* Create the server socket */
+        SOCKET newsock = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
+#ifdef _WIN32
+        if(newsock == INVALID_SOCKET)
+#else
+        if(newsock < 0)
+#endif
+        {
+            UA_LOG_WARNING(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
+                           "Error opening the server socket");
+            continue;
+        }
+
+        /* Some Linux distributions have net.ipv6.bindv6only not activated. So
+         * sockets can double-bind to IPv4 and IPv6. This leads to problems. Use
+         * AF_INET6 sockets only for IPv6. */
+        int optval = 1;
+        if(ai->ai_family == AF_INET6 &&
+           setsockopt(newsock, IPPROTO_IPV6, IPV6_V6ONLY,
+                      (const char*)&optval, sizeof(optval)) == -1) {
+            UA_LOG_WARNING(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
+                           "Could not set an IPv6 socket to IPv6 only");
+            CLOSESOCKET(newsock);
+            continue;
+        }
+
+        if(setsockopt(newsock, SOL_SOCKET, SO_REUSEADDR,
+                      (const char *)&optval, sizeof(optval)) == -1) {
+            UA_LOG_WARNING(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
+                           "Could not make the socket reusable");
+            CLOSESOCKET(newsock);
+            continue;
+        }
+
+        if(socket_set_nonblocking(newsock) != UA_STATUSCODE_GOOD) {
+            UA_LOG_WARNING(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
+                           "Could not set the server socket to nonblocking");
+            CLOSESOCKET(newsock);
+            continue;
+        }
+
+        /* Bind socket to address */
+        if(bind(newsock, ai->ai_addr, WIN32_INT ai->ai_addrlen) < 0) {
+            UA_LOG_WARNING(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
+                           "Error binding a server socket: %i", errno__);
+            CLOSESOCKET(newsock);
+            continue;
+        }
+
+        /* Start listening */
+        if(listen(newsock, MAXBACKLOG) < 0) {
+            UA_LOG_WARNING(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
+                           "Error listening on server socket");
+            CLOSESOCKET(newsock);
+            continue;
+        }
+
+        layer->serverSockets[layer->serverSocketsSize] = (UA_Int32)newsock;
+        layer->serverSocketsSize++;
+    }
+}
+
 static UA_StatusCode
 ServerNetworkLayerTCP_start(UA_ServerNetworkLayer *nl) {
     ServerNetworkLayerTCP *layer = (ServerNetworkLayerTCP *)nl->handle;
@@ -366,83 +389,47 @@ ServerNetworkLayerTCP_start(UA_ServerNetworkLayer *nl) {
     }
     UA_String_copy(&du, &nl->discoveryUrl);
 
-    /* Create the server socket */
-    SOCKET newsock = socket(AF_INET6, SOCK_STREAM, 0);
-#ifdef _WIN32
-    if(newsock == INVALID_SOCKET)
+    /* Get addrinfo of the server and create server sockets */
+    char portno[6];
+#ifndef _MSC_VER
+    snprintf(portno, 6, "%d", layer->port);
 #else
-    if(newsock < 0)
+    _snprintf_s(portno, 6, _TRUNCATE, "%d", layer->port);
 #endif
-    {
-        UA_LOG_WARNING(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
-                       "Error opening the server socket");
-        return UA_STATUSCODE_BADINTERNALERROR;
-    }
-
-    /* Set socket options */
-    int optval = 1;
-    if(setsockopt(newsock, SOL_SOCKET, SO_REUSEADDR,
-                  (const char *)&optval, sizeof(optval)) == -1 ||
-       socket_set_nonblocking(newsock) != UA_STATUSCODE_GOOD) {
-        UA_LOG_WARNING(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
-                       "Error during setting of server socket options");
-        CLOSESOCKET(newsock);
-        return UA_STATUSCODE_BADINTERNALERROR;
-    }
-
-    /* Bind socket to address */
-    struct sockaddr_in6 serv_addr;
-    memset(&serv_addr, 0, sizeof(serv_addr));
-    serv_addr.sin6_family = AF_INET6;
-    serv_addr.sin6_port = htons(layer->port);
-    serv_addr.sin6_addr = in6addr_any;
-    if(bind(newsock, (const struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) {
-        UA_LOG_WARNING(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
-                       "Error during binding of the server socket");
-        CLOSESOCKET(newsock);
-        return UA_STATUSCODE_BADINTERNALERROR;
-    }
 
-    /* Start listening */
-    if(listen(newsock, MAXBACKLOG) < 0) {
-        UA_LOG_WARNING(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
-                       "Error listening on server socket");
-        CLOSESOCKET(newsock);
-        return UA_STATUSCODE_BADINTERNALERROR;
-    }
+    struct addrinfo hints, *res;
+    memset(&hints, 0, sizeof hints);
+    hints.ai_family = AF_UNSPEC;
+    hints.ai_socktype = SOCK_STREAM;
+    hints.ai_flags = AI_PASSIVE;
+    getaddrinfo(NULL, portno, &hints, &res);
+    addServerSockets(layer, res);
+    freeaddrinfo(res);
 
-    layer->serversockfd = (UA_Int32)newsock; /* cast on win32 */
     UA_LOG_INFO(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
                 "TCP network layer listening on %.*s",
                 nl->discoveryUrl.length, nl->discoveryUrl.data);
     return UA_STATUSCODE_GOOD;
 }
 
-/* Remove closed connections before going into "listen". Returns whether a
- * connection was removed. */
-static void
-removeClosedConnections(ServerNetworkLayerTCP *layer, UA_Server *server) {
-    ConnectionEntry *e, *e_tmp;
-    LIST_FOREACH_SAFE(e, &layer->connections, pointers, e_tmp) {
-        if(e->connection.state != UA_CONNECTION_CLOSED)
-            continue;
-        LIST_REMOVE(e, pointers);
-        UA_Server_removeConnection(server, &e->connection);
-    }
-}
-
-/* After every select, we need to reset the sockets we want to listen on */
+/* After every select, reset the sockets to listen on */
 static UA_Int32
 setFDSet(ServerNetworkLayerTCP *layer, fd_set *fdset) {
     FD_ZERO(fdset);
-    UA_fd_set(layer->serversockfd, fdset);
-    UA_Int32 highestfd = layer->serversockfd;
+    UA_Int32 highestfd = 0;
+    for(UA_UInt16 i = 0; i < layer->serverSocketsSize; i++) {
+        UA_fd_set(layer->serverSockets[i], fdset);
+        if(layer->serverSockets[i] > highestfd)
+            highestfd = layer->serverSockets[i];
+    }
+
     ConnectionEntry *e;
     LIST_FOREACH(e, &layer->connections, pointers) {
         UA_fd_set(e->connection.sockfd, fdset);
         if(e->connection.sockfd > highestfd)
             highestfd = e->connection.sockfd;
     }
+
     return highestfd;
 }
 
@@ -452,47 +439,63 @@ ServerNetworkLayerTCP_listen(UA_ServerNetworkLayer *nl, UA_Server *server,
     /* Every open socket can generate two jobs */
     ServerNetworkLayerTCP *layer = (ServerNetworkLayerTCP *)nl->handle;
 
-    /* Remove closed sockets */
-    removeClosedConnections(layer, server);
-
     /* Listen on open sockets (including the server) */
     fd_set fdset, errset;
     UA_Int32 highestfd = setFDSet(layer, &fdset);
     setFDSet(layer, &errset);
     struct timeval tmptv = {0, timeout * 1000};
-    UA_Int32 resultsize = select(highestfd+1, &fdset, NULL, &errset, &tmptv);
+    select(highestfd+1, &fdset, NULL, &errset, &tmptv);
 
-    /* Accept new connection via the server socket (can only be a single one) */
-    if(UA_fd_isset(layer->serversockfd, &fdset)) {
-        --resultsize;
-        SOCKET newsockfd = accept((SOCKET)layer->serversockfd, NULL, NULL);
+    /* Accept new connections via the server sockets */
+    for(UA_UInt16 i = 0; i < layer->serverSocketsSize; i++) {
+        if(!UA_fd_isset(layer->serverSockets[i], &fdset))
+            continue;
+
+        struct sockaddr_storage remote;
+        socklen_t remote_size = sizeof(remote);
+        SOCKET newsockfd = accept((SOCKET)layer->serverSockets[i],
+                                  (struct sockaddr*)&remote, &remote_size);
 #ifdef _WIN32
-        if(newsockfd != INVALID_SOCKET)
+        if(newsockfd == INVALID_SOCKET)
 #else
-        if(newsockfd >= 0)
+        if(newsockfd < 0)
 #endif
-        {
-            socket_set_nonblocking(newsockfd);
-            /* Do not merge packets on the socket (disable Nagle's algorithm) */
-            int i = 1;
-            setsockopt(newsockfd, IPPROTO_TCP, TCP_NODELAY, (const char *)&i, sizeof(i));
-            ServerNetworkLayerTCP_add(layer, (UA_Int32)newsockfd);
-        }
+            continue;
+
+        socket_set_nonblocking(newsockfd);
+        /* Do not merge packets on the socket (disable Nagle's algorithm) */
+        int dummy = 1;
+        setsockopt(newsockfd, IPPROTO_TCP, TCP_NODELAY,
+                   (const char *)&dummy, sizeof(dummy));
+        ServerNetworkLayerTCP_add(layer, (UA_Int32)newsockfd, &remote);
     }
 
     /* Read from established sockets */
     ConnectionEntry *e, *e_tmp;
     LIST_FOREACH_SAFE(e, &layer->connections, pointers, e_tmp) {
-        UA_ByteString buf = UA_BYTESTRING_NULL;
         if(!UA_fd_isset(e->connection.sockfd, &errset) &&
            !UA_fd_isset(e->connection.sockfd, &fdset))
           continue;
 
-        UA_StatusCode retval = socket_recv(&e->connection, &buf, 0);
+        UA_ByteString buf = UA_BYTESTRING_NULL;
+        UA_StatusCode retval = connection_recv(&e->connection, &buf, 0);
+
         if(retval == UA_STATUSCODE_GOOD) {
+            /* Process packets */
             UA_Server_processBinaryMessage(server, &e->connection, &buf);
-        } else if (retval == UA_STATUSCODE_BADCONNECTIONCLOSED) {
+        } else if(retval == UA_STATUSCODE_BADCONNECTIONCLOSED) {
+            /* The socket is shutdown but not closed */
+            if(e->connection.state != UA_CONNECTION_CLOSED) {
+                UA_LOG_INFO(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
+                            "Connection %d was closed by the client",
+                            e->connection.sockfd);
+            } else {
+                UA_LOG_INFO(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
+                            "Connection %d was closed by the server",
+                            e->connection.sockfd);
+            }
             LIST_REMOVE(e, pointers);
+            CLOSESOCKET(e->connection.sockfd);
             UA_Server_removeConnection(server, &e->connection);
         }
     }
@@ -504,13 +507,23 @@ ServerNetworkLayerTCP_stop(UA_ServerNetworkLayer *nl, UA_Server *server) {
     ServerNetworkLayerTCP *layer = (ServerNetworkLayerTCP *)nl->handle;
     UA_LOG_INFO(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
                 "Shutting down the TCP network layer");
-    shutdown((SOCKET)layer->serversockfd,2);
-    CLOSESOCKET(layer->serversockfd);
-    ConnectionEntry *e, *e_tmp;
-    LIST_FOREACH_SAFE(e, &layer->connections, pointers, e_tmp) {
-        LIST_REMOVE(e, pointers);
-        UA_Server_removeConnection(server, &e->connection);
+
+    /* Close the server sockets */
+    for(UA_UInt16 i = 0; i < layer->serverSocketsSize; i++) {
+        shutdown((SOCKET)layer->serverSockets[i], 2);
+        CLOSESOCKET(layer->serverSockets[i]);
     }
+    layer->serverSocketsSize = 0;
+
+    /* Close open connections */
+    ConnectionEntry *e, *e_tmp;
+    LIST_FOREACH_SAFE(e, &layer->connections, pointers, e_tmp)
+        connection_close(&e->connection);
+
+    /* Run recv on client sockets. This picks up the closed sockets and frees
+     * the connection. */
+    ServerNetworkLayerTCP_listen(nl, server, 0);
+
 #ifdef _WIN32
     WSACleanup();
 #endif
@@ -535,10 +548,11 @@ UA_ServerNetworkLayerTCP(UA_ConnectionConfig conf, UA_UInt16 port) {
 
     UA_ServerNetworkLayer nl;
     memset(&nl, 0, sizeof(UA_ServerNetworkLayer));
-    ServerNetworkLayerTCP *layer = (ServerNetworkLayerTCP *)calloc(1,sizeof(ServerNetworkLayerTCP));
+    ServerNetworkLayerTCP *layer =
+        (ServerNetworkLayerTCP *)calloc(1,sizeof(ServerNetworkLayerTCP));
     if(!layer)
         return nl;
-    
+
     layer->conf = conf;
     layer->port = port;
 
@@ -554,35 +568,6 @@ UA_ServerNetworkLayerTCP(UA_ConnectionConfig conf, UA_UInt16 port) {
 /* Client NetworkLayer TCP */
 /***************************/
 
-static UA_StatusCode
-ClientNetworkLayerGetBuffer(UA_Connection *connection, size_t length,
-                            UA_ByteString *buf) {
-    if(length > connection->remoteConf.recvBufferSize)
-        return UA_STATUSCODE_BADCOMMUNICATIONERROR;
-    if(connection->state == UA_CONNECTION_CLOSED)
-        return UA_STATUSCODE_BADCONNECTIONCLOSED;
-    return UA_ByteString_allocBuffer(buf, connection->remoteConf.recvBufferSize);
-}
-
-static void
-ClientNetworkLayerReleaseBuffer(UA_Connection *connection, UA_ByteString *buf) {
-    UA_ByteString_deleteMembers(buf);
-}
-
-static void
-ClientNetworkLayerClose(UA_Connection *connection) {
-#ifdef UA_ENABLE_MULTITHREADING
-    if(uatomic_xchg(&connection->state, UA_CONNECTION_CLOSED) == UA_CONNECTION_CLOSED)
-        return;
-#else
-    if(connection->state == UA_CONNECTION_CLOSED)
-        return;
-    connection->state = UA_CONNECTION_CLOSED;
-#endif
-    socket_close(connection);
-}
-
-/* we have no networklayer. instead, attach the reusable buffer to the handle */
 UA_Connection
 UA_ClientConnectionTCP(UA_ConnectionConfig conf, const char *endpointUrl) {
 #ifdef _WIN32
@@ -597,13 +582,13 @@ UA_ClientConnectionTCP(UA_ConnectionConfig conf, const char *endpointUrl) {
     connection.state = UA_CONNECTION_OPENING;
     connection.localConf = conf;
     connection.remoteConf = conf;
-    connection.send = socket_write;
-    connection.recv = socket_recv;
-    connection.close = ClientNetworkLayerClose;
-    connection.free = NULL; /* Not used in the client */
-    connection.getSendBuffer = ClientNetworkLayerGetBuffer;
-    connection.releaseSendBuffer = ClientNetworkLayerReleaseBuffer;
-    connection.releaseRecvBuffer = ClientNetworkLayerReleaseBuffer;
+    connection.send = connection_write;
+    connection.recv = connection_recv;
+    connection.close = connection_close;
+    connection.free = NULL;
+    connection.getSendBuffer = connection_getsendbuffer;
+    connection.releaseSendBuffer = connection_releasesendbuffer;
+    connection.releaseRecvBuffer = connection_releaserecvbuffer;
 
     UA_String endpointUrlString = UA_STRING((char*)(uintptr_t)endpointUrl);
     UA_String hostnameString = UA_STRING_NULL;
@@ -618,18 +603,19 @@ UA_ClientConnectionTCP(UA_ConnectionConfig conf, const char *endpointUrl) {
                        "Server url is invalid: %s", endpointUrl);
         return connection;
     }
+    memcpy(hostname, hostnameString.data, hostnameString.length);
+    hostname[hostnameString.length] = 0;
+
     if(port == 0) {
         port = 4840;
         UA_LOG_INFO(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
-                    "No port defined, using standard port %d", port);
+                    "No port defined, using default port %d", port);
     }
-    memcpy(hostname, hostnameString.data, hostnameString.length);
-    hostname[hostnameString.length] = 0;
 
     struct addrinfo hints, *server;
     memset(&hints, 0, sizeof(hints));
+    hints.ai_family = AF_UNSPEC;
     hints.ai_socktype = SOCK_STREAM;
-    hints.ai_family = AF_INET;
     char portStr[6];
 #ifndef _MSC_VER
     snprintf(portStr, 6, "%d", port);
@@ -662,33 +648,19 @@ UA_ClientConnectionTCP(UA_ConnectionConfig conf, const char *endpointUrl) {
     connection.sockfd = (UA_Int32)clientsockfd; /* cast for win32 */
     error = connect(clientsockfd, server->ai_addr, WIN32_INT server->ai_addrlen);
     freeaddrinfo(server);
+
     if(error < 0) {
-        ClientNetworkLayerClose(&connection);
-#ifdef _WIN32
-        wchar_t *s = NULL;
-        FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER |
-                       FORMAT_MESSAGE_FROM_SYSTEM |
-                       FORMAT_MESSAGE_IGNORE_INSERTS,
-                       NULL, WSAGetLastError(),
-                       MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
-                       (LPWSTR)&s, 0, NULL);
-        UA_LOG_WARNING(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
-                       "Connection to %s failed. Error: %d: %S",
-                       endpointUrl, WSAGetLastError(), s);
-        LocalFree(s);
-#else
+        connection_close(&connection);
         UA_LOG_WARNING(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
-                       "Connection to %s failed. Error: %d: %s",
-                       endpointUrl, errno, strerror(errno));
-#endif
+                       "Connection to %s failed with error %d",
+                       endpointUrl, errno__);
         return connection;
     }
 
 #ifdef SO_NOSIGPIPE
     int val = 1;
-    int sso_result = setsockopt(connection.sockfd,
-                                SOL_SOCKET, SO_NOSIGPIPE,
-                                (void*)&val, sizeof(val));
+    int sso_result = setsockopt(connection.sockfd, SOL_SOCKET,
+                                SO_NOSIGPIPE, (void*)&val, sizeof(val));
     if(sso_result < 0)
         UA_LOG_WARNING(UA_Log_Stdout, UA_LOGCATEGORY_NETWORK,
                        "Couldn't set SO_NOSIGPIPE");

src/server/ua_server_binary.c

@@ -710,7 +710,6 @@ UA_Server_processBinaryMessage(UA_Server *server, UA_Connection *connection,
 
 #endif
 
-/* Remove a connection after it was closed in the network layer */
 #ifdef UA_ENABLE_MULTITHREADING
 static void
 deleteConnectionTrampoline(UA_Server *server, void *data) {