summaryrefslogtreecommitdiff
path: root/open-vm-tools/lib/sslDirect/sslDirect.c
diff options
context:
space:
mode:
Diffstat (limited to 'open-vm-tools/lib/sslDirect/sslDirect.c')
-rw-r--r--open-vm-tools/lib/sslDirect/sslDirect.c791
1 files changed, 791 insertions, 0 deletions
diff --git a/open-vm-tools/lib/sslDirect/sslDirect.c b/open-vm-tools/lib/sslDirect/sslDirect.c
new file mode 100644
index 00000000..a86722f8
--- /dev/null
+++ b/open-vm-tools/lib/sslDirect/sslDirect.c
@@ -0,0 +1,791 @@
+/*********************************************************
+ * Copyright (C) 2014-2015 VMware, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation version 2.1 and no later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the Lesser GNU General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ *********************************************************/
+
+/*
+ * sslDirect.c --
+ *
+ * Mostly direct call stubs for AsyncSocket SSL functions.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <stdarg.h>
+
+#include "str.h"
+
+#ifdef _WIN32
+#include <winsock2.h>
+#else
+#include <sys/socket.h>
+#include <fcntl.h>
+#include <unistd.h>
+#endif
+
+#include "log.h"
+#include "debug.h"
+#include "err.h"
+#include "msg.h"
+#include "sslDirect.h"
+#include "vm_assert.h"
+
+#define LOGLEVEL_MODULE SSLDirect
+#include "loglevel_user.h"
+
+#include <openssl/err.h>
+#include <openssl/ssl.h>
+#include <openssl/rand.h>
+#include <openssl/engine.h>
+
+#define SSL_LOG(x) Debug x
+
+struct SSLSockStruct {
+ SSL *sslCnx;
+ int fd;
+ Bool encrypted;
+ Bool closeFdOnShutdown;
+ Bool connectionFailed;
+#ifdef __APPLE__
+ Bool loggedKernelReadBug;
+#endif
+
+ int sslIOError;
+};
+
+
+static Bool SSLModuleInitialized = FALSE;
+
+#ifndef SOCKET_ERROR
+#define SOCKET_ERROR (-1)
+#endif
+
+enum {
+ SSL_SOCK_WANT_RETRY,
+ SSL_SOCK_LOST_CONNECTION,
+};
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SSLPrintErrors
+ *
+ * Print out all the errors in the SSL error queue.
+ *
+ * Results:
+ * None
+ *
+ * Side effects:
+ * Clears out the SSL error stack
+ *
+ *----------------------------------------------------------------------
+ */
+
+static void
+SSLPrintErrors(uint32 logLevel) // IN:
+{
+ /*
+ * Code inspection of the unsafe ERR_error_string function
+ * implementation in crypto/err.c shows a static buffer of 256
+ * characters. Presumably most messages will fit in that buffer.
+ */
+ enum { SSL_ERR_MAX_STRING = 256 };
+
+ int errNum;
+ char errString[SSL_ERR_MAX_STRING];
+ while ((errNum = ERR_get_error())) {
+ errString[0] = '\0';
+ ERR_error_string_n(errNum, errString, ARRAYSIZE(errString));
+ /* TODO: use LogV */
+ if (logLevel == VMW_LOG_WARNING) {
+ Warning("SSL Error: %s\n", errString);
+ } else {
+ Log("SSL Error: %s\n", errString);
+ }
+ }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SSLSetSystemError
+ *
+ * Maps the ssl error state into an appropriate errno / WSA error.
+ *
+ * Results:
+ * None
+ *
+ * Side effects:
+ * None
+ *----------------------------------------------------------------------
+ */
+
+static void
+SSLSetSystemError(int err)
+{
+ switch (err) {
+ case SSL_SOCK_WANT_RETRY:
+#ifdef _WIN32
+ WSASetLastError(WSAEWOULDBLOCK);
+#else
+ errno = EAGAIN;
+#endif
+ break;
+ case SSL_SOCK_LOST_CONNECTION:
+ /*
+ * no good way to know what the real error was (could have been
+ * a failure to load certificates in an accept), so return
+ * something generic.
+ */
+#ifdef _WIN32
+ WSASetLastError(WSAEACCES);
+#else
+ errno = EPERM;
+#endif
+ break;
+ default:
+ NOT_REACHED();
+ }
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SSLSetErrorState
+ *
+ * Each ssl read / write could result in several reads and writes on
+ * the underlying socket. In this case the actual value for errno
+ * will not be correct. Manually setup the error value so that
+ * clients will do the right thing.
+ *
+ * XXX: Mapping the SSL_ERROR_WANT_<something> errors to a single error code
+ * is not good. Applications using non-blocking IO would not know whether
+ * they should put the FD in a read wait or a write wait. Note that SSL_read
+ * can return SSL_ERROR_WANT_WRITE and SSL_write may return
+ * SSL_ERROR_WANT_READ.
+ *
+ * Results:
+ * None
+ *
+ * Side effects:
+ * errno / windows error might be set.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static int
+SSLSetErrorState(SSL *ssl,
+ int result)
+{
+ int sslError = SSL_get_error(ssl, result);
+ switch (sslError) {
+ case SSL_ERROR_NONE:
+ SSL_LOG(("SSL: action success, %d bytes\n", result));
+ break;
+ case SSL_ERROR_ZERO_RETURN:
+ SSL_LOG(("SSL: Zero return\n"));
+ break;
+ case SSL_ERROR_WANT_READ:
+ SSL_LOG(("SSL: Want read\n"));
+ SSLSetSystemError(SSL_SOCK_WANT_RETRY);
+ break;
+ case SSL_ERROR_WANT_WRITE:
+ SSL_LOG(("SSL: Want write\n"));
+ SSLSetSystemError(SSL_SOCK_WANT_RETRY);
+ break;
+ case SSL_ERROR_WANT_X509_LOOKUP:
+ SSL_LOG(("SSL: want x509 lookup\n"));
+ break;
+ case SSL_ERROR_SYSCALL:
+ SSL_LOG(("SSL: syscall error\n"));
+ SSLPrintErrors(VMW_LOG_INFO);
+ if (result == 0) {
+ Log("SSL: EOF in violation of protocol\n");
+ } else {
+ Log("SSL: syscall error %d: %s\n", Err_Errno(), Err_ErrString());
+ }
+ break;
+ case SSL_ERROR_SSL:
+ Warning("SSL: Unknown SSL Error\n");
+ SSLPrintErrors(VMW_LOG_INFO);
+ break;
+ }
+ return sslError;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SSL_Init --
+ *
+ * Initializes the SSL library and prepares the session context.
+ *
+ * Results:
+ * None.
+ *
+ * Side effects:
+ * Lots.
+ *----------------------------------------------------------------------
+ */
+
+void
+SSL_Init(SSLLibFn *getLibFn, // Ignored
+ const char *defaultLib, // Ignored
+ const char *name) // Ignored
+{
+ /*
+ * Silently ignore any attempts to initialize module more than once.
+ */
+ if (SSLModuleInitialized) {
+ return;
+ }
+
+ SSL_library_init();
+ SSL_load_error_strings();
+
+ OpenSSL_add_all_algorithms();
+
+ /*
+ * Force the PRNG to be initialized early, as opposed to at the
+ * time when the SSL connection is made. A call to RAND_status
+ * forces this initialization to happen. Initializing the PRNG
+ * as early as possible in the process makes it take much less
+ * time (e.g. 1sec. vs. sometimes 20sec.) compared to
+ * initializing it later in the process, as may be the case on
+ * the first SSL_accept() or SSL_connect(). That's because the
+ * PRNG initialization walks the process heap and the total heap
+ * is smaller at startup.
+ *
+ * If SSL_InitEx could not be called early enough in the
+ * process, then the caller could just call RAND_status() by
+ * itself. Only the first call to RAND_status will have the side
+ * effect of initializing the PRNG, so calling it subsequently
+ * would be a NOOP.
+ */
+ RAND_status();
+
+ ENGINE_register_all_ciphers();
+ ENGINE_register_all_digests();
+
+ SSLModuleInitialized = TRUE;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SSL_New()
+ *
+ * Results:
+ * Returns a freshly allocated SSLSock structure.
+ *
+ * Side effects:
+ * None
+ *
+ *----------------------------------------------------------------------
+ */
+
+SSLSock
+SSL_New(int fd, // IN
+ Bool closeFdOnShutdown) // IN
+{
+ SSLSock sslConnection;
+
+ sslConnection = (SSLSock)calloc(1, sizeof(struct SSLSockStruct));
+ VERIFY(sslConnection);
+ sslConnection->fd = fd;
+ sslConnection->closeFdOnShutdown = closeFdOnShutdown;
+
+ return sslConnection;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SSL_SetupAcceptWithContext()
+ *
+ * Setting up the ssl connection and states to do a SSL accept operation
+ *
+ * Results:
+ * Returns TRUE on success, FALSE on failure.
+ *
+ * Side effects:
+ * The server's certificate & private key may be loaded from disk.
+ *
+ *----------------------------------------------------------------------
+ */
+
+Bool SSL_SetupAcceptWithContext(SSLSock sSock, // IN: SSL socket
+ void *ctx) // IN: OpenSSL context (SSL_CTX *)
+{
+ Bool ret = TRUE;
+
+ ASSERT(SSLModuleInitialized);
+ ASSERT(sSock);
+ ASSERT(ctx);
+
+ sSock->sslCnx = SSL_new(ctx);
+ if (!sSock->sslCnx) {
+ SSLPrintErrors(VMW_LOG_WARNING);
+ Warning("Error Creating SSL connection structure\n");
+ sSock->connectionFailed = TRUE;
+ ret = FALSE;
+ goto end;
+ }
+ SSL_set_accept_state(sSock->sslCnx);
+
+ SSL_LOG(("SSL: ssl created\n"));
+ if (!SSL_set_fd(sSock->sslCnx, sSock->fd)) {
+ SSLPrintErrors(VMW_LOG_WARNING);
+ Warning("Error setting fd for SSL connection\n");
+ sSock->connectionFailed = TRUE;
+ ret = FALSE;
+ goto end;
+ }
+ SSL_LOG(("SSL: fd set done\n"));
+
+ sSock->encrypted = TRUE;
+
+end:
+ return ret;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SSL_Read --
+ *
+ * Functional equivalent of the read() syscall.
+ *
+ * Results:
+ * Returns the number of bytes read, or -1 on error. The
+ * data read will be placed in buf.
+ *
+ * Side effects:
+ *
+ *
+ *----------------------------------------------------------------------
+ */
+
+ssize_t
+SSL_Read(SSLSock ssl, // IN
+ char *buf, // OUT
+ size_t num) // IN
+{
+ int ret;
+ ASSERT(ssl);
+
+ if (ssl->connectionFailed) {
+ SSLSetSystemError(SSL_SOCK_LOST_CONNECTION);
+ ret = SOCKET_ERROR;
+ goto end;
+ }
+
+ if (ssl->encrypted) {
+ int result = SSL_read(ssl->sslCnx, buf, (int)num);
+
+ ssl->sslIOError = SSLSetErrorState(ssl->sslCnx, result);
+ if (ssl->sslIOError != SSL_ERROR_NONE) {
+ SSL_LOG(("SSL: Read(%d, %p, %"FMTSZ"u): %d\n",
+ ssl->fd, buf, num, result));
+ result = SOCKET_ERROR;
+ }
+ ret = result;
+ } else {
+ ret = SSLGeneric_read(ssl->fd, buf, (int)num);
+
+#ifdef __APPLE__
+ /*
+ * Detect bug 161237 (Apple bug 5202831), which should no longer be
+ * happening due to a workaround in our code.
+ *
+ * There is a bug on Mac OS 10.4 and 10.5 where passing an fd
+ * over a socket can result in that fd being in an inconsistent state.
+ * We can detect when this happens when read(2) returns zero
+ * even if the other end of the socket is not disconnected.
+ * We verify this by calling write(ssl->fd, "", 0) and
+ * see if it is okay. (If the socket was really closed, it would
+ * return -1 with errno==EPIPE.)
+ */
+ if (ret == 0) {
+ ssize_t writeRet;
+#ifdef VMX86_DEBUG
+ struct stat statBuffer;
+
+ /*
+ * Make sure we're using a socket.
+ */
+ ASSERT((fstat(ssl->fd, &statBuffer) == 0) &&
+ ((statBuffer.st_mode & S_IFSOCK) == S_IFSOCK));
+
+#endif
+ writeRet = write(ssl->fd, "", 0);
+ if (writeRet == 0) {
+ /*
+ * The socket is still good. read(2) should not have returned zero.
+ */
+ if (! ssl->loggedKernelReadBug) {
+ Log("Error: Encountered Apple bug #5202831. Disconnecting.\n");
+ ssl->loggedKernelReadBug = TRUE;
+ }
+ }
+ }
+#endif
+ }
+
+ end:
+ return ret;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SSL_RecvDataAndFd --
+ *
+ * recvmsg wrapper which can receive only file descriptors, not other
+ * control data.
+ *
+ * Results:
+ * Returns the number of bytes received, or -1 on error. The
+ * data read will be placed in buf. *fd is either -1 if no fd was
+ * received, or descriptor...
+ *
+ * Side effects:
+ *
+ *
+ *----------------------------------------------------------------------
+ */
+
+ssize_t
+SSL_RecvDataAndFd(SSLSock ssl, // IN/OUT: socket
+ char *buf, // OUT: buffer
+ size_t num, // IN: length of buffer
+ int *fd) // OUT: descriptor received
+{
+ int ret;
+ ASSERT(ssl);
+ ASSERT(fd);
+
+ *fd = -1;
+ if (ssl->connectionFailed) {
+ SSLSetSystemError(SSL_SOCK_LOST_CONNECTION);
+ ret = SOCKET_ERROR;
+ goto end;
+ }
+
+ /*
+ * No fd passing over SSL or Windows. Windows needs different code.
+ */
+#ifdef _WIN32
+ return SSL_Read(ssl, buf, num);
+#else
+ if (ssl->encrypted) {
+ int result = SSL_read(ssl->sslCnx, buf, (int)num);
+
+ ssl->sslIOError = SSLSetErrorState(ssl->sslCnx, result);
+ if (ssl->sslIOError != SSL_ERROR_NONE) {
+ SSL_LOG(("SSL: Read(%d, %p, %"FMTSZ"u): %d\n",
+ ssl->fd, buf, num, result));
+ result = SOCKET_ERROR;
+ }
+ ret = result;
+ } else {
+ struct iovec iov;
+ struct msghdr msg = { 0 };
+ uint8 cmsgBuf[CMSG_SPACE(sizeof(int))];
+
+ iov.iov_base = buf;
+ iov.iov_len = num;
+ msg.msg_name = NULL;
+ msg.msg_namelen = 0;
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+ msg.msg_control = cmsgBuf;
+ msg.msg_controllen = sizeof cmsgBuf;
+ ret = SSLGeneric_recvmsg(ssl->fd, &msg, 0);
+ if (ret >= 0 && msg.msg_controllen != 0) {
+ struct cmsghdr *cmsg;
+
+ for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
+ if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
+ int receivedFd = *(int *)CMSG_DATA(cmsg);
+
+ ASSERT(*fd == -1);
+ *fd = receivedFd;
+ }
+ }
+ }
+ }
+#endif
+
+ end:
+ return ret;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SSL_Write()
+ *
+ * Functional equivalent of the write() syscall.
+ *
+ * Results:
+ * Returns the number of bytes written, or -1 on error.
+ *
+ * Side effects:
+ *
+ *----------------------------------------------------------------------
+ */
+
+ssize_t
+SSL_Write(SSLSock ssl, // IN
+ const char *buf, // IN
+ size_t num) // IN
+{
+ int ret;
+ ASSERT(ssl);
+
+ if (ssl->connectionFailed) {
+ SSLSetSystemError(SSL_SOCK_LOST_CONNECTION);
+ ret = SOCKET_ERROR;
+ goto end;
+ }
+ if (ssl->encrypted) {
+ int result = SSL_write(ssl->sslCnx, buf, (int)num);
+
+ ssl->sslIOError = SSLSetErrorState(ssl->sslCnx, result);
+ if (ssl->sslIOError != SSL_ERROR_NONE) {
+ SSL_LOG(("SSL: Write(%d)\n", ssl->fd));
+ result = SOCKET_ERROR;
+ }
+ ret = result;
+ } else {
+ ret = SSLGeneric_write(ssl->fd, buf, (int)num);
+ }
+
+ end:
+ return ret;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SSL_Pending()
+ *
+ * Functional equivalent of select when SSL is enabled
+ *
+ * Results:
+ * Obtain number of readable bytes buffered in an SSL object if SSL
+ * is enabled, otherwise, return 0
+ *
+ * Side effects:
+ *
+ *----------------------------------------------------------------------
+ */
+
+int
+SSL_Pending(SSLSock ssl) // IN
+{
+ int ret;
+ ASSERT(ssl);
+
+ if (ssl->encrypted) {
+ ret = SSL_pending(ssl->sslCnx);
+ } else {
+ ret = 0;
+ }
+
+ return ret;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SSL_SetCloseOnShutdownFlag()
+ *
+ * Sets closeFdOnShutdown flag.
+ *
+ * Results:
+ * None. Always succeeds. Do not call close/closesocket on
+ * the fd after this, call SSL_Shutdown() instead.
+ *
+ * Side effects:
+ *
+ *----------------------------------------------------------------------
+ */
+
+void
+SSL_SetCloseOnShutdownFlag(SSLSock ssl) // IN
+{
+ ASSERT(ssl);
+ ssl->closeFdOnShutdown = TRUE;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SSL_Shutdown()
+ *
+ * Functional equivalent of the close() syscall. Does
+ * not close the actual fd used for the connection.
+ *
+ *
+ * Results:
+ * 0 on success, -1 on failure.
+ *
+ * Side effects:
+ * closes the connection, freeing up the memory associated
+ * with the passed in socket object
+ *
+ *----------------------------------------------------------------------
+ */
+
+int
+SSL_Shutdown(SSLSock ssl) // IN
+{
+ int retVal = 0;
+ ASSERT(ssl);
+
+ SSL_LOG(("SSL: Starting shutdown for %d\n", ssl->fd));
+ if (ssl->encrypted) {
+ /* since quiet_shutdown is set, SSL_shutdown always succeeds */
+ SSL_shutdown(ssl->sslCnx);
+ }
+ if (ssl->sslCnx) {
+ SSL_free(ssl->sslCnx);
+ }
+
+ if (ssl->closeFdOnShutdown) {
+ SSL_LOG(("SSL: Trying to close %d\n", ssl->fd));
+ /*
+ * Apparently in the past coverity has complained about the lack
+ * of shutdown() before close() here. However we only want to
+ * shut down the SSL layer, not the socket layer since authd may
+ * handoff the fd to another process.
+ */
+ retVal = SSLGeneric_close(ssl->fd);
+ }
+
+ free(ssl);
+ SSL_LOG(("SSL: shutdown done\n"));
+
+ return retVal;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SSL_GetFd()
+ *
+ * Returns an SSL socket's file descriptor or handle.
+ *
+ * Side effects:
+ * None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+int
+SSL_GetFd(SSLSock ssl) // IN
+{
+ ASSERT(ssl);
+
+ return ssl->fd;
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SSL_WantRead()
+ *
+ * Wrapper around SSL_want_read.
+ *
+ * Results:
+ * That of SSL_want_read.
+ *
+ * Side effects:
+ * None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+int
+SSL_WantRead(const SSLSock ssl)
+{
+ ASSERT(ssl);
+ ASSERT(ssl->sslCnx);
+
+ return SSL_want_read(ssl->sslCnx);
+}
+
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * SSL_TryCompleteAccept()
+ *
+ * Call SSL_Accept() to start or redrive the SSL accept operation.
+ * Nonblocking.
+ *
+ * Results:
+ * > 0 if the SSL_accept completed successfully
+ * = 0 if the SSL_accept need a redrive
+ * < 0 if an error occurred
+ *
+ * Side effects:
+ * None.
+ *
+ *----------------------------------------------------------------------
+ */
+
+int
+SSL_TryCompleteAccept(SSLSock ssl) // IN
+{
+ int sslRet;
+
+ ASSERT(ssl);
+ ASSERT(ssl->sslCnx);
+
+ ERR_clear_error();
+ sslRet = SSL_accept(ssl->sslCnx);
+ ssl->sslIOError = SSL_get_error(ssl->sslCnx, sslRet);
+
+ switch (ssl->sslIOError) {
+ case SSL_ERROR_NONE:
+ return 1;
+ case SSL_ERROR_WANT_READ:
+ case SSL_ERROR_WANT_WRITE:
+ return 0;
+ default:
+ ssl->connectionFailed = TRUE;
+ SSLPrintErrors(VMW_LOG_INFO);
+ return -1;
+ }
+}
+
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-23 23:40:58 +0000