diff --git a/gck-rpc-dispatch.c b/gck-rpc-dispatch.c
index dfdf33ec32674e9544562b79f7c6f9998124bc18..c303db6c45087dba117c57be3622b703855bedc0 100644
--- a/gck-rpc-dispatch.c
+++ b/gck-rpc-dispatch.c
@@ -2601,7 +2601,9 @@ void gck_rpc_layer_uninitialize(void)
 	pkcs11_socket = -1;
 
 	/* Delete our unix socket */
-	if (pkcs11_socket_path[0])
+	if (pkcs11_socket_path[0] &&
+	    strncmp(pkcs11_socket_path, "tcp://", strlen("tcp://")) != 0 &&
+	    strncmp(pkcs11_socket_path, "tls://", strlen("tls://")) != 0)
 		unlink(pkcs11_socket_path);
 	pkcs11_socket_path[0] = 0;