diff --git a/api/server.go b/api/server.go
index 4af3734e58..9b36cbebac 100644
--- a/api/server.go
+++ b/api/server.go
@@ -18,6 +18,12 @@ func newUnixListener(addr string, tlsConfig *tls.Config) (net.Listener, error) {
 	if err := syscall.Unlink(addr); err != nil && !os.IsNotExist(err) {
 		return nil, err
 	}
+
+	// there is no way to specify the unix rights to use when
+	// creating the socket with net.Listener, so we use umask
+	// to create the file without rights and then we chmod
+	// to the desired unix rights. This prevent unwanted
+	// connections between the creation and the chmod
 	mask := syscall.Umask(0777)
 	defer syscall.Umask(mask)
 
@@ -26,7 +32,8 @@ func newUnixListener(addr string, tlsConfig *tls.Config) (net.Listener, error) {
 		return nil, err
 	}
 
-	if err := os.Chmod(addr, 0660); err != nil {
+	// only usable by the user who started swarm
+	if err := os.Chmod(addr, 0600); err != nil {
 		return nil, err
 	}