diff --git a/daemon/graphdriver/overlay2/overlay.go b/daemon/graphdriver/overlay2/overlay.go
index 603a8ff8ec..af0a536120 100644
--- a/daemon/graphdriver/overlay2/overlay.go
+++ b/daemon/graphdriver/overlay2/overlay.go
@@ -340,7 +340,15 @@ func (d *Driver) getLowerDirs(id string) ([]string, error) {
 
 // Remove cleans the directories that are created for this id.
 func (d *Driver) Remove(id string) error {
-	if err := os.RemoveAll(d.dir(id)); err != nil && !os.IsNotExist(err) {
+	dir := d.dir(id)
+	lid, err := ioutil.ReadFile(path.Join(dir, "link"))
+	if err == nil {
+		if err := os.RemoveAll(path.Join(d.home, linkDir, string(lid))); err != nil {
+			logrus.Debugf("Failed to remove link: %v", err)
+		}
+	}
+
+	if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) {
 		return err
 	}
 	return nil