graph: use tar archive entries for TarLayer

if there is a tar-data.json.gz present for an image layer, then use it
to create the tar archive, instead of the traditional graphdriver Diff.

Signed-off-by: Vincent Batts <vbatts@redhat.com>

Conflicts:
	graph/graph.go

graph/graph.go

@@ -87,7 +87,7 @@ const (
 	jsonFileName      = "json"
 	layersizeFileName = "layersize"
 	digestFileName    = "checksum"
-	tardataFileName   = "tar-data.json.gz"
+	tarDataFileName   = "tar-data.json.gz"
 )
 
 var (

graph/graph_unix.go

@@ -6,14 +6,18 @@ import (
 	"compress/gzip"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
 	"syscall"
 
+	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/system"
+	"github.com/vbatts/tar-split/tar/asm"
+	"github.com/vbatts/tar-split/tar/storage"
 )
 
 // setupInitLayer populates a directory with mountpoints suitable
@@ -92,14 +96,14 @@ func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader
 	// Store the layer. If layerData is not nil, unpack it into the new layer
 	if layerData != nil {
 		// this is saving the tar-split metadata
-		mf, err := os.OpenFile(filepath.Join(root, tardataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
+		mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
 		if err != nil {
 			return err
 		}
 		defer mf.Close()
 		mfz := gzip.NewWriter(mf)
 		defer mfz.Close()
-		metaPacker := storage.NewJSONPacker(mf)
+		metaPacker := storage.NewJSONPacker(mfz)
 
 		inflatedLayerData, err := archive.DecompressStream(layerData)
 		if err != nil {
@@ -134,6 +138,49 @@ func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader
 
 // TarLayer returns a tar archive of the image's filesystem layer.
 func (graph *Graph) TarLayer(img *image.Image) (arch archive.Archive, err error) {
-	// TODO(vbatts) let's reassemble!
-	return graph.driver.Diff(img.ID, img.Parent)
+	root := graph.imageRoot(img.ID)
+	mFileName := filepath.Join(root, tarDataFileName)
+	mf, err := os.Open(mFileName)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			logrus.Errorf("failed to open %q: %s", mFileName, err)
+		}
+		logrus.Debugf("[graph] TarLayer with traditional differ: %s", img.ID)
+		return graph.driver.Diff(img.ID, img.Parent)
+	}
+	pR, pW := io.Pipe()
+	// this will need to be in a goroutine, as we are returning the stream of a
+	// tar archive, but can not close the metadata reader early (when this
+	// function returns)...
+	go func() {
+		defer mf.Close()
+		// let's reassemble!
+		logrus.Debugf("[graph] TarLayer with reassembly: %s", img.ID)
+		mfz, err := gzip.NewReader(mf)
+		if err != nil {
+			pW.CloseWithError(fmt.Errorf("[graph] error with %s:  %s", mFileName, err))
+			return
+		}
+		defer mfz.Close()
+
+		// get our relative path to the container
+		fsLayer, err := graph.driver.Get(img.ID, "")
+		if err != nil {
+			pW.CloseWithError(err)
+			return
+		}
+		defer graph.driver.Put(img.ID)
+
+		metaUnpacker := storage.NewJSONUnpacker(mfz)
+		fileGetter := storage.NewPathFileGetter(fsLayer)
+		logrus.Debugf("[graph] %s is at %q", img.ID, fsLayer)
+		ots := asm.NewOutputTarStream(fileGetter, metaUnpacker)
+		defer ots.Close()
+		if _, err := io.Copy(pW, ots); err != nil {
+			pW.CloseWithError(err)
+			return
+		}
+		pW.Close()
+	}()
+	return pR, nil
 }

graph/graph_windows.go

@@ -6,6 +6,7 @@ import (
 	"compress/gzip"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 
@@ -13,6 +14,8 @@ import (
 	"github.com/docker/docker/daemon/graphdriver/windows"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/archive"
+	"github.com/vbatts/tar-split/tar/asm"
+	"github.com/vbatts/tar-split/tar/storage"
 )
 
 // setupInitLayer populates a directory with mountpoints suitable
@@ -118,14 +121,14 @@ func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader
 		// Store the layer. If layerData is not nil, unpack it into the new layer
 		if layerData != nil {
 			// this is saving the tar-split metadata
-			mf, err := os.OpenFile(filepath.Join(root, tardataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
+			mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
 			if err != nil {
 				return err
 			}
 			defer mf.Close()
 			mfz := gzip.NewWriter(mf)
 			defer mfz.Close()
-			metaPacker := storage.NewJSONPacker(mf)
+			metaPacker := storage.NewJSONPacker(mfz)
 
 			inflatedLayerData, err := archive.DecompressStream(layerData)
 			if err != nil {
@@ -180,7 +183,50 @@ func (graph *Graph) TarLayer(img *image.Image) (arch archive.Archive, err error)
 		// We keep this functionality here so that we can still work with the VFS
 		// driver during development. VFS is not supported (and just will not work)
 		// for Windows containers.
-		// TODO(vbatts) let's reassemble!
-		return graph.driver.Diff(img.ID, img.Parent)
+		root := graph.imageRoot(img.ID)
+		mFileName := filepath.Join(root, tarDataFileName)
+		mf, err := os.Open(mFileName)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				logrus.Errorf("failed to open %q: %s", mFileName, err)
+			}
+			logrus.Debugf("[graph] TarLayer with traditional differ: %s", img.ID)
+			return graph.driver.Diff(img.ID, img.Parent)
+		}
+		pR, pW := io.Pipe()
+		// this will need to be in a goroutine, as we are returning the stream of a
+		// tar archive, but can not close the metadata reader early (when this
+		// function returns)...
+		go func() {
+			defer mf.Close()
+			// let's reassemble!
+			logrus.Debugf("[graph] TarLayer with reassembly: %s", img.ID)
+			mfz, err := gzip.NewReader(mf)
+			if err != nil {
+				pW.CloseWithError(fmt.Errorf("[graph] error with %s:  %s", mFileName, err))
+				return
+			}
+			defer mfz.Close()
+
+			// get our relative path to the container
+			fsLayer, err := graph.driver.Get(img.ID, "")
+			if err != nil {
+				pW.CloseWithError(err)
+				return
+			}
+			defer graph.driver.Put(img.ID)
+
+			metaUnpacker := storage.NewJSONUnpacker(mfz)
+			fileGetter := storage.NewPathFileGetter(fsLayer)
+			logrus.Debugf("[graph] %s is at %q", img.ID, fsLayer)
+			ots := asm.NewOutputTarStream(fileGetter, metaUnpacker)
+			defer ots.Close()
+			if _, err := io.Copy(pW, ots); err != nil {
+				pW.CloseWithError(err)
+				return
+			}
+			pW.Close()
+		}()
+		return pR, nil
 	}
 }