From 5a00326d29efb161826ef13dfd63ed2732017cd1 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 23 Jun 2015 19:44:49 -0400 Subject: [PATCH] graph: use tar archive entries for TarLayer if there is a tar-data.json.gz present for an image layer, then use it to create the tar archive, instead of the traditional graphdriver Diff. Signed-off-by: Vincent Batts Conflicts: graph/graph.go --- graph/graph.go | 2 +- graph/graph_unix.go | 55 +++++++++++++++++++++++++++++++++++++++--- graph/graph_windows.go | 54 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 102 insertions(+), 9 deletions(-) diff --git a/graph/graph.go b/graph/graph.go index 3947b3a7b0..10da49a722 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -87,7 +87,7 @@ const ( jsonFileName = "json" layersizeFileName = "layersize" digestFileName = "checksum" - tardataFileName = "tar-data.json.gz" + tarDataFileName = "tar-data.json.gz" ) var ( diff --git a/graph/graph_unix.go b/graph/graph_unix.go index 76f37715cb..e9a7f76416 100644 --- a/graph/graph_unix.go +++ b/graph/graph_unix.go @@ -6,14 +6,18 @@ import ( "compress/gzip" "encoding/json" "fmt" + "io" "os" "path/filepath" "strings" "syscall" + "github.com/Sirupsen/logrus" "github.com/docker/docker/image" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/system" + "github.com/vbatts/tar-split/tar/asm" + "github.com/vbatts/tar-split/tar/storage" ) // setupInitLayer populates a directory with mountpoints suitable @@ -92,14 +96,14 @@ func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader // Store the layer. If layerData is not nil, unpack it into the new layer if layerData != nil { // this is saving the tar-split metadata - mf, err := os.OpenFile(filepath.Join(root, tardataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) + mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) if err != nil { return err } defer mf.Close() mfz := gzip.NewWriter(mf) defer mfz.Close() - metaPacker := storage.NewJSONPacker(mf) + metaPacker := storage.NewJSONPacker(mfz) inflatedLayerData, err := archive.DecompressStream(layerData) if err != nil { @@ -134,6 +138,49 @@ func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader // TarLayer returns a tar archive of the image's filesystem layer. func (graph *Graph) TarLayer(img *image.Image) (arch archive.Archive, err error) { - // TODO(vbatts) let's reassemble! - return graph.driver.Diff(img.ID, img.Parent) + root := graph.imageRoot(img.ID) + mFileName := filepath.Join(root, tarDataFileName) + mf, err := os.Open(mFileName) + if err != nil { + if !os.IsNotExist(err) { + logrus.Errorf("failed to open %q: %s", mFileName, err) + } + logrus.Debugf("[graph] TarLayer with traditional differ: %s", img.ID) + return graph.driver.Diff(img.ID, img.Parent) + } + pR, pW := io.Pipe() + // this will need to be in a goroutine, as we are returning the stream of a + // tar archive, but can not close the metadata reader early (when this + // function returns)... + go func() { + defer mf.Close() + // let's reassemble! + logrus.Debugf("[graph] TarLayer with reassembly: %s", img.ID) + mfz, err := gzip.NewReader(mf) + if err != nil { + pW.CloseWithError(fmt.Errorf("[graph] error with %s: %s", mFileName, err)) + return + } + defer mfz.Close() + + // get our relative path to the container + fsLayer, err := graph.driver.Get(img.ID, "") + if err != nil { + pW.CloseWithError(err) + return + } + defer graph.driver.Put(img.ID) + + metaUnpacker := storage.NewJSONUnpacker(mfz) + fileGetter := storage.NewPathFileGetter(fsLayer) + logrus.Debugf("[graph] %s is at %q", img.ID, fsLayer) + ots := asm.NewOutputTarStream(fileGetter, metaUnpacker) + defer ots.Close() + if _, err := io.Copy(pW, ots); err != nil { + pW.CloseWithError(err) + return + } + pW.Close() + }() + return pR, nil } diff --git a/graph/graph_windows.go b/graph/graph_windows.go index b9ae1abfff..95396127d3 100644 --- a/graph/graph_windows.go +++ b/graph/graph_windows.go @@ -6,6 +6,7 @@ import ( "compress/gzip" "encoding/json" "fmt" + "io" "os" "path/filepath" @@ -13,6 +14,8 @@ import ( "github.com/docker/docker/daemon/graphdriver/windows" "github.com/docker/docker/image" "github.com/docker/docker/pkg/archive" + "github.com/vbatts/tar-split/tar/asm" + "github.com/vbatts/tar-split/tar/storage" ) // setupInitLayer populates a directory with mountpoints suitable @@ -118,14 +121,14 @@ func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader // Store the layer. If layerData is not nil, unpack it into the new layer if layerData != nil { // this is saving the tar-split metadata - mf, err := os.OpenFile(filepath.Join(root, tardataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) + mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) if err != nil { return err } defer mf.Close() mfz := gzip.NewWriter(mf) defer mfz.Close() - metaPacker := storage.NewJSONPacker(mf) + metaPacker := storage.NewJSONPacker(mfz) inflatedLayerData, err := archive.DecompressStream(layerData) if err != nil { @@ -180,7 +183,50 @@ func (graph *Graph) TarLayer(img *image.Image) (arch archive.Archive, err error) // We keep this functionality here so that we can still work with the VFS // driver during development. VFS is not supported (and just will not work) // for Windows containers. - // TODO(vbatts) let's reassemble! - return graph.driver.Diff(img.ID, img.Parent) + root := graph.imageRoot(img.ID) + mFileName := filepath.Join(root, tarDataFileName) + mf, err := os.Open(mFileName) + if err != nil { + if !os.IsNotExist(err) { + logrus.Errorf("failed to open %q: %s", mFileName, err) + } + logrus.Debugf("[graph] TarLayer with traditional differ: %s", img.ID) + return graph.driver.Diff(img.ID, img.Parent) + } + pR, pW := io.Pipe() + // this will need to be in a goroutine, as we are returning the stream of a + // tar archive, but can not close the metadata reader early (when this + // function returns)... + go func() { + defer mf.Close() + // let's reassemble! + logrus.Debugf("[graph] TarLayer with reassembly: %s", img.ID) + mfz, err := gzip.NewReader(mf) + if err != nil { + pW.CloseWithError(fmt.Errorf("[graph] error with %s: %s", mFileName, err)) + return + } + defer mfz.Close() + + // get our relative path to the container + fsLayer, err := graph.driver.Get(img.ID, "") + if err != nil { + pW.CloseWithError(err) + return + } + defer graph.driver.Put(img.ID) + + metaUnpacker := storage.NewJSONUnpacker(mfz) + fileGetter := storage.NewPathFileGetter(fsLayer) + logrus.Debugf("[graph] %s is at %q", img.ID, fsLayer) + ots := asm.NewOutputTarStream(fileGetter, metaUnpacker) + defer ots.Close() + if _, err := io.Copy(pW, ots); err != nil { + pW.CloseWithError(err) + return + } + pW.Close() + }() + return pR, nil } }