From 878dcb89f38e8eb7bb07ccd4a4e5ce622252ff30 Mon Sep 17 00:00:00 2001 From: Patrick Devine Date: Tue, 31 Mar 2015 13:58:17 -0700 Subject: [PATCH] Make a docker-in-docker dynamic binary and add RPM target This change adds a new docker-in-docker dynamic binary make target which builds a centos container for creating the dynamically linked binary. To use it, you first must create the static binary and then call the dind-dynbinary target. You can call it like: $ hack/make.sh binary dind-dynbinary rpm This would then package the dynamic binary into the rpm after having created it in the centos build container. Unfortunately with this approach you can't create the rpms and the debs with the same command. They have to be created separately otherwise the wrong version (static vs. dynamic) gets packaged. Various RPM fixes including: - Adding missing RPM dependencies. - Add sysconfig configuration files to the RPM. - Add an epoch to silence the fpm warning. - Remove unnecessary empty package. Signed-off-by: Patrick Devine Signed-off-by: Chad Metcalf --- Dockerfile | 1 + Dockerfile.centos | 36 ++++++ hack/make.sh | 2 +- hack/make/.integration-daemon-start | 2 +- hack/make/dind-dynbinary | 23 ++++ hack/make/rpm | 193 ++++++++++++++++++++++++++++ 6 files changed, 255 insertions(+), 2 deletions(-) create mode 100644 Dockerfile.centos create mode 100644 hack/make/dind-dynbinary create mode 100644 hack/make/rpm diff --git a/Dockerfile b/Dockerfile index 2b49fc1e0b..0c85a86331 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,6 +47,7 @@ RUN apt-get update && apt-get install -y \ python-pip \ python-websocket \ reprepro \ + rpm \ ruby1.9.1 \ ruby1.9.1-dev \ s3cmd=1.1.0* \ diff --git a/Dockerfile.centos b/Dockerfile.centos new file mode 100644 index 0000000000..0d11f2f2f7 --- /dev/null +++ b/Dockerfile.centos @@ -0,0 +1,36 @@ +# This file creates a CentOS docker container which can be used to create the Docker +# RPMs, however it shouldn't be called directly. +# + +FROM centos:7.0.1406 +MAINTAINER Patrick Devine (@pdev110) + +RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +# Packaged dependencies +RUN yum groupinstall -y "Development Tools" + +RUN yum install -y \ + btrfs-progs-devel \ + device-mapper-devel \ + glibc-static \ + libselinux-devel \ + sqlite-devel + +VOLUME /go + +ENV LXC_VERSION 1.0.7 +ENV GO_VERSION 1.4.2 +ENV PATH /go/bin:/usr/local/go/bin:$PATH +ENV GOPATH /go:/go/src/github.com/docker/docker/vendor +ENV GOFMT_VERSION 1.3.3 + +# Add an unprivileged user to be used for tests which need it +RUN groupadd -r docker +RUN useradd --create-home --gid docker unprivilegeduser + +WORKDIR /go/src/github.com/docker/docker +ENV DOCKER_BUILDTAGS selinux btrfs_noversion + +# Wrap all commands in the "docker-in-docker" script to allow nested containers +#ENTRYPOINT ["hack/dind"] diff --git a/hack/make.sh b/hack/make.sh index 699bedb379..96ca5902bc 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -274,7 +274,7 @@ main() { # We want this to fail if the bundles already exist and cannot be removed. # This is to avoid mixing bundles from different versions of the code. mkdir -p bundles - if [ -e "bundles/$VERSION" ]; then + if [ -e "bundles/$VERSION" ] && [ -z ${KEEPBUNDLE} ]; then echo "bundles/$VERSION already exists. Removing." rm -fr "bundles/$VERSION" && mkdir "bundles/$VERSION" || exit 1 echo diff --git a/hack/make/.integration-daemon-start b/hack/make/.integration-daemon-start index 57fd525028..0a889ae7a2 100644 --- a/hack/make/.integration-daemon-start +++ b/hack/make/.integration-daemon-start @@ -2,7 +2,7 @@ # see test-integration-cli for example usage of this script -export PATH="$DEST/../binary:$DEST/../dynbinary:$DEST/../gccgo:$DEST/../dyngccgo:$PATH" +export PATH="$DEST/../dynbinary:$DEST/../binary:$DEST/../gccgo:$DEST/../dyngccgo:$PATH" if ! command -v docker &> /dev/null; then echo >&2 'error: binary or dynbinary must be run before .integration-daemon-start' diff --git a/hack/make/dind-dynbinary b/hack/make/dind-dynbinary new file mode 100644 index 0000000000..a577ca0763 --- /dev/null +++ b/hack/make/dind-dynbinary @@ -0,0 +1,23 @@ +#!/bin/bash + +DEST=$1 +DOCKERBIN=$DEST/../binary/docker +BUILDDIR=$DEST/../dynbinary/ +RPM_PATH=$DEST/../rpm/ + +build_rpm() { + if [ ! -x $DOCKERBIN ]; then + echo "No docker binary was found to execute. This step requires 'binary' to be run first." + exit 1 + fi + + $DOCKERBIN -d 2>/dev/null & + $DOCKERBIN build -t centos-build -f Dockerfile.centos ./ + $DOCKERBIN run -it --rm --privileged -v /go:/go -v /usr/local:/usr/local -e "KEEPBUNDLE=true" --name centos-build-container centos-build hack/make.sh dynbinary + + # turn off the docker daemon + $DOCKERBIN rmi centos-build + cat /var/run/docker.pid | xargs kill +} + +build_rpm diff --git a/hack/make/rpm b/hack/make/rpm new file mode 100644 index 0000000000..6340f79131 --- /dev/null +++ b/hack/make/rpm @@ -0,0 +1,193 @@ +#!/bin/bash + +DEST=$1 +PACKAGE_NAME=${PACKAGE_NAME:-docker-engine} + +# XXX - The package version in CentOS gets messed up and inserts a '~' +# (including the single quote) if we use the same package +# version scheme as the deb packages. This doesn't work with +# rpmbuild. +PKGVERSION="${VERSION}" +# if we have a "-dev" suffix or have change in Git, let's make this package version more complex so it works better +if [[ "$VERSION" == *-dev ]] || [ -n "$(git status --porcelain)" ]; then + GIT_UNIX="$(git log -1 --pretty='%at')" + GIT_DATE="$(date --date "@$GIT_UNIX" +'%Y%m%d.%H%M%S')" + GIT_COMMIT="$(git log -1 --pretty='%h')" + GIT_VERSION="git${GIT_DATE}.0.${GIT_COMMIT}" + # GIT_VERSION is now something like 'git20150128.112847.0.17e840a' + PKGVERSION="$PKGVERSION~$GIT_VERSION" +fi + +# $ dpkg --compare-versions 1.5.0 gt 1.5.0~rc1 && echo true || echo false +# true +# $ dpkg --compare-versions 1.5.0~rc1 gt 1.5.0~git20150128.112847.17e840a && echo true || echo false +# true +# $ dpkg --compare-versions 1.5.0~git20150128.112847.17e840a gt 1.5.0~dev~git20150128.112847.17e840a && echo true || echo false +# true + +# ie, 1.5.0 > 1.5.0~rc1 > 1.5.0~git20150128.112847.17e840a > 1.5.0~dev~git20150128.112847.17e840a + +PACKAGE_ARCHITECTURE=`uname -i` + +PACKAGE_URL="http://www.docker.com/" +PACKAGE_MAINTAINER="support@docker.com" +PACKAGE_DESCRIPTION="Linux container runtime +Docker complements LXC with a high-level API which operates at the process +level. It runs unix processes with strong guarantees of isolation and +repeatability across servers. +Docker is a great building block for automating distributed systems: +large-scale web deployments, database clusters, continuous deployment systems, +private PaaS, service-oriented architectures, etc." +PACKAGE_LICENSE="Apache-2.0" + +# bundle the RPM using FPM -- we may want to change this to rpmbuild at some point +bundle_rpm() { + DIR=$DEST/build + + # Include our udev rules + mkdir -p $DIR/etc/udev/rules.d + cp contrib/udev/80-docker.rules $DIR/etc/udev/rules.d/ + + mkdir -p $DIR/usr/lib/systemd/system + cp contrib/init/systemd/docker.{service,socket} $DIR/usr/lib/systemd/system + + cat > $DIR/usr/lib/systemd/system/docker.service <<'EOF' +[Unit] +Description=Docker Application Container Engine +Documentation=http://docs.docker.com +After=network.target docker.socket +Requires=docker.socket + +[Service] +Type=notify +EnvironmentFile=-/etc/sysconfig/docker +EnvironmentFile=-/etc/sysconfig/docker-storage +ExecStart=/usr/bin/docker -d $OPTIONS $DOCKER_STORAGE_OPTIONS +LimitNOFILE=1048576 +LimitNPROC=1048576 +MountFlags=private + +[Install] +WantedBy=multi-user.target +EOF + + mkdir -p $DIR/etc/sysconfig + cat > $DIR/etc/sysconfig/docker <<'EOF' +# /etc/sysconfig/docker + +# Modify these options if you want to change the way the docker daemon runs +OPTIONS=--selinux-enabled -H fd:// + +# Location used for temporary files, such as those created by +# docker load and build operations. Default is /var/lib/docker/tmp +# Can be overriden by setting the following environment variable. +# DOCKER_TMPDIR=/var/tmp +EOF + +cat > $DIR/etc/sysconfig/docker-storage <<'EOF' +# By default, Docker uses a loopback-mounted sparse file in +# /var/lib/docker. The loopback makes it slower, and there are some +# restrictive defaults, such as 100GB max storage. + +# If your installation did not set a custom storage for Docker, you +# may do it below. + +# Example: Use a custom pair of raw logical volumes (one for metadata, +# one for data). +# DOCKER_STORAGE_OPTIONS = --storage-opt dm.metadatadev=/dev/mylogvol/my-docker-metadata --storage-opt dm.datadev=/dev/mylogvol/my-docker-data + +DOCKER_STORAGE_OPTIONS= +EOF + + # Include contributed completions + mkdir -p $DIR/etc/bash_completion.d + cp contrib/completion/bash/docker $DIR/etc/bash_completion.d/ + mkdir -p $DIR/usr/share/zsh/vendor-completions + cp contrib/completion/zsh/_docker $DIR/usr/share/zsh/vendor-completions/ + mkdir -p $DIR/etc/fish/completions + cp contrib/completion/fish/docker.fish $DIR/etc/fish/completions/ + + # Include contributed man pages + docs/man/md2man-all.sh -q + manRoot="$DIR/usr/share/man" + mkdir -p "$manRoot" + for manDir in docs/man/man?; do + manBase="$(basename "$manDir")" # "man1" + for manFile in "$manDir"/*; do + manName="$(basename "$manFile")" # "docker-build.1" + mkdir -p "$manRoot/$manBase" + gzip -c "$manFile" > "$manRoot/$manBase/$manName.gz" + done + done + + # Copy the binary + # This will fail if the dynbinary bundle hasn't been built + mkdir -p $DIR/usr/bin + cp $DEST/../dynbinary/docker-$VERSION $DIR/usr/bin/docker + cp $DEST/../dynbinary/dockerinit-$VERSION $DIR/usr/bin/dockerinit + + # Generate postinst/prerm/postrm scripts + cat > $DEST/postinst <<'EOF' +EOF + + cat > $DEST/preinst <<'EOF' +if ! getent group docker > /dev/null; then + groupadd --system docker +fi +EOF + + cat > $DEST/prerm <<'EOF' +EOF + + cat > $DEST/postrm <<'EOF' +## In case this system is running systemd, we make systemd reload the unit files +## to pick up changes. +#if [ -d /run/systemd/system ] ; then +# systemctl --system daemon-reload > /dev/null || true +#fi +EOF + + chmod +x $DEST/postinst $DEST/prerm $DEST/postrm $DEST/preinst + + ( + # switch directories so we create *.deb in the right folder + cd $DEST + + # create PACKAGE_NAME-VERSION package + fpm -s dir -C $DIR \ + --name $PACKAGE_NAME-$VERSION --version "$PKGVERSION" \ + --epoch 7 \ + --before-install $DEST/preinst \ + --after-install $DEST/postinst \ + --before-remove $DEST/prerm \ + --after-remove $DEST/postrm \ + --architecture "$PACKAGE_ARCHITECTURE" \ + --prefix / \ + --depends iptables \ + --depends xz \ + --depends "systemd >= 208-20" \ + --depends "device-mapper-libs >= 7:1.02.90-1" \ + --depends "device-mapper-event-libs >= 7:1.02.90-1" \ + --depends libselinux \ + --depends libsepol \ + --depends sqlite \ + --description "$PACKAGE_DESCRIPTION" \ + --maintainer "$PACKAGE_MAINTAINER" \ + --conflicts docker \ + --conflicts docker-io \ + --conflicts lxc-docker-virtual-package \ + --conflicts lxc-docker \ + --url "$PACKAGE_URL" \ + --license "$PACKAGE_LICENSE" \ + --config-files etc/sysconfig \ + --config-files etc/udev/rules.d/80-docker.rules \ + --rpm-compression gzip \ + -t rpm . + ) + + # clean up after ourselves so we have a clean output directory + rm $DEST/postinst $DEST/prerm $DEST/postrm $DEST/preinst + rm -r $DIR +} + +bundle_rpm