From 8efccf320da5922d959b866c2190766a0c8e8980 Mon Sep 17 00:00:00 2001 From: Misty Stanley-Jones Date: Thu, 21 Dec 2017 11:40:07 -0800 Subject: [PATCH] Docs publishing tools --- .dockerignore | 6 ++ .gitignore | 10 +++ Dockerfile.builder | 18 ++++++ Dockerfile.builder.onbuild | 16 +++++ Dockerfile.nginx.onbuild | 19 ++++++ README.md | 116 +++++++++++++++++++++++++++++++++++ nginx-overrides.conf | 9 +++ scripts/compress-assets.sh | 30 +++++++++ scripts/create-permalinks.sh | 28 +++++++++ scripts/fix-archives.sh | 44 +++++++++++++ scripts/minify-assets.sh | 24 ++++++++ scripts/normalize-links.sh | 52 ++++++++++++++++ 12 files changed, 372 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 Dockerfile.builder create mode 100644 Dockerfile.builder.onbuild create mode 100644 Dockerfile.nginx.onbuild create mode 100644 README.md create mode 100644 nginx-overrides.conf create mode 100755 scripts/compress-assets.sh create mode 100755 scripts/create-permalinks.sh create mode 100755 scripts/fix-archives.sh create mode 100755 scripts/minify-assets.sh create mode 100755 scripts/normalize-links.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..5ccb333fa7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.dockerignore +Dockerfile +.git +.github +tests +_site \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..3375abca2b --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +**/.DS_Store +**/desktop.ini +.bundle/** +.jekyll-metadata +_site/** +.sass-cache/** +CNAME +Gemfile.lock +_samples/library/** +_kbase/** \ No newline at end of file diff --git a/Dockerfile.builder b/Dockerfile.builder new file mode 100644 index 0000000000..0c2a0a47e1 --- /dev/null +++ b/Dockerfile.builder @@ -0,0 +1,18 @@ +# Build minifier utility +FROM golang:1.9-alpine AS minifier +RUN apk add --no-cache git +RUN go get -d github.com/tdewolff/minify/cmd/minify \ + && go build -v -o /minify github.com/tdewolff/minify/cmd/minify + +# Set the version of Github Pages to use for each docs archive +FROM starefossen/github-pages:147 + +# Get some utilities we need for post-build steps +RUN apk update && apk add bash wget subversion gzip + +# Copy scripts used for static HTML post-processing. +COPY scripts /scripts +COPY --from=minifier /minify /scripts/minify + +# Print out a message if someone tries to run this image on its own +CMD echo 'This image is only meant to be used as a base image for building docs.' \ No newline at end of file diff --git a/Dockerfile.builder.onbuild b/Dockerfile.builder.onbuild new file mode 100644 index 0000000000..afadc126ce --- /dev/null +++ b/Dockerfile.builder.onbuild @@ -0,0 +1,16 @@ +# Get Jekyll build env +FROM docs/docker.github.io:docs-builder AS builder + +# Make the version accessible to this build-stage +ONBUILD ARG VER + +# Build the docs from this branch +ONBUILD COPY . /source +ONBUILD RUN jekyll build --source /source --destination /site/${VER} + +# Do post-processing on archive +ONBUILD RUN /scripts/fix-archives.sh /site/ ${VER} + +# Make an index.html and 404.html which will redirect / to /${VER}/ +ONBUILD RUN echo "Redirect for ${VER}

If you are not redirected automatically, click here.

" > /site/index.html +ONBUILD RUN echo "Redirect for ${VER}

If you are not redirected automatically, click here.

" > /site/404.html diff --git a/Dockerfile.nginx.onbuild b/Dockerfile.nginx.onbuild new file mode 100644 index 0000000000..8b57f715f5 --- /dev/null +++ b/Dockerfile.nginx.onbuild @@ -0,0 +1,19 @@ +# Base image to use for building documentation archives +# this image uses "ONBUILD" to perform all required steps in the archives +# and relies upon its parent image having a layer called `builder`. + +FROM nginx:alpine + +# Make the version accessible to this build-stage, and copy it to an ENV so that it persists in the final image +ONBUILD ARG VER +ONBUILD ENV VER=$VER + +# Clean out any existing HTML files, and copy the HTML from the builder stage to the default location for Nginx +ONBUILD RUN rm -rf /usr/share/nginx/html/* +ONBUILD COPY --from=builder /site /usr/share/nginx/html + +# Copy the Nginx config +COPY nginx-overrides.conf /etc/nginx/conf.d/default.conf + +# Start Nginx to serve the archive at / (which will redirect to the version-specific dir) +CMD echo -e "Docker docs are viewable at:\nhttp://0.0.0.0:4000"; exec nginx -g 'daemon off;' \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000000..619d680ce2 --- /dev/null +++ b/README.md @@ -0,0 +1,116 @@ +This branch contains Dockerfiles and configuration files which create base +images used by the Docker docs publication process. + + +> **Warning**: Each time a change is pushed to this branch, all the images built +from this branch will be automatically rebuilt on Docker Cloud. This will in +turn cause all the docs archives and the docs-base image (which has a copy of +each doc archive in a separate directory) to be rebuilt. + +## Overview of creating an archive image + +1. The archive's `Dockerfile` is invoked. + +2. It is based on the `docker.github.io/docs:docs-builder` image (built by the + [Dockerfile.builder](Dockerfile.builder) Dockerfile in the `publish-tools` + branch). That image in turn invokes the + `docker.github.io/docs:docs-builder-onbuild` image (built by the + [Dockerfile.builder.onbuild](Dockerfile.builder.onbuild) Dockerfile in the + `publish-tools` branch). Post-processing scripts included in this image. + + At the end of step 2, all the static HTML has been built and post-processing + has been done on it. + +3. The archive's `Dockerfile` resets to the + `docker.github.io/docs:nginx-onbuild` image (built by the + [Dockerfile.nginx](Dockerfile.nginx.onbuild) Dockerfile in the `publish-tools` + branch). This image contains a Nginx environment, our custom Nginx + configuration file, and some (tiny) scripts we use for post-processing HTML. + + At the end of step 3, the static HTML from step 2 has been copied into the + much smaller layer created by the `docker.github.io/docs:nginx-onbuild` + image, along with the Nginx configuration. The static HTML for the archive + is now self-browseable. + +The result of these three steps is the archive Dockerfile, which is tagged as +`docker.github.io/docs:v` as set in the Dockerfile in step 1. This image +has two uses: + +- It can be deployed as a standalone docs archive for that version. +- It is also incorporated into the process which builds the +[`docker.github.io/docs:docs-base`](https://github.com/docker/docker.github.io/tree/docs-base) +image. That image holds all of the archives, one per directory, and is the base +image for the documentation published on https://docs.docker.com/). + +## Build all of the required images locally + +All of the images are built using the auto-builder function of Docker Cloud. +To test the entire process end-to-end on your local system, you need to build +each of the required images locally and tag it appropriately: + +1. Locally build and tag all tooling images: + + ```bash + $ git checkout publish-tools + $ docker build -t docker.github.io/docs:docs-builder -f Dockerfile.builder . + $ docker build -t docker.github.io/docs:docs-builder-onbuild -f Dockerfile.builder.onbuild . + $ docker build -t docker.github.io/docs:nginx-onbuild -f Dockerfile.nginx.onbuild . + ``` + +2. For each archive branch (`v1.4` through whatever is the newest archive + (currently `v17.09`)), build that archive branch's image. This example does + that for the `v1.4` archive branch: + + ```bash + $ git checkout v1.4 + $ docker build -t docker.github.io/docs:v1.4 . + ``` + + > **Note**: The archive Dockerfile looks like this (comments have been + > removed). Each of the two `FROM` lines will use the `VER` build-time + > argument as a parameter. + > + > ```Dockerfile + > ARG VER=v1.4 + > FROM docs/docker.github.io:docs-builder-onbuild AS builder + > FROM docs/docker.github.io:nginx-onbuild + > ``` + +3. After repeating step 2 for each archive branch, build the `docs-base` image: + + ```bash + $ git checkout docs-base + $ docker build -t docker.github.io/docs:docs-base . + ``` + + This copies all the static HTML from each of the images created in step 2 + into the `docs-base` image. + +4. After building `docs-base`, build the image for `master`: + + ```bash + $ git checkout master + $ docker build -t docker.github.io/docs:latest -t docker.github.io/docs:livedocs . + ``` + + The resulting image has the static HTML for each archive and for the + contents of `master`. To test it: + + ```bash + $ docker run --rm -it -p 4000:4000 docker.github.io/docs:latest + ``` + +## When to change each file in this branch + +- `Dockerfile.builder`: to update the version of Jekyll or to add or modify + tools needed by the Jekyll environment. +- `Dockerfile.builder.onbuild`: to change the logic for building archives using + Jekyll or post-processing the static HTML. +- contents of the `scripts` directory: To change the behavior of any of the + individual post-processing scripts which run against the static HTML. +- `Dockerfile.nginx.onbuild`: To change the base Nginx image or to change the + command that starts Nginx for an archive. +- `nginx-overrides.conf`: To change the Nginx configuration used by all of the + images which serve static HTML. + + diff --git a/nginx-overrides.conf b/nginx-overrides.conf new file mode 100644 index 0000000000..7564d0c91a --- /dev/null +++ b/nginx-overrides.conf @@ -0,0 +1,9 @@ +server { + port_in_redirect off; + listen 4000; + error_page 403 404 /404.html; + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } +} \ No newline at end of file diff --git a/scripts/compress-assets.sh b/scripts/compress-assets.sh new file mode 100755 index 0000000000..15e90c7aec --- /dev/null +++ b/scripts/compress-assets.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +TARGET="$1" + +if [ -z "$TARGET" ]; then + echo "Usage: $0 " + echo "No target was given. Exiting." + exit 1 +fi + +if ! [ -d "$TARGET" ]; then + echo "Target directory $TARGET does not exist. Exiting." + exit 1 +fi + +# Pre-gzip files. note that the ngx_http_gzip_static_module requires both the +# compressed, and uncompressed files to be present see: +# http://nginx.org/en/docs/http/ngx_http_gzip_static_module.html +# +# Compressed content is roughly 80% smaller than uncompressed but will make the +# final image 20% bigger (due to both uncompressed and compressed content being +# included in the image) +printf "Compressing assets in $TARGET" +printf "."; find ${TARGET} -type f -iname "*.html" -exec gzip -f -9 --keep {} + +printf "."; find ${TARGET} -type f -iname "*.js" -exec gzip -f -9 --keep {} + +printf "."; find ${TARGET} -type f -iname "*.css" -exec gzip -f -9 --keep {} + +printf "."; find ${TARGET} -type f -iname "*.json" -exec gzip -f -9 --keep {} + +printf "."; find ${TARGET} -type f -iname "*.svg" -exec gzip -f -9 --keep {} + +printf "."; find ${TARGET} -type f -iname "*.txt" -exec gzip -f -9 --keep {} + +echo "done." \ No newline at end of file diff --git a/scripts/create-permalinks.sh b/scripts/create-permalinks.sh new file mode 100755 index 0000000000..6766f039cc --- /dev/null +++ b/scripts/create-permalinks.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +TARGET="$1" +VER="$2" + +if [ -z "$TARGET" ]; then + echo "Usage: $0 " + echo "No target provided. Exiting." + exit 1 +fi + +if [ -z "$VER" ]; then + echo "Usage: $0 " + echo "No version provided. Exiting." + exit 1 +else + BASEURL="$VER/" +fi + +if ! [ -d "$TARGET" ]; then + echo "Target directory $TARGET does not exist. Exiting." + exit 1 +fi + +# Create permalinks for archived versions +printf "Creating permalinks for $VER" +printf "."; find ${TARGET} -type f -name '*.html' -print0 | xargs -0 sed -i 's#\(src\|href\)=\("\{0,1\}\)/#\1=\2/'"$BASEURL"'#g'; +echo "done" \ No newline at end of file diff --git a/scripts/fix-archives.sh b/scripts/fix-archives.sh new file mode 100755 index 0000000000..bd7cfa4f92 --- /dev/null +++ b/scripts/fix-archives.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +TARGET="$1" + +if [ -z "$TARGET" ]; then + echo "Usage: $0 " + echo "No target provided. Exiting." + exit 1 +fi + +VER="$2" + +if [ -z "$VER" ]; then + echo "Usage: $0 " + echo "No version provided. Exiting." + exit 1 +fi + + +if ! [ -d "$TARGET/$VER" ]; then + echo "Target directory $TARGET/$VER does not exist. Exiting." + exit 1 +fi + + +echo "Doing extra processing for archive in $TARGET/$VER:" + +echo " Fixing links..." + +sh /scripts/normalize-links.sh "$TARGET" "$VER" + +echo " Minifying assets..." + +sh /scripts/minify-assets.sh "$TARGET" "$VER" + +echo " Creating permalinks..." + +sh /scripts/create-permalinks.sh "$TARGET" "$VER" + +echo " Compressing assets..." + +sh /scripts/compress-assets.sh "$TARGET" + +echo "Finished cleaning up $TARGET/$VER." \ No newline at end of file diff --git a/scripts/minify-assets.sh b/scripts/minify-assets.sh new file mode 100755 index 0000000000..326ea88c00 --- /dev/null +++ b/scripts/minify-assets.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +TARGET="$1" +VER="$2" + +if [ -z "$TARGET" -o -z "$VER" ]; then + echo "Usage: $0 " + echo "Either or is missing. Exiting." + exit 1 +fi + +if ! [ -d "$TARGET" ]; then + echo "Target directory $TARGET does not exist. Exiting." + exit 1 +fi + +# Minify assets. This benefits both the compressed, and uncompressed versions +printf "Optimizing " + +printf "html..."; /scripts/minify -r --type=html --match=\.html -o ${TARGET}/ ${TARGET} || true +printf "css..." ; /scripts/minify -r --type=css --match=\.css -o ${TARGET}/ ${TARGET} || true +printf "json..."; /scripts/minify -r --type=json --match=\.json -o ${TARGET}/ ${TARGET} || true + +echo "done." \ No newline at end of file diff --git a/scripts/normalize-links.sh b/scripts/normalize-links.sh new file mode 100755 index 0000000000..c8950a2151 --- /dev/null +++ b/scripts/normalize-links.sh @@ -0,0 +1,52 @@ +#!/bin/sh + +TARGET="$1" +VER="$2" + +if [ -z "$TARGET" ]; then + echo "Usage: $0 " + echo "No target provided. Exiting." + exit 1 +fi + +if [ -z "$VER" ]; then + echo "Usage: $0 " + echo "No version provided. Exiting." + exit 1 +else + BASEURL="$VER/" +fi + + +if ! [ -d "$TARGET/$VER" ]; then + echo "Target directory $TARGET/$VER does not exist. Exiting." + exit 1 +fi + +# Note: pattern '\(src\|href\)=\("\{0,1\}\)' matches: +# - src= +# - href= +# followed by an optional double quote + +printf "Cleaning up $VER" + +# Fix relative links for archive +printf "."; find ${TARGET} -type f -name '*.html' -print0 | xargs -0 sed -i 's#\(src\|href\)=\("\{0,1\}\)\(http\|https\)://\(docs\|docs-stage\).docker.com/#\1=\2/#g' + +# Substitute https:// for schema-less resources (src="//analytics.google.com") +# We're replacing them to prevent them being seen as absolute paths below +printf "."; find ${TARGET} -type f -name '*.html' -print0 | xargs -0 sed -i 's#\(src\|href\)=\("\{0,1\}\)//#\1="https://#g' + +# And some archive versions already have URLs starting with '/version/' +printf "."; find ${TARGET} -type f -name '*.html' -print0 | xargs -0 sed -i 's#\(src\|href\)=\("\{0,1\}\)/'"$BASEURL"'#\1="/#g' + +case "$VER" in v1.4|v1.5|v1.6|v1.7|v1.10) + # Archived versions 1.7 and under use some absolute links, and v1.10 uses + # "relative" links to sources (href="./css/"). Remove those to make them + # work :) + printf "."; find ${TARGET} -type f -name '*.html' -print0 | xargs -0 sed -i 's#\(src\|href\)=\("\{0,1\}\)\./#\1="/#g' + ;; +esac + +echo "done" +