Merge pull request #604 from d0ugal/gh-deploy

Remove ghp-import dependency and update gh-deploy

appveyor.yml

@@ -2,9 +2,12 @@ build: false
 environment:
   matrix:
     - TOXENV: py27-unittests
+    - TOXENV: py33-unittests
     - TOXENV: py34-unittests
     - TOXENV: py27-integration
+    - TOXENV: py33-integration
     - TOXENV: py34-integration
+    - TOXENV: flake8
 init:
   - "ECHO %TOXENV%"
   - ps: "ls C:\\Python*"

docs/about/release-notes.md

@@ -17,11 +17,13 @@ You can determine your currently installed version using `mkdocs --version`:
 
 * Improve Unicode handling by ensuring that all YAML strings are loaded as Unicode.
 * Remove dependancy on the six library. (#583)
+* Remove dependancy on the ghp-import library. (#547)
 * Add `--quiet` and `--verbose` options to all subcommands.
 * Add short options (`-a`) to most command line options.
 * Add copyright footer for readthedocs theme.
-* Bugfix: Fix a JavaScript encoding problem when searching with spaces. (#586)
 * Stack traces are no longer displayed on socket errors, just an error message.
+* Bugfix: Fix a JavaScript encoding problem when searching with spaces. (#586)
+* Bugfix: gh-deploy now works if the mkdocs.yml is not in the git repo root (#578)
 
 ## Version 0.13.3 (2015-06-02)
 

docs/user-guide/configuration.md

@@ -82,7 +82,14 @@ google_analytics: ['UA-36723568-3', 'mkdocs.org']
 
 ### remote_branch
 
-Set the remote branch to commit to when using `gh-deploy` to update Github Pages. This option can be overriden by a commandline option in `gh-deploy`.
+Set the remote branch to commit to when using `gh-deploy` to deploy to Github Pages. This option can be overridden by a command line option in `gh-deploy`.
+
+**default**: `gh-pages`
+
+
+### remote_name
+
+Set the remote name to push to when using `gh-deploy` to deploy to Github Pages. This option can be overridden by a command line option in `gh-deploy`.
 
 **default**: `gh-pages`
 
@@ -252,7 +259,7 @@ for that extension:
     The Python-Markdown documentation provides a [list of extensions][exts]
     which are available out-of-the-box. For a list of configuration options
     available for a given extension, see the documentation for that extension.
-    
+
     You may also install and use various [third party extensions][3rd]. Consult the
     documentation provided by those extensions for installation instructions and
     available configuration options.

mkdocs/cli.py

@@ -172,13 +172,15 @@ def json_command(clean, config_file, strict, site_dir):
 @click.option('-f', '--config-file', type=click.File('rb'), help=config_file_help)
 @click.option('-m', '--message', help=commit_message_help)
 @click.option('-b', '--remote-branch', help=remote_branch_help)
+@click.option('-r', '--remote-name', help=remote_branch_help)
 @common_options
-def gh_deploy_command(config_file, clean, message, remote_branch):
+def gh_deploy_command(config_file, clean, message, remote_branch, remote_name):
     """Deply your documentation to GitHub Pages"""
     try:
         config = load_config(
             config_file=config_file,
-            remote_branch=remote_branch
+            remote_branch=remote_branch,
+            remote_name=remote_name
         )
         build.build(config, clean_site_dir=clean)
         gh_deploy.gh_deploy(config, message=message)

mkdocs/config/defaults.py

@@ -108,6 +108,9 @@ DEFAULT_SCHEMA = (
     ('remote_branch', config_options.Type(
         utils.string_types, default='gh-pages')),
 
+    # the remote name to push to when using gh-deploy
+    ('remote_name', config_options.Type(utils.string_types, default='origin')),
+
     # extra is a mapping/dictionary of data that is passed to the template.
     # This allows template authors to require extra configuration that not
     # relevant to all themes and doesn't need to be explicitly supported by

mkdocs/gh_deploy.py

@@ -3,48 +3,41 @@ import logging
 import subprocess
 import os
 
+import mkdocs
+from mkdocs.utils import ghp_import
+
 log = logging.getLogger(__name__)
 
+default_message = """Deployed {sha} with MkDocs version: {version}"""
 
-def gh_deploy(config, message=None):
 
-    if not os.path.exists('.git'):
-        log.info('Cannot deploy - this directory does not appear to be a git '
-                 'repository')
-        return
+def _is_cwd_git_repo():
+    proc = subprocess.Popen(['git', 'rev-parse', '--is-inside-work-tree'],
+                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    proc.communicate()
+    return proc.wait() == 0
 
-    command = ['ghp-import', '-p', config['site_dir']]
 
-    command.extend(['-b', config['remote_branch']])
+def _get_current_sha():
 
-    if message is not None:
-        command.extend(['-m', message])
+    proc = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'],
+                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
-    log.info("Copying '%s' to '%s' branch and pushing to GitHub.",
-             config['site_dir'], config['remote_branch'])
+    stdout, _ = proc.communicate()
+    sha = stdout.decode('utf-8').strip()
+    return sha
 
-    try:
-        subprocess.check_call(command)
-    except Exception:
-        log.exception("Failed to deploy to GitHub.")
-        return
 
-    # Does this repository have a CNAME set for GitHub pages?
-    if os.path.isfile('CNAME'):
-        # This GitHub pages repository has a CNAME configured.
-        with(open('CNAME', 'r')) as f:
-            cname_host = f.read().strip()
-        log.info('Based on your CNAME file, your documentation should be '
-                 'available shortly at: http://%s', cname_host)
-        log.info('NOTE: Your DNS records must be configured appropriately for '
-                 'your CNAME URL to work.')
-        return
+def _get_remote_url(remote_name):
 
     # No CNAME found.  We will use the origin URL to determine the GitHub
     # pages location.
-    url = subprocess.check_output(["git", "config", "--get",
-                                   "remote.origin.url"])
-    url = url.decode('utf-8').strip()
+    remote = "remote.%s.url" % remote_name
+    proc = subprocess.Popen(["git", "config", "--get", remote],
+                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    stdout, _ = proc.communicate()
+    url = stdout.decode('utf-8').strip()
 
     host = None
     path = None
@@ -53,6 +46,42 @@ def gh_deploy(config, message=None):
     elif 'github.com:' in url:
         host, path = url.split('github.com:', 1)
 
+    return host, path
+
+
+def gh_deploy(config, message=None):
+
+    if not _is_cwd_git_repo():
+        log.error('Cannot deploy - this directory does not appear to be a git '
+                  'repository')
+
+    if message is None:
+        sha = _get_current_sha()
+        message = default_message.format(version=mkdocs.__version__, sha=sha)
+
+    remote_branch = config['remote_branch']
+    remote_name = config['remote_name']
+
+    log.info("Copying '%s' to '%s' branch and pushing to GitHub.",
+             config['site_dir'], config['remote_branch'])
+
+    ghp_import.ghp_import(config['site_dir'], message, remote_name,
+                          remote_branch)
+
+    cname_file = os.path.join(config['site_dir'], 'CNAME')
+    # Does this repository have a CNAME set for GitHub pages?
+    if os.path.isfile(cname_file):
+        # This GitHub pages repository has a CNAME configured.
+        with(open(cname_file, 'r')) as f:
+            cname_host = f.read().strip()
+        log.info('Based on your CNAME file, your documentation should be '
+                 'available shortly at: http://%s', cname_host)
+        log.info('NOTE: Your DNS records must be configured appropriately for '
+                 'your CNAME URL to work.')
+        return
+
+    host, path = _get_remote_url(remote_name)
+
     if host is None:
         # This could be a GitHub Enterprise deployment.
         log.info('Your documentation should be available shortly.')

mkdocs/tests/gh_deploy_tests.py

@@ -0,0 +1,100 @@
+from __future__ import unicode_literals
+
+import unittest
+import mock
+
+from mkdocs import gh_deploy
+from mkdocs.config import load_config
+
+
+class TestGitHubDeploy(unittest.TestCase):
+
+    @mock.patch('subprocess.Popen')
+    def test_is_cwd_git_repo(self, mock_popeno):
+
+        mock_popeno().wait.return_value = 0
+
+        self.assertTrue(gh_deploy._is_cwd_git_repo())
+
+    @mock.patch('subprocess.Popen')
+    def test_is_cwd_not_git_repo(self, mock_popeno):
+
+        mock_popeno().wait.return_value = 1
+
+        self.assertFalse(gh_deploy._is_cwd_git_repo())
+
+    @mock.patch('subprocess.Popen')
+    def test_get_current_sha(self, mock_popeno):
+
+        mock_popeno().communicate.return_value = (b'6d98394\n', b'')
+
+        self.assertEqual(gh_deploy._get_current_sha(), u'6d98394')
+
+    @mock.patch('subprocess.Popen')
+    def test_get_remote_url_ssh(self, mock_popeno):
+
+        mock_popeno().communicate.return_value = (
+            b'git@github.com:mkdocs/mkdocs.git\n',
+            b''
+        )
+
+        expected = (u'git@', u'mkdocs/mkdocs.git')
+        self.assertEqual(expected, gh_deploy._get_remote_url('origin'))
+
+    @mock.patch('subprocess.Popen')
+    def test_get_remote_url_http(self, mock_popeno):
+
+        mock_popeno().communicate.return_value = (
+            b'https://github.com/mkdocs/mkdocs.git\n',
+            b''
+        )
+
+        expected = (u'https://', u'mkdocs/mkdocs.git')
+        self.assertEqual(expected, gh_deploy._get_remote_url('origin'))
+
+    @mock.patch('subprocess.Popen')
+    def test_get_remote_url_enterprise(self, mock_popeno):
+
+        mock_popeno().communicate.return_value = (
+            b'https://notgh.com/mkdocs/mkdocs.git\n',
+            b''
+        )
+
+        expected = (None, None)
+        self.assertEqual(expected, gh_deploy._get_remote_url('origin'))
+
+    @mock.patch('mkdocs.gh_deploy._is_cwd_git_repo', return_value=True)
+    @mock.patch('mkdocs.gh_deploy._get_current_sha', return_value='shashas')
+    @mock.patch('mkdocs.gh_deploy._get_remote_url', return_value=(None, None))
+    @mock.patch('mkdocs.gh_deploy.ghp_import.ghp_import')
+    def test_deploy(self, mock_import, get_remote, get_sha, is_repo):
+
+        config = load_config(
+            remote_branch='test',
+        )
+        gh_deploy.gh_deploy(config)
+
+    @mock.patch('mkdocs.gh_deploy._is_cwd_git_repo', return_value=True)
+    @mock.patch('mkdocs.gh_deploy._get_current_sha', return_value='shashas')
+    @mock.patch('mkdocs.gh_deploy._get_remote_url', return_value=(None, None))
+    @mock.patch('mkdocs.gh_deploy.ghp_import.ghp_import')
+    @mock.patch('os.path.isfile', return_value=False)
+    def test_deploy_no_cname(self, mock_isfile, mock_import, get_remote,
+                             get_sha, is_repo):
+
+        config = load_config(
+            remote_branch='test',
+        )
+        gh_deploy.gh_deploy(config)
+
+    @mock.patch('mkdocs.gh_deploy._is_cwd_git_repo', return_value=True)
+    @mock.patch('mkdocs.gh_deploy._get_current_sha', return_value='shashas')
+    @mock.patch('mkdocs.gh_deploy._get_remote_url', return_value=(
+        u'git@', u'mkdocs/mkdocs.git'))
+    @mock.patch('mkdocs.gh_deploy.ghp_import.ghp_import')
+    def test_deploy_hostname(self, mock_import, get_remote, get_sha, is_repo):
+
+        config = load_config(
+            remote_branch='test',
+        )
+        gh_deploy.gh_deploy(config)

mkdocs/tests/utils/ghp_import_tests.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import mock
+import os
+import subprocess
+import tempfile
+import unittest
+import shutil
+
+from mkdocs.utils import ghp_import
+
+
+class UtilsTests(unittest.TestCase):
+
+    @mock.patch('subprocess.call', auto_spec=True)
+    @mock.patch('subprocess.Popen', auto_spec=True)
+    def test_try_rebase(self, mock_popen, mock_call):
+
+        popen = mock.Mock()
+        mock_popen.return_value = popen
+        popen.communicate.return_value = (
+            '4c82346e4b1b816be89dd709d35a6b169aa3df61\n', '')
+        popen.wait.return_value = 0
+
+        ghp_import.try_rebase('origin', 'gh-pages')
+
+        mock_popen.assert_called_once_with(
+            ['git', 'rev-list', '--max-count=1', 'origin/gh-pages'],
+            stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+        mock_call.assert_called_once_with(
+            ['git', 'update-ref', 'refs/heads/gh-pages',
+             '4c82346e4b1b816be89dd709d35a6b169aa3df61'])
+
+    @mock.patch('subprocess.Popen', auto_spec=True)
+    def test_get_prev_commit(self, mock_popen):
+
+        popen = mock.Mock()
+        mock_popen.return_value = popen
+        popen.communicate.return_value = (
+            b'4c82346e4b1b816be89dd709d35a6b169aa3df61\n', '')
+        popen.wait.return_value = 0
+
+        result = ghp_import.get_prev_commit('test-branch')
+
+        self.assertEqual(result, u'4c82346e4b1b816be89dd709d35a6b169aa3df61')
+        mock_popen.assert_called_once_with(
+            ['git', 'rev-list', '--max-count=1', 'test-branch', '--'],
+            stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+
+    @mock.patch('subprocess.Popen', auto_spec=True)
+    def test_get_config(self, mock_popen):
+
+        popen = mock.Mock()
+        mock_popen.return_value = popen
+        popen.communicate.return_value = (
+            b'Dougal Matthews\n', '')
+
+        result = ghp_import.get_config('user.name')
+
+        self.assertEqual(result, u'Dougal Matthews')
+        mock_popen.assert_called_once_with(
+            ['git', 'config', 'user.name'],
+            stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+
+    @mock.patch('mkdocs.utils.ghp_import.get_prev_commit')
+    @mock.patch('mkdocs.utils.ghp_import.get_config')
+    def test_start_commit(self, mock_get_config, mock_get_prev_commit):
+
+        pipe = mock.Mock()
+        mock_get_config.side_effect = ['username', 'email']
+        mock_get_prev_commit.return_value = 'SHA'
+
+        ghp_import.start_commit(pipe, 'test-branch', 'test-message')
+
+        mock_get_prev_commit.assert_called_once_with('test-branch')
+        self.assertEqual(pipe.stdin.write.call_count, 5)
+
+    @mock.patch('mkdocs.utils.ghp_import.try_rebase', return_value=True)
+    @mock.patch('mkdocs.utils.ghp_import.get_prev_commit', return_value='sha')
+    @mock.patch('mkdocs.utils.ghp_import.get_config', return_value='config')
+    @mock.patch('subprocess.call', auto_spec=True)
+    @mock.patch('subprocess.Popen', auto_spec=True)
+    def test_ghp_import(self, mock_popen, mock_call, mock_get_config,
+                        mock_get_prev_commit, mock_try_rebase):
+
+        directory = tempfile.mkdtemp()
+        open(os.path.join(directory, 'file'), 'a').close()
+
+        try:
+            popen = mock.Mock()
+            mock_popen.return_value = popen
+            popen.communicate.return_value = ('', '')
+            popen.wait.return_value = 0
+
+            ghp_import.ghp_import(directory, "test message",
+                                  remote='fake-remote-name',
+                                  branch='fake-branch-name')
+
+            self.assertEqual(mock_popen.call_count, 2)
+            self.assertEqual(mock_call.call_count, 0)
+        finally:
+            shutil.rmtree(directory)

mkdocs/utils/__init__.py

@@ -338,7 +338,7 @@ def convert_markdown(markdown_source, extensions=None, extension_configs=None):
 def get_theme_names():
     """Return a list containing all the names of all the builtin themes."""
 
-    return os.listdir(os.path.join(os.path.dirname(__file__), 'themes'))
+    return os.listdir(os.path.join(os.path.dirname(__file__), '..', 'themes'))
 
 
 def filename_to_title(filename):

mkdocs/utils/ghp_import.py

@@ -0,0 +1,173 @@
+#! /usr/bin/env python
+#
+# This file is part of the ghp-import package released under
+# the Tumbolia Public License.
+
+#                            Tumbolia Public License
+
+# Copyright 2013, Paul Davis <paul.joseph.davis@gmail.com>
+
+# Copying and distribution of this file, with or without modification, are
+# permitted in any medium without royalty provided the copyright notice and this
+# notice are preserved.
+
+# TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+#   0. opan saurce LOL
+
+from __future__ import unicode_literals
+
+import errno
+import logging
+import os
+import subprocess as sp
+import sys
+import time
+import unicodedata
+
+log = logging.getLogger(__name__)
+
+
+if sys.version_info[0] == 3:
+    def enc(text):
+        if isinstance(text, bytes):
+            return text
+        return text.encode()
+
+    def dec(text):
+        if isinstance(text, bytes):
+            return text.decode('utf-8')
+        return text
+
+    def write(pipe, data):
+        try:
+            pipe.stdin.write(data)
+        except IOError as e:
+            if e.errno != errno.EPIPE:
+                raise
+else:
+    def enc(text):
+        if isinstance(text, unicode):
+            return text.encode('utf-8')
+        return text
+
+    def dec(text):
+        if isinstance(text, unicode):
+            return text
+        return text.decode('utf-8')
+
+    def write(pipe, data):
+        pipe.stdin.write(data)
+
+
+def normalize_path(path):
+    # Fix unicode pathnames on OS X
+    # See: http://stackoverflow.com/a/5582439/44289
+    if sys.platform == "darwin":
+        return unicodedata.normalize("NFKC", dec(path))
+    return path
+
+
+def try_rebase(remote, branch):
+    cmd = ['git', 'rev-list', '--max-count=1', '%s/%s' % (remote, branch)]
+    p = sp.Popen(cmd, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE)
+    (rev, _) = p.communicate()
+    if p.wait() != 0:
+        return True
+    cmd = ['git', 'update-ref', 'refs/heads/%s' % branch, rev.strip()]
+    if sp.call(cmd) != 0:
+        return False
+    return True
+
+
+def get_config(key):
+    p = sp.Popen(['git', 'config', key], stdin=sp.PIPE, stdout=sp.PIPE)
+    (value, _) = p.communicate()
+    return value.decode('utf-8').strip()
+
+
+def get_prev_commit(branch):
+    cmd = ['git', 'rev-list', '--max-count=1', branch, '--']
+    p = sp.Popen(cmd, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE)
+    (rev, _) = p.communicate()
+    if p.wait() != 0:
+        return None
+    return rev.decode('utf-8').strip()
+
+
+def mk_when(timestamp=None):
+    if timestamp is None:
+        timestamp = int(time.time())
+    currtz = "%+05d" % (-1 * time.timezone / 36)  # / 3600 * 100
+    return "%s %s" % (timestamp, currtz)
+
+
+def start_commit(pipe, branch, message):
+    uname = dec(get_config("user.name"))
+    email = dec(get_config("user.email"))
+    write(pipe, enc('commit refs/heads/%s\n' % branch))
+    write(pipe, enc('committer %s <%s> %s\n' % (uname, email, mk_when())))
+    write(pipe, enc('data %d\n%s\n' % (len(message), message)))
+    head = get_prev_commit(branch)
+    if head:
+        write(pipe, enc('from %s\n' % head))
+    write(pipe, enc('deleteall\n'))
+
+
+def add_file(pipe, srcpath, tgtpath):
+    with open(srcpath, "rb") as handle:
+        if os.access(srcpath, os.X_OK):
+            write(pipe, enc('M 100755 inline %s\n' % tgtpath))
+        else:
+            write(pipe, enc('M 100644 inline %s\n' % tgtpath))
+        data = handle.read()
+        write(pipe, enc('data %d\n' % len(data)))
+        write(pipe, enc(data))
+        write(pipe, enc('\n'))
+
+
+def add_nojekyll(pipe):
+    write(pipe, enc('M 100644 inline .nojekyll\n'))
+    write(pipe, enc('data 0\n'))
+    write(pipe, enc('\n'))
+
+
+def gitpath(fname):
+    norm = os.path.normpath(fname)
+    return "/".join(norm.split(os.path.sep))
+
+
+def run_import(srcdir, branch, message, nojekyll):
+    cmd = ['git', 'fast-import', '--date-format=raw', '--quiet']
+    kwargs = {"stdin": sp.PIPE}
+    if sys.version_info >= (3, 2, 0):
+        kwargs["universal_newlines"] = False
+    pipe = sp.Popen(cmd, **kwargs)
+    start_commit(pipe, branch, message)
+    for path, _, fnames in os.walk(srcdir):
+        for fn in fnames:
+            fpath = os.path.join(path, fn)
+            fpath = normalize_path(fpath)
+            gpath = gitpath(os.path.relpath(fpath, start=srcdir))
+            add_file(pipe, fpath, gpath)
+    if nojekyll:
+        add_nojekyll(pipe)
+    write(pipe, enc('\n'))
+    pipe.stdin.close()
+    if pipe.wait() != 0:
+        sys.stdout.write(enc("Failed to process commit.\n"))
+
+
+def ghp_import(directory, message, remote='origin', branch='gh-pages'):
+
+    if not try_rebase(remote, branch):
+        log.error("Failed to rebase %s branch.", branch)
+
+    nojekyll = True
+
+    run_import(directory, branch, message, nojekyll)
+
+    proc = sp.Popen(['git', 'push', remote, branch],
+                    stdout=sp.PIPE, stderr=sp.PIPE)
+    proc.communicate()
+    return proc.wait() == 0

requirements/project.txt

@@ -1,5 +1,4 @@
 click>=4.0
-ghp-import>=0.4.1
 Jinja2>=2.7.1
 livereload>=2.3.2
 Markdown>=2.5

setup.py

@@ -59,7 +59,6 @@ setup(
     package_data=get_package_data("mkdocs"),
     install_requires=[
         'click>=4.0',
-        'ghp-import>=0.4.1',
         'Jinja2>=2.7.1',
         'livereload>=2.3.2',
         'Markdown>=2.3.1,<2.5' if PY26 else 'Markdown>=2.3.1',