diff --git a/appveyor.yml b/appveyor.yml index c7d053bb..4ef82284 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,9 +2,12 @@ build: false environment: matrix: - TOXENV: py27-unittests + - TOXENV: py33-unittests - TOXENV: py34-unittests - TOXENV: py27-integration + - TOXENV: py33-integration - TOXENV: py34-integration + - TOXENV: flake8 init: - "ECHO %TOXENV%" - ps: "ls C:\\Python*" diff --git a/docs/about/release-notes.md b/docs/about/release-notes.md index 620ea45f..3e2dbae1 100644 --- a/docs/about/release-notes.md +++ b/docs/about/release-notes.md @@ -17,11 +17,13 @@ You can determine your currently installed version using `mkdocs --version`: * Improve Unicode handling by ensuring that all YAML strings are loaded as Unicode. * Remove dependancy on the six library. (#583) +* Remove dependancy on the ghp-import library. (#547) * Add `--quiet` and `--verbose` options to all subcommands. * Add short options (`-a`) to most command line options. * Add copyright footer for readthedocs theme. -* Bugfix: Fix a JavaScript encoding problem when searching with spaces. (#586) * Stack traces are no longer displayed on socket errors, just an error message. +* Bugfix: Fix a JavaScript encoding problem when searching with spaces. (#586) +* Bugfix: gh-deploy now works if the mkdocs.yml is not in the git repo root (#578) ## Version 0.13.3 (2015-06-02) diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index cda78129..a8c2e31e 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -82,7 +82,14 @@ google_analytics: ['UA-36723568-3', 'mkdocs.org'] ### remote_branch -Set the remote branch to commit to when using `gh-deploy` to update Github Pages. This option can be overriden by a commandline option in `gh-deploy`. +Set the remote branch to commit to when using `gh-deploy` to deploy to Github Pages. This option can be overridden by a command line option in `gh-deploy`. + +**default**: `gh-pages` + + +### remote_name + +Set the remote name to push to when using `gh-deploy` to deploy to Github Pages. This option can be overridden by a command line option in `gh-deploy`. **default**: `gh-pages` @@ -252,7 +259,7 @@ for that extension: The Python-Markdown documentation provides a [list of extensions][exts] which are available out-of-the-box. For a list of configuration options available for a given extension, see the documentation for that extension. - + You may also install and use various [third party extensions][3rd]. Consult the documentation provided by those extensions for installation instructions and available configuration options. diff --git a/mkdocs/cli.py b/mkdocs/cli.py index fdf973ad..9fb495e4 100644 --- a/mkdocs/cli.py +++ b/mkdocs/cli.py @@ -172,13 +172,15 @@ def json_command(clean, config_file, strict, site_dir): @click.option('-f', '--config-file', type=click.File('rb'), help=config_file_help) @click.option('-m', '--message', help=commit_message_help) @click.option('-b', '--remote-branch', help=remote_branch_help) +@click.option('-r', '--remote-name', help=remote_branch_help) @common_options -def gh_deploy_command(config_file, clean, message, remote_branch): +def gh_deploy_command(config_file, clean, message, remote_branch, remote_name): """Deply your documentation to GitHub Pages""" try: config = load_config( config_file=config_file, - remote_branch=remote_branch + remote_branch=remote_branch, + remote_name=remote_name ) build.build(config, clean_site_dir=clean) gh_deploy.gh_deploy(config, message=message) diff --git a/mkdocs/config/defaults.py b/mkdocs/config/defaults.py index f84d9483..2a3d935f 100644 --- a/mkdocs/config/defaults.py +++ b/mkdocs/config/defaults.py @@ -108,6 +108,9 @@ DEFAULT_SCHEMA = ( ('remote_branch', config_options.Type( utils.string_types, default='gh-pages')), + # the remote name to push to when using gh-deploy + ('remote_name', config_options.Type(utils.string_types, default='origin')), + # extra is a mapping/dictionary of data that is passed to the template. # This allows template authors to require extra configuration that not # relevant to all themes and doesn't need to be explicitly supported by diff --git a/mkdocs/gh_deploy.py b/mkdocs/gh_deploy.py index 5c557f47..949a8019 100644 --- a/mkdocs/gh_deploy.py +++ b/mkdocs/gh_deploy.py @@ -3,48 +3,41 @@ import logging import subprocess import os +import mkdocs +from mkdocs.utils import ghp_import + log = logging.getLogger(__name__) +default_message = """Deployed {sha} with MkDocs version: {version}""" -def gh_deploy(config, message=None): - if not os.path.exists('.git'): - log.info('Cannot deploy - this directory does not appear to be a git ' - 'repository') - return +def _is_cwd_git_repo(): + proc = subprocess.Popen(['git', 'rev-parse', '--is-inside-work-tree'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc.communicate() + return proc.wait() == 0 - command = ['ghp-import', '-p', config['site_dir']] - command.extend(['-b', config['remote_branch']]) +def _get_current_sha(): - if message is not None: - command.extend(['-m', message]) + proc = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) - log.info("Copying '%s' to '%s' branch and pushing to GitHub.", - config['site_dir'], config['remote_branch']) + stdout, _ = proc.communicate() + sha = stdout.decode('utf-8').strip() + return sha - try: - subprocess.check_call(command) - except Exception: - log.exception("Failed to deploy to GitHub.") - return - # Does this repository have a CNAME set for GitHub pages? - if os.path.isfile('CNAME'): - # This GitHub pages repository has a CNAME configured. - with(open('CNAME', 'r')) as f: - cname_host = f.read().strip() - log.info('Based on your CNAME file, your documentation should be ' - 'available shortly at: http://%s', cname_host) - log.info('NOTE: Your DNS records must be configured appropriately for ' - 'your CNAME URL to work.') - return +def _get_remote_url(remote_name): # No CNAME found. We will use the origin URL to determine the GitHub # pages location. - url = subprocess.check_output(["git", "config", "--get", - "remote.origin.url"]) - url = url.decode('utf-8').strip() + remote = "remote.%s.url" % remote_name + proc = subprocess.Popen(["git", "config", "--get", remote], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + stdout, _ = proc.communicate() + url = stdout.decode('utf-8').strip() host = None path = None @@ -53,6 +46,42 @@ def gh_deploy(config, message=None): elif 'github.com:' in url: host, path = url.split('github.com:', 1) + return host, path + + +def gh_deploy(config, message=None): + + if not _is_cwd_git_repo(): + log.error('Cannot deploy - this directory does not appear to be a git ' + 'repository') + + if message is None: + sha = _get_current_sha() + message = default_message.format(version=mkdocs.__version__, sha=sha) + + remote_branch = config['remote_branch'] + remote_name = config['remote_name'] + + log.info("Copying '%s' to '%s' branch and pushing to GitHub.", + config['site_dir'], config['remote_branch']) + + ghp_import.ghp_import(config['site_dir'], message, remote_name, + remote_branch) + + cname_file = os.path.join(config['site_dir'], 'CNAME') + # Does this repository have a CNAME set for GitHub pages? + if os.path.isfile(cname_file): + # This GitHub pages repository has a CNAME configured. + with(open(cname_file, 'r')) as f: + cname_host = f.read().strip() + log.info('Based on your CNAME file, your documentation should be ' + 'available shortly at: http://%s', cname_host) + log.info('NOTE: Your DNS records must be configured appropriately for ' + 'your CNAME URL to work.') + return + + host, path = _get_remote_url(remote_name) + if host is None: # This could be a GitHub Enterprise deployment. log.info('Your documentation should be available shortly.') diff --git a/mkdocs/tests/gh_deploy_tests.py b/mkdocs/tests/gh_deploy_tests.py new file mode 100644 index 00000000..c63e9f22 --- /dev/null +++ b/mkdocs/tests/gh_deploy_tests.py @@ -0,0 +1,100 @@ +from __future__ import unicode_literals + +import unittest +import mock + +from mkdocs import gh_deploy +from mkdocs.config import load_config + + +class TestGitHubDeploy(unittest.TestCase): + + @mock.patch('subprocess.Popen') + def test_is_cwd_git_repo(self, mock_popeno): + + mock_popeno().wait.return_value = 0 + + self.assertTrue(gh_deploy._is_cwd_git_repo()) + + @mock.patch('subprocess.Popen') + def test_is_cwd_not_git_repo(self, mock_popeno): + + mock_popeno().wait.return_value = 1 + + self.assertFalse(gh_deploy._is_cwd_git_repo()) + + @mock.patch('subprocess.Popen') + def test_get_current_sha(self, mock_popeno): + + mock_popeno().communicate.return_value = (b'6d98394\n', b'') + + self.assertEqual(gh_deploy._get_current_sha(), u'6d98394') + + @mock.patch('subprocess.Popen') + def test_get_remote_url_ssh(self, mock_popeno): + + mock_popeno().communicate.return_value = ( + b'git@github.com:mkdocs/mkdocs.git\n', + b'' + ) + + expected = (u'git@', u'mkdocs/mkdocs.git') + self.assertEqual(expected, gh_deploy._get_remote_url('origin')) + + @mock.patch('subprocess.Popen') + def test_get_remote_url_http(self, mock_popeno): + + mock_popeno().communicate.return_value = ( + b'https://github.com/mkdocs/mkdocs.git\n', + b'' + ) + + expected = (u'https://', u'mkdocs/mkdocs.git') + self.assertEqual(expected, gh_deploy._get_remote_url('origin')) + + @mock.patch('subprocess.Popen') + def test_get_remote_url_enterprise(self, mock_popeno): + + mock_popeno().communicate.return_value = ( + b'https://notgh.com/mkdocs/mkdocs.git\n', + b'' + ) + + expected = (None, None) + self.assertEqual(expected, gh_deploy._get_remote_url('origin')) + + @mock.patch('mkdocs.gh_deploy._is_cwd_git_repo', return_value=True) + @mock.patch('mkdocs.gh_deploy._get_current_sha', return_value='shashas') + @mock.patch('mkdocs.gh_deploy._get_remote_url', return_value=(None, None)) + @mock.patch('mkdocs.gh_deploy.ghp_import.ghp_import') + def test_deploy(self, mock_import, get_remote, get_sha, is_repo): + + config = load_config( + remote_branch='test', + ) + gh_deploy.gh_deploy(config) + + @mock.patch('mkdocs.gh_deploy._is_cwd_git_repo', return_value=True) + @mock.patch('mkdocs.gh_deploy._get_current_sha', return_value='shashas') + @mock.patch('mkdocs.gh_deploy._get_remote_url', return_value=(None, None)) + @mock.patch('mkdocs.gh_deploy.ghp_import.ghp_import') + @mock.patch('os.path.isfile', return_value=False) + def test_deploy_no_cname(self, mock_isfile, mock_import, get_remote, + get_sha, is_repo): + + config = load_config( + remote_branch='test', + ) + gh_deploy.gh_deploy(config) + + @mock.patch('mkdocs.gh_deploy._is_cwd_git_repo', return_value=True) + @mock.patch('mkdocs.gh_deploy._get_current_sha', return_value='shashas') + @mock.patch('mkdocs.gh_deploy._get_remote_url', return_value=( + u'git@', u'mkdocs/mkdocs.git')) + @mock.patch('mkdocs.gh_deploy.ghp_import.ghp_import') + def test_deploy_hostname(self, mock_import, get_remote, get_sha, is_repo): + + config = load_config( + remote_branch='test', + ) + gh_deploy.gh_deploy(config) diff --git a/mkdocs/tests/utils/__init__.py b/mkdocs/tests/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mkdocs/tests/utils/ghp_import_tests.py b/mkdocs/tests/utils/ghp_import_tests.py new file mode 100644 index 00000000..4b95a3bc --- /dev/null +++ b/mkdocs/tests/utils/ghp_import_tests.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# coding: utf-8 + +from __future__ import unicode_literals + +import mock +import os +import subprocess +import tempfile +import unittest +import shutil + +from mkdocs.utils import ghp_import + + +class UtilsTests(unittest.TestCase): + + @mock.patch('subprocess.call', auto_spec=True) + @mock.patch('subprocess.Popen', auto_spec=True) + def test_try_rebase(self, mock_popen, mock_call): + + popen = mock.Mock() + mock_popen.return_value = popen + popen.communicate.return_value = ( + '4c82346e4b1b816be89dd709d35a6b169aa3df61\n', '') + popen.wait.return_value = 0 + + ghp_import.try_rebase('origin', 'gh-pages') + + mock_popen.assert_called_once_with( + ['git', 'rev-list', '--max-count=1', 'origin/gh-pages'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + mock_call.assert_called_once_with( + ['git', 'update-ref', 'refs/heads/gh-pages', + '4c82346e4b1b816be89dd709d35a6b169aa3df61']) + + @mock.patch('subprocess.Popen', auto_spec=True) + def test_get_prev_commit(self, mock_popen): + + popen = mock.Mock() + mock_popen.return_value = popen + popen.communicate.return_value = ( + b'4c82346e4b1b816be89dd709d35a6b169aa3df61\n', '') + popen.wait.return_value = 0 + + result = ghp_import.get_prev_commit('test-branch') + + self.assertEqual(result, u'4c82346e4b1b816be89dd709d35a6b169aa3df61') + mock_popen.assert_called_once_with( + ['git', 'rev-list', '--max-count=1', 'test-branch', '--'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + @mock.patch('subprocess.Popen', auto_spec=True) + def test_get_config(self, mock_popen): + + popen = mock.Mock() + mock_popen.return_value = popen + popen.communicate.return_value = ( + b'Dougal Matthews\n', '') + + result = ghp_import.get_config('user.name') + + self.assertEqual(result, u'Dougal Matthews') + mock_popen.assert_called_once_with( + ['git', 'config', 'user.name'], + stdout=subprocess.PIPE, stdin=subprocess.PIPE) + + @mock.patch('mkdocs.utils.ghp_import.get_prev_commit') + @mock.patch('mkdocs.utils.ghp_import.get_config') + def test_start_commit(self, mock_get_config, mock_get_prev_commit): + + pipe = mock.Mock() + mock_get_config.side_effect = ['username', 'email'] + mock_get_prev_commit.return_value = 'SHA' + + ghp_import.start_commit(pipe, 'test-branch', 'test-message') + + mock_get_prev_commit.assert_called_once_with('test-branch') + self.assertEqual(pipe.stdin.write.call_count, 5) + + @mock.patch('mkdocs.utils.ghp_import.try_rebase', return_value=True) + @mock.patch('mkdocs.utils.ghp_import.get_prev_commit', return_value='sha') + @mock.patch('mkdocs.utils.ghp_import.get_config', return_value='config') + @mock.patch('subprocess.call', auto_spec=True) + @mock.patch('subprocess.Popen', auto_spec=True) + def test_ghp_import(self, mock_popen, mock_call, mock_get_config, + mock_get_prev_commit, mock_try_rebase): + + directory = tempfile.mkdtemp() + open(os.path.join(directory, 'file'), 'a').close() + + try: + popen = mock.Mock() + mock_popen.return_value = popen + popen.communicate.return_value = ('', '') + popen.wait.return_value = 0 + + ghp_import.ghp_import(directory, "test message", + remote='fake-remote-name', + branch='fake-branch-name') + + self.assertEqual(mock_popen.call_count, 2) + self.assertEqual(mock_call.call_count, 0) + finally: + shutil.rmtree(directory) diff --git a/mkdocs/tests/utils_tests.py b/mkdocs/tests/utils/utils_tests.py similarity index 100% rename from mkdocs/tests/utils_tests.py rename to mkdocs/tests/utils/utils_tests.py diff --git a/mkdocs/utils.py b/mkdocs/utils/__init__.py similarity index 99% rename from mkdocs/utils.py rename to mkdocs/utils/__init__.py index 951ef2e4..13877394 100644 --- a/mkdocs/utils.py +++ b/mkdocs/utils/__init__.py @@ -338,7 +338,7 @@ def convert_markdown(markdown_source, extensions=None, extension_configs=None): def get_theme_names(): """Return a list containing all the names of all the builtin themes.""" - return os.listdir(os.path.join(os.path.dirname(__file__), 'themes')) + return os.listdir(os.path.join(os.path.dirname(__file__), '..', 'themes')) def filename_to_title(filename): diff --git a/mkdocs/utils/ghp_import.py b/mkdocs/utils/ghp_import.py new file mode 100644 index 00000000..c7cc85c0 --- /dev/null +++ b/mkdocs/utils/ghp_import.py @@ -0,0 +1,173 @@ +#! /usr/bin/env python +# +# This file is part of the ghp-import package released under +# the Tumbolia Public License. + +# Tumbolia Public License + +# Copyright 2013, Paul Davis + +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice and this +# notice are preserved. + +# TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + +# 0. opan saurce LOL + +from __future__ import unicode_literals + +import errno +import logging +import os +import subprocess as sp +import sys +import time +import unicodedata + +log = logging.getLogger(__name__) + + +if sys.version_info[0] == 3: + def enc(text): + if isinstance(text, bytes): + return text + return text.encode() + + def dec(text): + if isinstance(text, bytes): + return text.decode('utf-8') + return text + + def write(pipe, data): + try: + pipe.stdin.write(data) + except IOError as e: + if e.errno != errno.EPIPE: + raise +else: + def enc(text): + if isinstance(text, unicode): + return text.encode('utf-8') + return text + + def dec(text): + if isinstance(text, unicode): + return text + return text.decode('utf-8') + + def write(pipe, data): + pipe.stdin.write(data) + + +def normalize_path(path): + # Fix unicode pathnames on OS X + # See: http://stackoverflow.com/a/5582439/44289 + if sys.platform == "darwin": + return unicodedata.normalize("NFKC", dec(path)) + return path + + +def try_rebase(remote, branch): + cmd = ['git', 'rev-list', '--max-count=1', '%s/%s' % (remote, branch)] + p = sp.Popen(cmd, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE) + (rev, _) = p.communicate() + if p.wait() != 0: + return True + cmd = ['git', 'update-ref', 'refs/heads/%s' % branch, rev.strip()] + if sp.call(cmd) != 0: + return False + return True + + +def get_config(key): + p = sp.Popen(['git', 'config', key], stdin=sp.PIPE, stdout=sp.PIPE) + (value, _) = p.communicate() + return value.decode('utf-8').strip() + + +def get_prev_commit(branch): + cmd = ['git', 'rev-list', '--max-count=1', branch, '--'] + p = sp.Popen(cmd, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE) + (rev, _) = p.communicate() + if p.wait() != 0: + return None + return rev.decode('utf-8').strip() + + +def mk_when(timestamp=None): + if timestamp is None: + timestamp = int(time.time()) + currtz = "%+05d" % (-1 * time.timezone / 36) # / 3600 * 100 + return "%s %s" % (timestamp, currtz) + + +def start_commit(pipe, branch, message): + uname = dec(get_config("user.name")) + email = dec(get_config("user.email")) + write(pipe, enc('commit refs/heads/%s\n' % branch)) + write(pipe, enc('committer %s <%s> %s\n' % (uname, email, mk_when()))) + write(pipe, enc('data %d\n%s\n' % (len(message), message))) + head = get_prev_commit(branch) + if head: + write(pipe, enc('from %s\n' % head)) + write(pipe, enc('deleteall\n')) + + +def add_file(pipe, srcpath, tgtpath): + with open(srcpath, "rb") as handle: + if os.access(srcpath, os.X_OK): + write(pipe, enc('M 100755 inline %s\n' % tgtpath)) + else: + write(pipe, enc('M 100644 inline %s\n' % tgtpath)) + data = handle.read() + write(pipe, enc('data %d\n' % len(data))) + write(pipe, enc(data)) + write(pipe, enc('\n')) + + +def add_nojekyll(pipe): + write(pipe, enc('M 100644 inline .nojekyll\n')) + write(pipe, enc('data 0\n')) + write(pipe, enc('\n')) + + +def gitpath(fname): + norm = os.path.normpath(fname) + return "/".join(norm.split(os.path.sep)) + + +def run_import(srcdir, branch, message, nojekyll): + cmd = ['git', 'fast-import', '--date-format=raw', '--quiet'] + kwargs = {"stdin": sp.PIPE} + if sys.version_info >= (3, 2, 0): + kwargs["universal_newlines"] = False + pipe = sp.Popen(cmd, **kwargs) + start_commit(pipe, branch, message) + for path, _, fnames in os.walk(srcdir): + for fn in fnames: + fpath = os.path.join(path, fn) + fpath = normalize_path(fpath) + gpath = gitpath(os.path.relpath(fpath, start=srcdir)) + add_file(pipe, fpath, gpath) + if nojekyll: + add_nojekyll(pipe) + write(pipe, enc('\n')) + pipe.stdin.close() + if pipe.wait() != 0: + sys.stdout.write(enc("Failed to process commit.\n")) + + +def ghp_import(directory, message, remote='origin', branch='gh-pages'): + + if not try_rebase(remote, branch): + log.error("Failed to rebase %s branch.", branch) + + nojekyll = True + + run_import(directory, branch, message, nojekyll) + + proc = sp.Popen(['git', 'push', remote, branch], + stdout=sp.PIPE, stderr=sp.PIPE) + proc.communicate() + return proc.wait() == 0 diff --git a/requirements/project.txt b/requirements/project.txt index 15b16e28..a07047c6 100644 --- a/requirements/project.txt +++ b/requirements/project.txt @@ -1,5 +1,4 @@ click>=4.0 -ghp-import>=0.4.1 Jinja2>=2.7.1 livereload>=2.3.2 Markdown>=2.5 diff --git a/setup.py b/setup.py index 53ed104a..c2020ca0 100755 --- a/setup.py +++ b/setup.py @@ -59,7 +59,6 @@ setup( package_data=get_package_data("mkdocs"), install_requires=[ 'click>=4.0', - 'ghp-import>=0.4.1', 'Jinja2>=2.7.1', 'livereload>=2.3.2', 'Markdown>=2.3.1,<2.5' if PY26 else 'Markdown>=2.3.1',