mirror of
https://github.com/mkdocs/mkdocs.git
synced 2026-03-27 09:58:31 +07:00
Move code to external mkdocs-get-deps dependency (#3477)
The "get-deps" command will still be provided inside MkDocs like before (and is not deprecated at all) but is now implemented in a separate repository and can be used directly from there as well. This separation of code was done because obtaining just the "get-deps" part with very few dependencies can be useful: one can run it in a main environment but then install actual MkDocs with dependencies in a virtual environment.
This commit is contained in:
Oleh Prypin
@@ -327,19 +327,26 @@ def gh_deploy_command(
|
||||
@click.option(
|
||||
'-p',
|
||||
'--projects-file',
|
||||
default='https://raw.githubusercontent.com/mkdocs/catalog/main/projects.yaml',
|
||||
default=None,
|
||||
help=projects_file_help,
|
||||
show_default=True,
|
||||
)
|
||||
def get_deps_command(config_file, projects_file):
|
||||
"""Show required PyPI packages inferred from plugins in mkdocs.yml."""
|
||||
from mkdocs.commands import get_deps
|
||||
from mkdocs_get_deps import get_deps, get_projects_file
|
||||
|
||||
from mkdocs.config.base import _open_config_file
|
||||
|
||||
warning_counter = utils.CountHandler()
|
||||
warning_counter.setLevel(logging.WARNING)
|
||||
logging.getLogger('mkdocs').addHandler(warning_counter)
|
||||
|
||||
get_deps.get_deps(projects_file_url=projects_file, config_file_path=config_file)
|
||||
with get_projects_file(projects_file) as p:
|
||||
with _open_config_file(config_file) as f:
|
||||
deps = get_deps(config_file=f, projects_file=p)
|
||||
|
||||
for dep in deps:
|
||||
print(dep) # noqa: T201
|
||||
|
||||
if warning_counter.get_counts():
|
||||
sys.exit(1)
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import datetime
|
||||
import functools
|
||||
import logging
|
||||
import sys
|
||||
from typing import Mapping, Sequence
|
||||
|
||||
if sys.version_info >= (3, 10):
|
||||
from importlib.metadata import EntryPoint, entry_points
|
||||
else:
|
||||
from importlib_metadata import EntryPoint, entry_points
|
||||
|
||||
import yaml
|
||||
|
||||
from mkdocs import utils
|
||||
from mkdocs.config.base import _open_config_file
|
||||
from mkdocs.utils.cache import download_and_cache_url
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Note: do not rely on functions in this module, it is not public API.
|
||||
|
||||
|
||||
class YamlLoader(yaml.SafeLoader):
|
||||
pass
|
||||
|
||||
|
||||
# Prevent errors from trying to access external modules which may not be installed yet.
|
||||
YamlLoader.add_constructor("!ENV", lambda loader, node: None) # type: ignore
|
||||
YamlLoader.add_constructor("!relative", lambda loader, node: None) # type: ignore
|
||||
YamlLoader.add_multi_constructor(
|
||||
"tag:yaml.org,2002:python/name:", lambda loader, suffix, node: None
|
||||
)
|
||||
YamlLoader.add_multi_constructor(
|
||||
"tag:yaml.org,2002:python/object/apply:", lambda loader, suffix, node: None
|
||||
)
|
||||
|
||||
NotFound = ()
|
||||
|
||||
|
||||
def dig(cfg, keys: str):
|
||||
"""
|
||||
Receives a string such as 'foo.bar' and returns `cfg['foo']['bar']`, or `NotFound`.
|
||||
|
||||
A list of single-item dicts gets converted to a flat dict. This is intended for `plugins` config.
|
||||
"""
|
||||
key, _, rest = keys.partition('.')
|
||||
try:
|
||||
cfg = cfg[key]
|
||||
except (KeyError, TypeError):
|
||||
return NotFound
|
||||
if isinstance(cfg, list):
|
||||
orig_cfg = cfg
|
||||
cfg = {}
|
||||
for item in reversed(orig_cfg):
|
||||
if isinstance(item, dict) and len(item) == 1:
|
||||
cfg.update(item)
|
||||
elif isinstance(item, str):
|
||||
cfg[item] = {}
|
||||
if not rest:
|
||||
return cfg
|
||||
return dig(cfg, rest)
|
||||
|
||||
|
||||
def strings(obj) -> Sequence[str]:
|
||||
if isinstance(obj, str):
|
||||
return (obj,)
|
||||
else:
|
||||
return tuple(obj)
|
||||
|
||||
|
||||
@functools.lru_cache
|
||||
def _entry_points(group: str) -> Mapping[str, EntryPoint]:
|
||||
eps = {ep.name: ep for ep in entry_points(group=group)}
|
||||
log.debug(f"Available '{group}' entry points: {sorted(eps)}")
|
||||
return eps
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class PluginKind:
|
||||
projects_key: str
|
||||
entry_points_key: str
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.projects_key.rpartition('_')[-1]
|
||||
|
||||
|
||||
def get_deps(projects_file_url: str, config_file_path: str | None = None) -> None:
|
||||
"""
|
||||
Print PyPI package dependencies inferred from a mkdocs.yml file based on a reverse mapping of known projects.
|
||||
|
||||
Args:
|
||||
projects_file_url: URL or local path of the registry file that declares all known MkDocs-related projects.
|
||||
The file is in YAML format and contains `projects: [{mkdocs_theme:, mkdocs_plugin:, markdown_extension:}]
|
||||
config_file_path: Non-default path to mkdocs.yml.
|
||||
"""
|
||||
with _open_config_file(config_file_path) as f:
|
||||
cfg = utils.yaml_load(f, loader=YamlLoader) # type: ignore
|
||||
|
||||
packages_to_install = set()
|
||||
|
||||
if all(c not in cfg for c in ('site_name', 'theme', 'plugins', 'markdown_extensions')):
|
||||
log.warning("The passed config file doesn't seem to be a mkdocs.yml config file")
|
||||
else:
|
||||
if dig(cfg, 'theme.locale') not in (NotFound, 'en'):
|
||||
packages_to_install.add('mkdocs[i18n]')
|
||||
else:
|
||||
packages_to_install.add('mkdocs')
|
||||
|
||||
try:
|
||||
theme = cfg['theme']['name']
|
||||
except (KeyError, TypeError):
|
||||
theme = cfg.get('theme')
|
||||
themes = {theme} if theme else set()
|
||||
|
||||
plugins = set(strings(dig(cfg, 'plugins')))
|
||||
extensions = set(strings(dig(cfg, 'markdown_extensions')))
|
||||
|
||||
wanted_plugins = (
|
||||
(PluginKind('mkdocs_theme', 'mkdocs.themes'), themes - {'mkdocs', 'readthedocs'}),
|
||||
(PluginKind('mkdocs_plugin', 'mkdocs.plugins'), plugins - {'search'}),
|
||||
(PluginKind('markdown_extension', 'markdown.extensions'), extensions),
|
||||
)
|
||||
for kind, wanted in wanted_plugins:
|
||||
log.debug(f'Wanted {kind}s: {sorted(wanted)}')
|
||||
|
||||
content = download_and_cache_url(projects_file_url, datetime.timedelta(days=7))
|
||||
projects = yaml.safe_load(content)['projects']
|
||||
|
||||
for project in projects:
|
||||
for kind, wanted in wanted_plugins:
|
||||
available = strings(project.get(kind.projects_key, ()))
|
||||
for entry_name in available:
|
||||
if ( # Also check theme-namespaced plugin names against the current theme.
|
||||
'/' in entry_name
|
||||
and theme is not None
|
||||
and kind.projects_key == 'mkdocs_plugin'
|
||||
and entry_name.startswith(f'{theme}/')
|
||||
and entry_name[len(theme) + 1 :] in wanted
|
||||
and entry_name not in wanted
|
||||
):
|
||||
entry_name = entry_name[len(theme) + 1 :]
|
||||
if entry_name in wanted:
|
||||
if 'pypi_id' in project:
|
||||
install_name = project['pypi_id']
|
||||
elif 'github_id' in project:
|
||||
install_name = 'git+https://github.com/{github_id}'.format_map(project)
|
||||
else:
|
||||
log.error(
|
||||
f"Can't find how to install {kind} '{entry_name}' although it was identified as {project}"
|
||||
)
|
||||
continue
|
||||
packages_to_install.add(install_name)
|
||||
for extra_key, extra_pkgs in project.get('extra_dependencies', {}).items():
|
||||
if dig(cfg, extra_key) is not NotFound:
|
||||
packages_to_install.update(strings(extra_pkgs))
|
||||
|
||||
wanted.remove(entry_name)
|
||||
|
||||
for kind, wanted in wanted_plugins:
|
||||
for entry_name in sorted(wanted):
|
||||
dist_name = None
|
||||
ep = _entry_points(kind.entry_points_key).get(entry_name)
|
||||
if ep is not None and ep.dist is not None:
|
||||
dist_name = ep.dist.name
|
||||
if dist_name not in ('mkdocs', 'Markdown'):
|
||||
warning = f"{str(kind).capitalize()} '{entry_name}' is not provided by any registered project"
|
||||
if ep is not None:
|
||||
warning += " but is installed locally"
|
||||
if dist_name:
|
||||
warning += f" from '{dist_name}'"
|
||||
log.info(warning)
|
||||
else:
|
||||
log.warning(warning)
|
||||
|
||||
for pkg in sorted(packages_to_install):
|
||||
print(pkg) # noqa: T201
|
||||
@@ -1,172 +0,0 @@
|
||||
import contextlib
|
||||
import io
|
||||
import os
|
||||
import textwrap
|
||||
import unittest
|
||||
|
||||
from mkdocs.commands.get_deps import get_deps
|
||||
from mkdocs.tests.base import tempdir
|
||||
|
||||
_projects_file_path = os.path.join(
|
||||
os.path.abspath(os.path.dirname(__file__)), 'integration', 'projects.yaml'
|
||||
)
|
||||
|
||||
|
||||
class TestGetDeps(unittest.TestCase):
|
||||
@contextlib.contextmanager
|
||||
def _assert_logs(self, expected):
|
||||
with self.assertLogs('mkdocs.commands.get_deps') as cm:
|
||||
yield
|
||||
msgs = [f'{r.levelname}:{r.message}' for r in cm.records]
|
||||
self.assertEqual('\n'.join(msgs), textwrap.dedent(expected).strip('\n'))
|
||||
|
||||
@tempdir()
|
||||
def _test_get_deps(self, tempdir, yml, expected):
|
||||
if yml:
|
||||
yml = 'site_name: Test\n' + textwrap.dedent(yml)
|
||||
projects_path = os.path.join(tempdir, 'projects.yaml')
|
||||
with open(projects_path, 'w', encoding='utf-8') as f:
|
||||
f.write(yml)
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
get_deps(_projects_file_path, projects_path)
|
||||
self.assertEqual(buf.getvalue().split(), expected)
|
||||
|
||||
def test_empty_config(self):
|
||||
expected_logs = "WARNING:The passed config file doesn't seem to be a mkdocs.yml config file"
|
||||
with self._assert_logs(expected_logs):
|
||||
self._test_get_deps('', [])
|
||||
|
||||
def test_just_search(self):
|
||||
cfg = '''
|
||||
plugins: [search]
|
||||
'''
|
||||
self._test_get_deps(cfg, ['mkdocs'])
|
||||
|
||||
def test_mkdocs_config(self):
|
||||
cfg = '''
|
||||
site_name: MkDocs
|
||||
theme:
|
||||
name: mkdocs
|
||||
locale: en
|
||||
markdown_extensions:
|
||||
- toc:
|
||||
permalink:
|
||||
- attr_list
|
||||
- def_list
|
||||
- tables
|
||||
- pymdownx.highlight:
|
||||
use_pygments: false
|
||||
- pymdownx.snippets
|
||||
- pymdownx.superfences
|
||||
- callouts
|
||||
- mdx_gh_links:
|
||||
user: mkdocs
|
||||
repo: mkdocs
|
||||
- mkdocs-click
|
||||
plugins:
|
||||
- search
|
||||
- redirects:
|
||||
- autorefs
|
||||
- literate-nav:
|
||||
nav_file: README.md
|
||||
implicit_index: true
|
||||
- mkdocstrings:
|
||||
handlers:
|
||||
python:
|
||||
options:
|
||||
docstring_section_style: list
|
||||
'''
|
||||
self._test_get_deps(
|
||||
cfg,
|
||||
[
|
||||
'markdown-callouts',
|
||||
'mdx-gh-links',
|
||||
'mkdocs',
|
||||
'mkdocs-autorefs',
|
||||
'mkdocs-click',
|
||||
'mkdocs-literate-nav',
|
||||
'mkdocs-redirects',
|
||||
'mkdocstrings',
|
||||
'mkdocstrings-python',
|
||||
'pymdown-extensions',
|
||||
],
|
||||
)
|
||||
|
||||
def test_dict_keys_and_ignores_env(self):
|
||||
cfg = '''
|
||||
theme:
|
||||
name: material
|
||||
plugins:
|
||||
code-validator:
|
||||
enabled: !ENV [LINT, false]
|
||||
markdown_extensions:
|
||||
pymdownx.emoji:
|
||||
emoji_index: !!python/name:materialx.emoji.twemoji
|
||||
emoji_generator: !!python/name:materialx.emoji.to_svg
|
||||
'''
|
||||
self._test_get_deps(
|
||||
cfg, ['mkdocs', 'mkdocs-code-validator', 'mkdocs-material', 'pymdown-extensions']
|
||||
)
|
||||
|
||||
def test_theme_precedence(self):
|
||||
cfg = '''
|
||||
plugins:
|
||||
- tags
|
||||
theme: material
|
||||
'''
|
||||
self._test_get_deps(cfg, ['mkdocs', 'mkdocs-material'])
|
||||
|
||||
cfg = '''
|
||||
plugins:
|
||||
- material/tags
|
||||
'''
|
||||
self._test_get_deps(cfg, ['mkdocs', 'mkdocs-material'])
|
||||
|
||||
cfg = '''
|
||||
plugins:
|
||||
- tags
|
||||
'''
|
||||
self._test_get_deps(cfg, ['mkdocs', 'mkdocs-plugin-tags'])
|
||||
|
||||
def test_nonexistent(self):
|
||||
cfg = '''
|
||||
plugins:
|
||||
- taglttghhmdu
|
||||
- syyisjupkbpo
|
||||
- redirects
|
||||
theme: qndyakplooyh
|
||||
markdown_extensions:
|
||||
- saqdhyndpvpa
|
||||
'''
|
||||
expected_logs = """
|
||||
WARNING:Theme 'qndyakplooyh' is not provided by any registered project
|
||||
WARNING:Plugin 'syyisjupkbpo' is not provided by any registered project
|
||||
WARNING:Plugin 'taglttghhmdu' is not provided by any registered project
|
||||
WARNING:Extension 'saqdhyndpvpa' is not provided by any registered project
|
||||
"""
|
||||
with self._assert_logs(expected_logs):
|
||||
self._test_get_deps(cfg, ['mkdocs', 'mkdocs-redirects'])
|
||||
|
||||
def test_git_and_shadowed(self):
|
||||
cfg = '''
|
||||
theme: bootstrap4
|
||||
plugins: [blog]
|
||||
'''
|
||||
self._test_get_deps(
|
||||
cfg, ['git+https://github.com/andyoakley/mkdocs-blog', 'mkdocs', 'mkdocs-bootstrap4']
|
||||
)
|
||||
|
||||
def test_multi_theme(self):
|
||||
cfg = '''
|
||||
theme: minty
|
||||
'''
|
||||
self._test_get_deps(cfg, ['mkdocs', 'mkdocs-bootswatch'])
|
||||
|
||||
def test_with_locale(self):
|
||||
cfg = '''
|
||||
theme:
|
||||
name: mkdocs
|
||||
locale: uk
|
||||
'''
|
||||
self._test_get_deps(cfg, ['mkdocs[i18n]'])
|
||||
@@ -1,63 +1,36 @@
|
||||
import datetime
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Callable
|
||||
|
||||
import click
|
||||
import platformdirs
|
||||
import mkdocs_get_deps.cache
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
import mkdocs
|
||||
|
||||
|
||||
def download_url(url: str) -> bytes:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": f"mkdocs/{mkdocs.__version__}"})
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
return resp.read()
|
||||
|
||||
|
||||
def download_and_cache_url(
|
||||
url: str,
|
||||
cache_duration: datetime.timedelta,
|
||||
comment: bytes = b'# ',
|
||||
*,
|
||||
download: Callable[[str], bytes] = download_url,
|
||||
comment: bytes = b"# ",
|
||||
) -> bytes:
|
||||
"""
|
||||
Downloads a file from the URL, stores it under ~/.cache/, and returns its content.
|
||||
|
||||
If the URL is a local path, it is simply read and returned instead.
|
||||
|
||||
For tracking the age of the content, a prefix is inserted into the stored file, rather than relying on mtime.
|
||||
|
||||
Args:
|
||||
url: URL or local path of the file to use.
|
||||
cache_duration: how long to consider the URL content cached.
|
||||
url: URL to use.
|
||||
download: Callback that will accept the URL and actually perform the download.
|
||||
cache_duration: How long to consider the URL content cached.
|
||||
comment: The appropriate comment prefix for this file format.
|
||||
"""
|
||||
if urllib.parse.urlsplit(url).scheme not in ('http', 'https'):
|
||||
with open(url, 'rb') as f:
|
||||
return f.read()
|
||||
|
||||
directory = os.path.join(platformdirs.user_cache_dir('mkdocs'), 'mkdocs_url_cache')
|
||||
name_hash = hashlib.sha256(url.encode()).hexdigest()[:32]
|
||||
path = os.path.join(directory, name_hash + os.path.splitext(url)[1])
|
||||
|
||||
now = int(datetime.datetime.now(datetime.timezone.utc).timestamp())
|
||||
prefix = b'%s%s downloaded at timestamp ' % (comment, url.encode())
|
||||
# Check for cached file and try to return it
|
||||
if os.path.isfile(path):
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
line = f.readline()
|
||||
if line.startswith(prefix):
|
||||
line = line[len(prefix) :]
|
||||
timestamp = int(line)
|
||||
if datetime.timedelta(seconds=(now - timestamp)) <= cache_duration:
|
||||
log.debug(f"Using cached '{path}' for '{url}'")
|
||||
return f.read()
|
||||
except (OSError, ValueError) as e:
|
||||
log.debug(f'{type(e).__name__}: {e}')
|
||||
|
||||
# Download and cache the file
|
||||
log.debug(f"Downloading '{url}' to '{path}'")
|
||||
with urllib.request.urlopen(url) as resp:
|
||||
content = resp.read()
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
with click.open_file(path, 'wb', atomic=True) as f:
|
||||
f.write(b'%s%d\n' % (prefix, now))
|
||||
f.write(content)
|
||||
return content
|
||||
return mkdocs_get_deps.cache.download_and_cache_url(
|
||||
url=url, cache_duration=cache_duration, download=download, comment=comment
|
||||
)
|
||||
|
||||
@@ -45,7 +45,7 @@ dependencies = [
|
||||
"packaging >=20.5",
|
||||
"mergedeep >=1.3.4",
|
||||
"pathspec >=0.11.1",
|
||||
"platformdirs >=2.2.0",
|
||||
"mkdocs-get-deps >=0.2.0",
|
||||
"colorama >=0.4; platform_system == 'Windows'",
|
||||
]
|
||||
[project.optional-dependencies]
|
||||
@@ -66,7 +66,7 @@ min-versions = [
|
||||
"packaging ==20.5",
|
||||
"mergedeep ==1.3.4",
|
||||
"pathspec ==0.11.1",
|
||||
"platformdirs ==2.2.0",
|
||||
"mkdocs-get-deps ==0.2.0",
|
||||
"colorama ==0.4; platform_system == 'Windows'",
|
||||
"babel ==2.9.0",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user