diff --git a/mkdocs/__main__.py b/mkdocs/__main__.py index baf67adc..0df2069b 100644 --- a/mkdocs/__main__.py +++ b/mkdocs/__main__.py @@ -327,19 +327,26 @@ def gh_deploy_command( @click.option( '-p', '--projects-file', - default='https://raw.githubusercontent.com/mkdocs/catalog/main/projects.yaml', + default=None, help=projects_file_help, show_default=True, ) def get_deps_command(config_file, projects_file): """Show required PyPI packages inferred from plugins in mkdocs.yml.""" - from mkdocs.commands import get_deps + from mkdocs_get_deps import get_deps, get_projects_file + + from mkdocs.config.base import _open_config_file warning_counter = utils.CountHandler() warning_counter.setLevel(logging.WARNING) logging.getLogger('mkdocs').addHandler(warning_counter) - get_deps.get_deps(projects_file_url=projects_file, config_file_path=config_file) + with get_projects_file(projects_file) as p: + with _open_config_file(config_file) as f: + deps = get_deps(config_file=f, projects_file=p) + + for dep in deps: + print(dep) # noqa: T201 if warning_counter.get_counts(): sys.exit(1) diff --git a/mkdocs/commands/get_deps.py b/mkdocs/commands/get_deps.py deleted file mode 100644 index c2bd2520..00000000 --- a/mkdocs/commands/get_deps.py +++ /dev/null @@ -1,179 +0,0 @@ -from __future__ import annotations - -import dataclasses -import datetime -import functools -import logging -import sys -from typing import Mapping, Sequence - -if sys.version_info >= (3, 10): - from importlib.metadata import EntryPoint, entry_points -else: - from importlib_metadata import EntryPoint, entry_points - -import yaml - -from mkdocs import utils -from mkdocs.config.base import _open_config_file -from mkdocs.utils.cache import download_and_cache_url - -log = logging.getLogger(__name__) - -# Note: do not rely on functions in this module, it is not public API. - - -class YamlLoader(yaml.SafeLoader): - pass - - -# Prevent errors from trying to access external modules which may not be installed yet. -YamlLoader.add_constructor("!ENV", lambda loader, node: None) # type: ignore -YamlLoader.add_constructor("!relative", lambda loader, node: None) # type: ignore -YamlLoader.add_multi_constructor( - "tag:yaml.org,2002:python/name:", lambda loader, suffix, node: None -) -YamlLoader.add_multi_constructor( - "tag:yaml.org,2002:python/object/apply:", lambda loader, suffix, node: None -) - -NotFound = () - - -def dig(cfg, keys: str): - """ - Receives a string such as 'foo.bar' and returns `cfg['foo']['bar']`, or `NotFound`. - - A list of single-item dicts gets converted to a flat dict. This is intended for `plugins` config. - """ - key, _, rest = keys.partition('.') - try: - cfg = cfg[key] - except (KeyError, TypeError): - return NotFound - if isinstance(cfg, list): - orig_cfg = cfg - cfg = {} - for item in reversed(orig_cfg): - if isinstance(item, dict) and len(item) == 1: - cfg.update(item) - elif isinstance(item, str): - cfg[item] = {} - if not rest: - return cfg - return dig(cfg, rest) - - -def strings(obj) -> Sequence[str]: - if isinstance(obj, str): - return (obj,) - else: - return tuple(obj) - - -@functools.lru_cache -def _entry_points(group: str) -> Mapping[str, EntryPoint]: - eps = {ep.name: ep for ep in entry_points(group=group)} - log.debug(f"Available '{group}' entry points: {sorted(eps)}") - return eps - - -@dataclasses.dataclass(frozen=True) -class PluginKind: - projects_key: str - entry_points_key: str - - def __str__(self) -> str: - return self.projects_key.rpartition('_')[-1] - - -def get_deps(projects_file_url: str, config_file_path: str | None = None) -> None: - """ - Print PyPI package dependencies inferred from a mkdocs.yml file based on a reverse mapping of known projects. - - Args: - projects_file_url: URL or local path of the registry file that declares all known MkDocs-related projects. - The file is in YAML format and contains `projects: [{mkdocs_theme:, mkdocs_plugin:, markdown_extension:}] - config_file_path: Non-default path to mkdocs.yml. - """ - with _open_config_file(config_file_path) as f: - cfg = utils.yaml_load(f, loader=YamlLoader) # type: ignore - - packages_to_install = set() - - if all(c not in cfg for c in ('site_name', 'theme', 'plugins', 'markdown_extensions')): - log.warning("The passed config file doesn't seem to be a mkdocs.yml config file") - else: - if dig(cfg, 'theme.locale') not in (NotFound, 'en'): - packages_to_install.add('mkdocs[i18n]') - else: - packages_to_install.add('mkdocs') - - try: - theme = cfg['theme']['name'] - except (KeyError, TypeError): - theme = cfg.get('theme') - themes = {theme} if theme else set() - - plugins = set(strings(dig(cfg, 'plugins'))) - extensions = set(strings(dig(cfg, 'markdown_extensions'))) - - wanted_plugins = ( - (PluginKind('mkdocs_theme', 'mkdocs.themes'), themes - {'mkdocs', 'readthedocs'}), - (PluginKind('mkdocs_plugin', 'mkdocs.plugins'), plugins - {'search'}), - (PluginKind('markdown_extension', 'markdown.extensions'), extensions), - ) - for kind, wanted in wanted_plugins: - log.debug(f'Wanted {kind}s: {sorted(wanted)}') - - content = download_and_cache_url(projects_file_url, datetime.timedelta(days=7)) - projects = yaml.safe_load(content)['projects'] - - for project in projects: - for kind, wanted in wanted_plugins: - available = strings(project.get(kind.projects_key, ())) - for entry_name in available: - if ( # Also check theme-namespaced plugin names against the current theme. - '/' in entry_name - and theme is not None - and kind.projects_key == 'mkdocs_plugin' - and entry_name.startswith(f'{theme}/') - and entry_name[len(theme) + 1 :] in wanted - and entry_name not in wanted - ): - entry_name = entry_name[len(theme) + 1 :] - if entry_name in wanted: - if 'pypi_id' in project: - install_name = project['pypi_id'] - elif 'github_id' in project: - install_name = 'git+https://github.com/{github_id}'.format_map(project) - else: - log.error( - f"Can't find how to install {kind} '{entry_name}' although it was identified as {project}" - ) - continue - packages_to_install.add(install_name) - for extra_key, extra_pkgs in project.get('extra_dependencies', {}).items(): - if dig(cfg, extra_key) is not NotFound: - packages_to_install.update(strings(extra_pkgs)) - - wanted.remove(entry_name) - - for kind, wanted in wanted_plugins: - for entry_name in sorted(wanted): - dist_name = None - ep = _entry_points(kind.entry_points_key).get(entry_name) - if ep is not None and ep.dist is not None: - dist_name = ep.dist.name - if dist_name not in ('mkdocs', 'Markdown'): - warning = f"{str(kind).capitalize()} '{entry_name}' is not provided by any registered project" - if ep is not None: - warning += " but is installed locally" - if dist_name: - warning += f" from '{dist_name}'" - log.info(warning) - else: - log.warning(warning) - - for pkg in sorted(packages_to_install): - print(pkg) # noqa: T201 diff --git a/mkdocs/tests/get_deps_tests.py b/mkdocs/tests/get_deps_tests.py deleted file mode 100644 index 4fc4be68..00000000 --- a/mkdocs/tests/get_deps_tests.py +++ /dev/null @@ -1,172 +0,0 @@ -import contextlib -import io -import os -import textwrap -import unittest - -from mkdocs.commands.get_deps import get_deps -from mkdocs.tests.base import tempdir - -_projects_file_path = os.path.join( - os.path.abspath(os.path.dirname(__file__)), 'integration', 'projects.yaml' -) - - -class TestGetDeps(unittest.TestCase): - @contextlib.contextmanager - def _assert_logs(self, expected): - with self.assertLogs('mkdocs.commands.get_deps') as cm: - yield - msgs = [f'{r.levelname}:{r.message}' for r in cm.records] - self.assertEqual('\n'.join(msgs), textwrap.dedent(expected).strip('\n')) - - @tempdir() - def _test_get_deps(self, tempdir, yml, expected): - if yml: - yml = 'site_name: Test\n' + textwrap.dedent(yml) - projects_path = os.path.join(tempdir, 'projects.yaml') - with open(projects_path, 'w', encoding='utf-8') as f: - f.write(yml) - buf = io.StringIO() - with contextlib.redirect_stdout(buf): - get_deps(_projects_file_path, projects_path) - self.assertEqual(buf.getvalue().split(), expected) - - def test_empty_config(self): - expected_logs = "WARNING:The passed config file doesn't seem to be a mkdocs.yml config file" - with self._assert_logs(expected_logs): - self._test_get_deps('', []) - - def test_just_search(self): - cfg = ''' - plugins: [search] - ''' - self._test_get_deps(cfg, ['mkdocs']) - - def test_mkdocs_config(self): - cfg = ''' - site_name: MkDocs - theme: - name: mkdocs - locale: en - markdown_extensions: - - toc: - permalink:  - - attr_list - - def_list - - tables - - pymdownx.highlight: - use_pygments: false - - pymdownx.snippets - - pymdownx.superfences - - callouts - - mdx_gh_links: - user: mkdocs - repo: mkdocs - - mkdocs-click - plugins: - - search - - redirects: - - autorefs - - literate-nav: - nav_file: README.md - implicit_index: true - - mkdocstrings: - handlers: - python: - options: - docstring_section_style: list - ''' - self._test_get_deps( - cfg, - [ - 'markdown-callouts', - 'mdx-gh-links', - 'mkdocs', - 'mkdocs-autorefs', - 'mkdocs-click', - 'mkdocs-literate-nav', - 'mkdocs-redirects', - 'mkdocstrings', - 'mkdocstrings-python', - 'pymdown-extensions', - ], - ) - - def test_dict_keys_and_ignores_env(self): - cfg = ''' - theme: - name: material - plugins: - code-validator: - enabled: !ENV [LINT, false] - markdown_extensions: - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg - ''' - self._test_get_deps( - cfg, ['mkdocs', 'mkdocs-code-validator', 'mkdocs-material', 'pymdown-extensions'] - ) - - def test_theme_precedence(self): - cfg = ''' - plugins: - - tags - theme: material - ''' - self._test_get_deps(cfg, ['mkdocs', 'mkdocs-material']) - - cfg = ''' - plugins: - - material/tags - ''' - self._test_get_deps(cfg, ['mkdocs', 'mkdocs-material']) - - cfg = ''' - plugins: - - tags - ''' - self._test_get_deps(cfg, ['mkdocs', 'mkdocs-plugin-tags']) - - def test_nonexistent(self): - cfg = ''' - plugins: - - taglttghhmdu - - syyisjupkbpo - - redirects - theme: qndyakplooyh - markdown_extensions: - - saqdhyndpvpa - ''' - expected_logs = """ - WARNING:Theme 'qndyakplooyh' is not provided by any registered project - WARNING:Plugin 'syyisjupkbpo' is not provided by any registered project - WARNING:Plugin 'taglttghhmdu' is not provided by any registered project - WARNING:Extension 'saqdhyndpvpa' is not provided by any registered project - """ - with self._assert_logs(expected_logs): - self._test_get_deps(cfg, ['mkdocs', 'mkdocs-redirects']) - - def test_git_and_shadowed(self): - cfg = ''' - theme: bootstrap4 - plugins: [blog] - ''' - self._test_get_deps( - cfg, ['git+https://github.com/andyoakley/mkdocs-blog', 'mkdocs', 'mkdocs-bootstrap4'] - ) - - def test_multi_theme(self): - cfg = ''' - theme: minty - ''' - self._test_get_deps(cfg, ['mkdocs', 'mkdocs-bootswatch']) - - def test_with_locale(self): - cfg = ''' - theme: - name: mkdocs - locale: uk - ''' - self._test_get_deps(cfg, ['mkdocs[i18n]']) diff --git a/mkdocs/utils/cache.py b/mkdocs/utils/cache.py index 578bb087..1be94abd 100644 --- a/mkdocs/utils/cache.py +++ b/mkdocs/utils/cache.py @@ -1,63 +1,36 @@ import datetime -import hashlib -import logging -import os -import urllib.parse import urllib.request +from typing import Callable -import click -import platformdirs +import mkdocs_get_deps.cache -log = logging.getLogger(__name__) +import mkdocs + + +def download_url(url: str) -> bytes: + req = urllib.request.Request(url, headers={"User-Agent": f"mkdocs/{mkdocs.__version__}"}) + with urllib.request.urlopen(req) as resp: + return resp.read() def download_and_cache_url( url: str, cache_duration: datetime.timedelta, - comment: bytes = b'# ', + *, + download: Callable[[str], bytes] = download_url, + comment: bytes = b"# ", ) -> bytes: """ Downloads a file from the URL, stores it under ~/.cache/, and returns its content. - If the URL is a local path, it is simply read and returned instead. - For tracking the age of the content, a prefix is inserted into the stored file, rather than relying on mtime. Args: - url: URL or local path of the file to use. - cache_duration: how long to consider the URL content cached. + url: URL to use. + download: Callback that will accept the URL and actually perform the download. + cache_duration: How long to consider the URL content cached. comment: The appropriate comment prefix for this file format. """ - if urllib.parse.urlsplit(url).scheme not in ('http', 'https'): - with open(url, 'rb') as f: - return f.read() - - directory = os.path.join(platformdirs.user_cache_dir('mkdocs'), 'mkdocs_url_cache') - name_hash = hashlib.sha256(url.encode()).hexdigest()[:32] - path = os.path.join(directory, name_hash + os.path.splitext(url)[1]) - - now = int(datetime.datetime.now(datetime.timezone.utc).timestamp()) - prefix = b'%s%s downloaded at timestamp ' % (comment, url.encode()) - # Check for cached file and try to return it - if os.path.isfile(path): - try: - with open(path, 'rb') as f: - line = f.readline() - if line.startswith(prefix): - line = line[len(prefix) :] - timestamp = int(line) - if datetime.timedelta(seconds=(now - timestamp)) <= cache_duration: - log.debug(f"Using cached '{path}' for '{url}'") - return f.read() - except (OSError, ValueError) as e: - log.debug(f'{type(e).__name__}: {e}') - - # Download and cache the file - log.debug(f"Downloading '{url}' to '{path}'") - with urllib.request.urlopen(url) as resp: - content = resp.read() - os.makedirs(directory, exist_ok=True) - with click.open_file(path, 'wb', atomic=True) as f: - f.write(b'%s%d\n' % (prefix, now)) - f.write(content) - return content + return mkdocs_get_deps.cache.download_and_cache_url( + url=url, cache_duration=cache_duration, download=download, comment=comment + ) diff --git a/pyproject.toml b/pyproject.toml index 0a9a102d..423351e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ dependencies = [ "packaging >=20.5", "mergedeep >=1.3.4", "pathspec >=0.11.1", - "platformdirs >=2.2.0", + "mkdocs-get-deps >=0.2.0", "colorama >=0.4; platform_system == 'Windows'", ] [project.optional-dependencies] @@ -66,7 +66,7 @@ min-versions = [ "packaging ==20.5", "mergedeep ==1.3.4", "pathspec ==0.11.1", - "platformdirs ==2.2.0", + "mkdocs-get-deps ==0.2.0", "colorama ==0.4; platform_system == 'Windows'", "babel ==2.9.0", ]