New get-deps command: infer PyPI depedencies from mkdocs.yml

The user story is that the following command should let you "just build" any MkDocs site:

    pip install $(mkdocs get-deps) && mkdocs build

This cross-references 2 files:

* mkdocs.yml - `theme`, `plugins`, `markdown_extensions`
* projects.yaml - a registry of all popular MkDocs-related projects and which entry points they provide - downloaded on the fly

-and prints the names of Python packages from PyPI that need to be installed to build the current MkDocs project

mkdocs/__main__.py

@@ -136,6 +136,9 @@ watch_theme_help = (
 )
 shell_help = "Use the shell when invoking Git."
 watch_help = "A directory or file to watch for live reloading. Can be supplied multiple times."
+projects_file_help = (
+    "URL or local path of the registry file that declares all known MkDocs-related projects."
+)
 
 
 def add_options(*opts):
@@ -201,7 +204,7 @@ PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"
 PKG_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
-@click.group(context_settings={'help_option_names': ['-h', '--help']})
+@click.group(context_settings=dict(help_option_names=['-h', '--help'], max_content_width=120))
 @click.version_option(
     __version__,
     '-V',
@@ -287,6 +290,30 @@ def gh_deploy_command(
     )
 
 
+@cli.command(name="get-deps")
+@verbose_option
+@click.option('-f', '--config-file', type=click.File('rb'), help=config_help)
+@click.option(
+    '-p',
+    '--projects-file',
+    default='https://raw.githubusercontent.com/mkdocs/best-of-mkdocs/main/projects.yaml',
+    help=projects_file_help,
+    show_default=True,
+)
+def get_deps_command(config_file, projects_file):
+    """Show required PyPI packages inferred from plugins in mkdocs.yml"""
+    from mkdocs.commands import get_deps
+
+    warning_counter = utils.CountHandler()
+    warning_counter.setLevel(logging.WARNING)
+    logging.getLogger('mkdocs').addHandler(warning_counter)
+
+    get_deps.get_deps(projects_file_url=projects_file, config_file_path=config_file)
+
+    if warning_counter.get_counts():
+        sys.exit(1)
+
+
 @cli.command(name="new")
 @click.argument("project_directory")
 @common_options

mkdocs/commands/get_deps.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import dataclasses
+import datetime
+import functools
+import logging
+from typing import Iterator, Mapping, Optional
+
+import yaml
+
+from mkdocs import utils
+from mkdocs.config.base import _open_config_file
+from mkdocs.plugins import EntryPoint, entry_points
+from mkdocs.utils.cache import download_and_cache_url
+
+log = logging.getLogger(__name__)
+
+
+def _extract_names(cfg, key: str) -> Iterator[str]:
+    """Get names of plugins/extensions from the config - in either a list of dicts or a dict."""
+    try:
+        items = iter(cfg.get(key, ()))
+    except TypeError:
+        log.error(f"Invalid config entry '{key}'")
+    for item in items:
+        try:
+            if not isinstance(item, str):
+                [item] = item
+            yield item
+        except (ValueError, TypeError):
+            log.error(f"Invalid config entry '{key}': {item}")
+
+
+@functools.lru_cache()
+def _entry_points(group: str) -> Mapping[str, EntryPoint]:
+    eps = {ep.name: ep for ep in entry_points(group=group)}
+    log.debug(f"Available '{group}' entry points: {sorted(eps)}")
+    return eps
+
+
+@dataclasses.dataclass(frozen=True)
+class PluginKind:
+    projects_key: str
+    entry_points_key: str
+
+    def __str__(self) -> str:
+        return self.projects_key.rpartition('_')[-1]
+
+
+def get_deps(projects_file_url: str, config_file_path: Optional[str] = None) -> None:
+    """
+    Print PyPI package dependencies inferred from a mkdocs.yml file based on a reverse mapping of known projects.
+
+    Parameters:
+        projects_file_url: URL or local path of the registry file that declares all known MkDocs-related projects.
+            The file is in YAML format and contains `projects: [{mkdocs_theme:, mkdocs_plugin:, markdown_extension:}]
+        config_file_path: Non-default path to mkdocs.yml.
+    """
+    with _open_config_file(config_file_path) as f:
+        cfg = utils.yaml_load(f)
+
+    if all(c not in cfg for c in ('site_name', 'theme', 'plugins', 'markdown_extensions')):
+        log.warning("The passed config file doesn't seem to be a mkdocs.yml config file")
+
+    try:
+        theme = cfg['theme']['name']
+    except (KeyError, TypeError):
+        theme = cfg.get('theme')
+    themes = {theme} if theme else set()
+
+    plugins = set(_extract_names(cfg, 'plugins'))
+    extensions = set(_extract_names(cfg, 'markdown_extensions'))
+
+    wanted_plugins = (
+        (PluginKind('mkdocs_theme', 'mkdocs.themes'), themes - {'mkdocs', 'readthedocs'}),
+        (PluginKind('mkdocs_plugin', 'mkdocs.plugins'), plugins - {'search'}),
+        (PluginKind('markdown_extension', 'markdown.extensions'), extensions),
+    )
+    for kind, wanted in wanted_plugins:
+        log.debug(f'Wanted {kind}s: {sorted(wanted)}')
+
+    content = download_and_cache_url(projects_file_url, datetime.timedelta(days=7))
+    projects = yaml.safe_load(content)['projects']
+
+    packages_to_install = set()
+    for project in projects:
+        for kind, wanted in wanted_plugins:
+            available = project.get(kind.projects_key, ())
+            if isinstance(available, str):
+                available = (available,)
+            for entry_name in available:
+                if entry_name in wanted or (
+                    # Also check theme-namespaced plugin names against the current theme.
+                    '/' in entry_name
+                    and theme is not None
+                    and kind.projects_key == 'mkdocs_plugin'
+                    and entry_name.startswith(f'{theme}/')
+                    and entry_name[len(theme) + 1 :] in wanted
+                ):
+                    if 'pypi_id' in project:
+                        install_name = project['pypi_id']
+                    elif 'github_id' in project:
+                        install_name = 'git+https://github.com/{github_id}'.format_map(project)
+                    else:
+                        log.error(
+                            f"Can't find how to install {kind} '{entry_name}' although it was identified as {project}"
+                        )
+                        continue
+                    packages_to_install.add(install_name)
+                    wanted.remove(entry_name)
+
+    for kind, wanted in wanted_plugins:
+        for entry_name in sorted(wanted):
+            dist_name = None
+            ep = _entry_points(kind.entry_points_key).get(entry_name)
+            if ep is not None and ep.dist is not None:
+                dist_name = ep.dist.name
+            if dist_name not in ('mkdocs', 'Markdown'):
+                warning = f"{str(kind).capitalize()} '{entry_name}' is not provided by any registered project"
+                if ep is not None:
+                    warning += " but is installed locally"
+                    if dist_name:
+                        warning += f" from '{dist_name}'"
+                    log.info(warning)
+                else:
+                    log.warning(warning)
+
+    for pkg in sorted(packages_to_install):
+        print(pkg)

mkdocs/utils/cache.py

@@ -0,0 +1,64 @@
+import datetime
+import hashlib
+import logging
+import os
+import urllib.parse
+import urllib.request
+
+import click
+import platformdirs
+
+log = logging.getLogger(__name__)
+
+
+def download_and_cache_url(
+    url: str,
+    cache_duration: datetime.timedelta,
+    comment: bytes = b'# ',
+) -> bytes:
+    """
+    Downloads a file from the URL, stores it under ~/.cache/, and returns its content.
+
+    If the URL is a local path, it is simply read and returned instead.
+
+    For tracking the age of the content, a prefix is inserted into the stored file, rather than relying on mtime.
+
+    Parameters:
+        url: URL or local path of the file to use.
+        cache_duration: how long to consider the URL content cached.
+        comment: The appropriate comment prefix for this file format.
+    """
+
+    if urllib.parse.urlsplit(url).scheme not in ('http', 'https'):
+        with open(url, 'rb') as f:
+            return f.read()
+
+    directory = os.path.join(platformdirs.user_cache_dir('mkdocs'), 'mkdocs_url_cache')
+    name_hash = hashlib.sha256(url.encode()).hexdigest()[:32]
+    path = os.path.join(directory, name_hash + os.path.splitext(url)[1])
+
+    now = int(datetime.datetime.now(datetime.timezone.utc).timestamp())
+    prefix = b'%s%s downloaded at timestamp ' % (comment, url.encode())
+    # Check for cached file and try to return it
+    if os.path.isfile(path):
+        try:
+            with open(path, 'rb') as f:
+                line = f.readline()
+                if line.startswith(prefix):
+                    line = line[len(prefix) :]
+                    timestamp = int(line)
+                    if datetime.timedelta(seconds=(now - timestamp)) <= cache_duration:
+                        log.debug(f"Using cached '{path}' for '{url}'")
+                        return f.read()
+        except (IOError, ValueError) as e:
+            log.debug(f'{type(e).__name__}: {e}')
+
+    # Download and cache the file
+    log.debug(f"Downloading '{url}' to '{path}'")
+    with urllib.request.urlopen(url) as resp:
+        content = resp.read()
+    os.makedirs(directory, exist_ok=True)
+    with click.open_file(path, 'wb', atomic=True) as f:
+        f.write(b'%s%d\n' % (prefix, now))
+        f.write(content)
+    return content

pyproject.toml

@@ -44,6 +44,7 @@ dependencies = [
     "typing_extensions >=3.10; python_version < '3.8'",
     "packaging >=20.5",
     "mergedeep >=1.3.4",
+    "platformdirs >=2.2.0",
     "colorama >=0.4; platform_system == 'Windows'",
 ]
 [project.optional-dependencies]
@@ -63,6 +64,7 @@ min-versions = [
     "typing_extensions ==3.10; python_version < '3.8'",
     "packaging ==20.5",
     "mergedeep ==1.3.4",
+    "platformdirs ==2.2.0",
     "colorama ==0.4; platform_system == 'Windows'",
     "babel ==2.9.0",
 ]