From 36205e30dddc79a99c706dbfdce96e159fb2a2bc Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Sat, 29 Apr 2023 10:45:38 +0200 Subject: [PATCH 1/5] New `get-deps` command: infer PyPI depedencies from mkdocs.yml The user story is that the following command should let you "just build" any MkDocs site: pip install $(mkdocs get-deps) && mkdocs build This cross-references 2 files: * mkdocs.yml - `theme`, `plugins`, `markdown_extensions` * projects.yaml - a registry of all popular MkDocs-related projects and which entry points they provide - downloaded on the fly -and prints the names of Python packages from PyPI that need to be installed to build the current MkDocs project --- mkdocs/__main__.py | 29 +++++++- mkdocs/commands/get_deps.py | 129 ++++++++++++++++++++++++++++++++++++ mkdocs/utils/cache.py | 64 ++++++++++++++++++ pyproject.toml | 2 + 4 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 mkdocs/commands/get_deps.py create mode 100644 mkdocs/utils/cache.py diff --git a/mkdocs/__main__.py b/mkdocs/__main__.py index 84e7d3f0..9c8b40fc 100644 --- a/mkdocs/__main__.py +++ b/mkdocs/__main__.py @@ -136,6 +136,9 @@ watch_theme_help = ( ) shell_help = "Use the shell when invoking Git." watch_help = "A directory or file to watch for live reloading. Can be supplied multiple times." +projects_file_help = ( + "URL or local path of the registry file that declares all known MkDocs-related projects." +) def add_options(*opts): @@ -201,7 +204,7 @@ PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}" PKG_DIR = os.path.dirname(os.path.abspath(__file__)) -@click.group(context_settings={'help_option_names': ['-h', '--help']}) +@click.group(context_settings=dict(help_option_names=['-h', '--help'], max_content_width=120)) @click.version_option( __version__, '-V', @@ -287,6 +290,30 @@ def gh_deploy_command( ) +@cli.command(name="get-deps") +@verbose_option +@click.option('-f', '--config-file', type=click.File('rb'), help=config_help) +@click.option( + '-p', + '--projects-file', + default='https://raw.githubusercontent.com/mkdocs/best-of-mkdocs/main/projects.yaml', + help=projects_file_help, + show_default=True, +) +def get_deps_command(config_file, projects_file): + """Show required PyPI packages inferred from plugins in mkdocs.yml""" + from mkdocs.commands import get_deps + + warning_counter = utils.CountHandler() + warning_counter.setLevel(logging.WARNING) + logging.getLogger('mkdocs').addHandler(warning_counter) + + get_deps.get_deps(projects_file_url=projects_file, config_file_path=config_file) + + if warning_counter.get_counts(): + sys.exit(1) + + @cli.command(name="new") @click.argument("project_directory") @common_options diff --git a/mkdocs/commands/get_deps.py b/mkdocs/commands/get_deps.py new file mode 100644 index 00000000..d303ebda --- /dev/null +++ b/mkdocs/commands/get_deps.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import dataclasses +import datetime +import functools +import logging +from typing import Iterator, Mapping, Optional + +import yaml + +from mkdocs import utils +from mkdocs.config.base import _open_config_file +from mkdocs.plugins import EntryPoint, entry_points +from mkdocs.utils.cache import download_and_cache_url + +log = logging.getLogger(__name__) + + +def _extract_names(cfg, key: str) -> Iterator[str]: + """Get names of plugins/extensions from the config - in either a list of dicts or a dict.""" + try: + items = iter(cfg.get(key, ())) + except TypeError: + log.error(f"Invalid config entry '{key}'") + for item in items: + try: + if not isinstance(item, str): + [item] = item + yield item + except (ValueError, TypeError): + log.error(f"Invalid config entry '{key}': {item}") + + +@functools.lru_cache() +def _entry_points(group: str) -> Mapping[str, EntryPoint]: + eps = {ep.name: ep for ep in entry_points(group=group)} + log.debug(f"Available '{group}' entry points: {sorted(eps)}") + return eps + + +@dataclasses.dataclass(frozen=True) +class PluginKind: + projects_key: str + entry_points_key: str + + def __str__(self) -> str: + return self.projects_key.rpartition('_')[-1] + + +def get_deps(projects_file_url: str, config_file_path: Optional[str] = None) -> None: + """ + Print PyPI package dependencies inferred from a mkdocs.yml file based on a reverse mapping of known projects. + + Parameters: + projects_file_url: URL or local path of the registry file that declares all known MkDocs-related projects. + The file is in YAML format and contains `projects: [{mkdocs_theme:, mkdocs_plugin:, markdown_extension:}] + config_file_path: Non-default path to mkdocs.yml. + """ + with _open_config_file(config_file_path) as f: + cfg = utils.yaml_load(f) + + if all(c not in cfg for c in ('site_name', 'theme', 'plugins', 'markdown_extensions')): + log.warning("The passed config file doesn't seem to be a mkdocs.yml config file") + + try: + theme = cfg['theme']['name'] + except (KeyError, TypeError): + theme = cfg.get('theme') + themes = {theme} if theme else set() + + plugins = set(_extract_names(cfg, 'plugins')) + extensions = set(_extract_names(cfg, 'markdown_extensions')) + + wanted_plugins = ( + (PluginKind('mkdocs_theme', 'mkdocs.themes'), themes - {'mkdocs', 'readthedocs'}), + (PluginKind('mkdocs_plugin', 'mkdocs.plugins'), plugins - {'search'}), + (PluginKind('markdown_extension', 'markdown.extensions'), extensions), + ) + for kind, wanted in wanted_plugins: + log.debug(f'Wanted {kind}s: {sorted(wanted)}') + + content = download_and_cache_url(projects_file_url, datetime.timedelta(days=7)) + projects = yaml.safe_load(content)['projects'] + + packages_to_install = set() + for project in projects: + for kind, wanted in wanted_plugins: + available = project.get(kind.projects_key, ()) + if isinstance(available, str): + available = (available,) + for entry_name in available: + if entry_name in wanted or ( + # Also check theme-namespaced plugin names against the current theme. + '/' in entry_name + and theme is not None + and kind.projects_key == 'mkdocs_plugin' + and entry_name.startswith(f'{theme}/') + and entry_name[len(theme) + 1 :] in wanted + ): + if 'pypi_id' in project: + install_name = project['pypi_id'] + elif 'github_id' in project: + install_name = 'git+https://github.com/{github_id}'.format_map(project) + else: + log.error( + f"Can't find how to install {kind} '{entry_name}' although it was identified as {project}" + ) + continue + packages_to_install.add(install_name) + wanted.remove(entry_name) + + for kind, wanted in wanted_plugins: + for entry_name in sorted(wanted): + dist_name = None + ep = _entry_points(kind.entry_points_key).get(entry_name) + if ep is not None and ep.dist is not None: + dist_name = ep.dist.name + if dist_name not in ('mkdocs', 'Markdown'): + warning = f"{str(kind).capitalize()} '{entry_name}' is not provided by any registered project" + if ep is not None: + warning += " but is installed locally" + if dist_name: + warning += f" from '{dist_name}'" + log.info(warning) + else: + log.warning(warning) + + for pkg in sorted(packages_to_install): + print(pkg) diff --git a/mkdocs/utils/cache.py b/mkdocs/utils/cache.py new file mode 100644 index 00000000..fce05b7c --- /dev/null +++ b/mkdocs/utils/cache.py @@ -0,0 +1,64 @@ +import datetime +import hashlib +import logging +import os +import urllib.parse +import urllib.request + +import click +import platformdirs + +log = logging.getLogger(__name__) + + +def download_and_cache_url( + url: str, + cache_duration: datetime.timedelta, + comment: bytes = b'# ', +) -> bytes: + """ + Downloads a file from the URL, stores it under ~/.cache/, and returns its content. + + If the URL is a local path, it is simply read and returned instead. + + For tracking the age of the content, a prefix is inserted into the stored file, rather than relying on mtime. + + Parameters: + url: URL or local path of the file to use. + cache_duration: how long to consider the URL content cached. + comment: The appropriate comment prefix for this file format. + """ + + if urllib.parse.urlsplit(url).scheme not in ('http', 'https'): + with open(url, 'rb') as f: + return f.read() + + directory = os.path.join(platformdirs.user_cache_dir('mkdocs'), 'mkdocs_url_cache') + name_hash = hashlib.sha256(url.encode()).hexdigest()[:32] + path = os.path.join(directory, name_hash + os.path.splitext(url)[1]) + + now = int(datetime.datetime.now(datetime.timezone.utc).timestamp()) + prefix = b'%s%s downloaded at timestamp ' % (comment, url.encode()) + # Check for cached file and try to return it + if os.path.isfile(path): + try: + with open(path, 'rb') as f: + line = f.readline() + if line.startswith(prefix): + line = line[len(prefix) :] + timestamp = int(line) + if datetime.timedelta(seconds=(now - timestamp)) <= cache_duration: + log.debug(f"Using cached '{path}' for '{url}'") + return f.read() + except (IOError, ValueError) as e: + log.debug(f'{type(e).__name__}: {e}') + + # Download and cache the file + log.debug(f"Downloading '{url}' to '{path}'") + with urllib.request.urlopen(url) as resp: + content = resp.read() + os.makedirs(directory, exist_ok=True) + with click.open_file(path, 'wb', atomic=True) as f: + f.write(b'%s%d\n' % (prefix, now)) + f.write(content) + return content diff --git a/pyproject.toml b/pyproject.toml index 9c32c949..3ebcc192 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dependencies = [ "typing_extensions >=3.10; python_version < '3.8'", "packaging >=20.5", "mergedeep >=1.3.4", + "platformdirs >=2.2.0", "colorama >=0.4; platform_system == 'Windows'", ] [project.optional-dependencies] @@ -63,6 +64,7 @@ min-versions = [ "typing_extensions ==3.10; python_version < '3.8'", "packaging ==20.5", "mergedeep ==1.3.4", + "platformdirs ==2.2.0", "colorama ==0.4; platform_system == 'Windows'", "babel ==2.9.0", ] From 3363c615de2efe61f067d4ce3517da00a27ac8a6 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Sun, 4 Jun 2023 14:54:46 +0200 Subject: [PATCH 2/5] Update docs - "best-of-mkdocs" became "catalog" --- README.md | 4 ++-- docs/dev-guide/plugins.md | 11 ++++++++--- docs/dev-guide/themes.md | 6 +++--- docs/index.md | 2 +- docs/user-guide/choosing-your-theme.md | 4 ++-- docs/user-guide/configuration.md | 4 ++-- mkdocs/__main__.py | 2 +- 7 files changed, 19 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 4a7aaaff..0c294f0f 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Make sure to stick around to answer some questions as well! - [Official Documentation][mkdocs] - [Latest Release Notes][release-notes] -- [Best-of-MkDocs][best-of] (Third-party themes, recipes, plugins and more) +- [Catalog of third-party plugins, themes and recipes][catalog] ## Contributing to MkDocs @@ -72,7 +72,7 @@ discussion forums is expected to follow the [PyPA Code of Conduct]. [release-notes]: https://www.mkdocs.org/about/release-notes/ [Contributing Guide]: https://www.mkdocs.org/about/contributing/ [PyPA Code of Conduct]: https://www.pypa.io/en/latest/code-of-conduct/ -[best-of]: https://github.com/mkdocs/best-of-mkdocs +[catalog]: https://github.com/mkdocs/catalog ## License diff --git a/docs/dev-guide/plugins.md b/docs/dev-guide/plugins.md index 30f61cb2..613c89a5 100644 --- a/docs/dev-guide/plugins.md +++ b/docs/dev-guide/plugins.md @@ -16,8 +16,8 @@ pip install mkdocs-foo-plugin ``` Once a plugin has been successfully installed, it is ready to use. It just needs -to be [enabled](#using-plugins) in the configuration file. The [Best-of-MkDocs] -page has a large list of plugins that you can install and use. +to be [enabled](#using-plugins) in the configuration file. The [Catalog] +repository has a large ranked list of plugins that you can install and use. ## Using Plugins @@ -507,6 +507,10 @@ entry_points={ Note that registering a plugin does not activate it. The user still needs to tell MkDocs to use it via the config. +### Publishing a Plugin + +You should publish a package on [PyPI], then add it to the [Catalog] for discoverability. Plugins are strongly recommended to have a unique plugin name (entry point name) according to the catalog. + [BasePlugin]:#baseplugin [config]: ../user-guide/configuration.md#plugins [entry point]: #entry-point @@ -519,5 +523,6 @@ tell MkDocs to use it via the config. [post_template]: #on_post_template [static_templates]: ../user-guide/configuration.md#static_templates [Template Events]: #template-events -[Best-of-MkDocs]: https://github.com/mkdocs/best-of-mkdocs +[catalog]: https://github.com/mkdocs/catalog [on_build_error]: #on_build_error +[PyPI]: https://pypi.org/ diff --git a/docs/dev-guide/themes.md b/docs/dev-guide/themes.md index 504b98ea..32496f04 100644 --- a/docs/dev-guide/themes.md +++ b/docs/dev-guide/themes.md @@ -6,8 +6,8 @@ A guide to creating and distributing custom themes. NOTE: If you are looking for existing third party themes, they are listed in the -[community wiki] page and [Best-of-MkDocs]. If you want to share a theme you create, you -should list it there. +[community wiki] page and the [MkDocs project catalog][catalog]. If you want to +share a theme you create, you should list it there. When creating a new theme, you can either follow the steps in this guide to create one from scratch or you can download the `mkdocs-basic-theme` as a @@ -16,7 +16,7 @@ this base theme on [GitHub][basic theme]**. It contains detailed comments in the code to describe the different features and their usage. [community wiki]: https://github.com/mkdocs/mkdocs/wiki/MkDocs-Themes -[Best-of-MkDocs]: https://github.com/mkdocs/best-of-mkdocs#-theming +[catalog]: https://github.com/mkdocs/catalog#-theming [basic theme]: https://github.com/mkdocs/mkdocs-basic-theme ## Creating a custom theme diff --git a/docs/index.md b/docs/index.md index 634ee6b6..069d8222 100644 --- a/docs/index.md +++ b/docs/index.md @@ -33,7 +33,7 @@ configuration file. Start by reading the [introductory tutorial], then check the readthedocs, select one of the third-party themes (on the MkDocs Themes wiki page - as well as Best-of-MkDocs), + as well as the MkDocs Catalog), or build your own.

diff --git a/docs/user-guide/choosing-your-theme.md b/docs/user-guide/choosing-your-theme.md index 74c7f988..58a738c9 100644 --- a/docs/user-guide/choosing-your-theme.md +++ b/docs/user-guide/choosing-your-theme.md @@ -218,7 +218,7 @@ theme supports the following options: ## Third Party Themes -A list of third party themes can be found at the [community wiki] page and [Best-of-MkDocs]. If you have created your own, please add them there. +A list of third party themes can be found at the [community wiki] page and [the ranked catalog][catalog]. If you have created your own, please add them there. [third party themes]: #third-party-themes [theme]: configuration.md#theme @@ -229,5 +229,5 @@ A list of third party themes can be found at the [community wiki] page and [Best [upgrade-GA4]: https://support.google.com/analytics/answer/9744165?hl=en&ref_topic=9303319 [Read the Docs]: https://readthedocs.org/ [community wiki]: https://github.com/mkdocs/mkdocs/wiki/MkDocs-Themes -[Best-of-MkDocs]: https://github.com/mkdocs/best-of-mkdocs#-theming +[catalog]: https://github.com/mkdocs/catalog#-theming [localizing your theme]: localizing-your-theme.md diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index e084ee64..f4497024 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -575,7 +575,7 @@ This alternative syntax is required if you intend to override some options via > which are available out-of-the-box. For a list of configuration options > available for a given extension, see the documentation for that extension. > -> You may also install and use various third party extensions ([Python-Markdown wiki], [Best-of-MkDocs]). Consult +> You may also install and use various third party extensions ([Python-Markdown wiki], [MkDocs project catalog][catalog]). Consult > the documentation provided by those extensions for installation instructions > and available configuration options. @@ -964,7 +964,7 @@ path based options in the primary configuration file only. [smarty]: https://python-markdown.github.io/extensions/smarty/ [exts]: https://python-markdown.github.io/extensions/ [Python-Markdown wiki]: https://github.com/Python-Markdown/markdown/wiki/Third-Party-Extensions -[Best-of-MkDocs]: https://github.com/mkdocs/best-of-mkdocs +[catalog]: https://github.com/mkdocs/catalog [configuring pages and navigation]: writing-your-docs.md#configure-pages-and-navigation [theme_dir]: customizing-your-theme.md#using-the-theme_dir [choosing your theme]: choosing-your-theme.md diff --git a/mkdocs/__main__.py b/mkdocs/__main__.py index 9c8b40fc..677a2f78 100644 --- a/mkdocs/__main__.py +++ b/mkdocs/__main__.py @@ -296,7 +296,7 @@ def gh_deploy_command( @click.option( '-p', '--projects-file', - default='https://raw.githubusercontent.com/mkdocs/best-of-mkdocs/main/projects.yaml', + default='https://raw.githubusercontent.com/mkdocs/catalog/main/projects.yaml', help=projects_file_help, show_default=True, ) From e175070243ec3a531ea5688550f50c6e07f6e237 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Sun, 4 Jun 2023 23:22:56 +0200 Subject: [PATCH 3/5] Let plugins declare their own optional dependencies This is inferred from particular config keys The current known plugin in need of this is mkdocstrings (handlers) --- mkdocs/commands/get_deps.py | 57 +++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/mkdocs/commands/get_deps.py b/mkdocs/commands/get_deps.py index b84ae519..6cc07875 100644 --- a/mkdocs/commands/get_deps.py +++ b/mkdocs/commands/get_deps.py @@ -4,7 +4,7 @@ import dataclasses import datetime import functools import logging -from typing import Iterator, Mapping +from typing import Mapping, Sequence import yaml @@ -15,20 +15,39 @@ from mkdocs.utils.cache import download_and_cache_url log = logging.getLogger(__name__) +# Note: do not rely on functions in this module, it is not public API. -def _extract_names(cfg, key: str) -> Iterator[str]: - """Get names of plugins/extensions from the config - in either a list of dicts or a dict.""" +NotFound = () + + +def dig(cfg, keys: str): + """Receives a string such as 'foo.bar' and returns `cfg['foo']['bar']`, or `NotFound`. + + A list of single-item dicts gets converted to a flat dict. This is intended for `plugins` config. + """ + key, _, rest = keys.partition('.') try: - items = iter(cfg.get(key, ())) - except TypeError: - log.error(f"Invalid config entry '{key}'") - for item in items: - try: - if not isinstance(item, str): - [item] = item - yield item - except (ValueError, TypeError): - log.error(f"Invalid config entry '{key}': {item}") + cfg = cfg[key] + except (KeyError, TypeError): + return NotFound + if isinstance(cfg, list): + orig_cfg = cfg + cfg = {} + for item in reversed(orig_cfg): + if isinstance(item, dict) and len(item) == 1: + cfg.update(item) + elif isinstance(item, str): + cfg[item] = {} + if not rest: + return cfg + return dig(cfg, rest) + + +def strings(obj) -> Sequence[str]: + if isinstance(obj, str): + return (obj,) + else: + return tuple(obj) @functools.lru_cache() @@ -68,8 +87,8 @@ def get_deps(projects_file_url: str, config_file_path: str | None = None) -> Non theme = cfg.get('theme') themes = {theme} if theme else set() - plugins = set(_extract_names(cfg, 'plugins')) - extensions = set(_extract_names(cfg, 'markdown_extensions')) + plugins = set(strings(dig(cfg, 'plugins'))) + extensions = set(strings(dig(cfg, 'markdown_extensions'))) wanted_plugins = ( (PluginKind('mkdocs_theme', 'mkdocs.themes'), themes - {'mkdocs', 'readthedocs'}), @@ -85,9 +104,7 @@ def get_deps(projects_file_url: str, config_file_path: str | None = None) -> Non packages_to_install = set() for project in projects: for kind, wanted in wanted_plugins: - available = project.get(kind.projects_key, ()) - if isinstance(available, str): - available = (available,) + available = strings(project.get(kind.projects_key, ())) for entry_name in available: if entry_name in wanted or ( # Also check theme-namespaced plugin names against the current theme. @@ -107,6 +124,10 @@ def get_deps(projects_file_url: str, config_file_path: str | None = None) -> Non ) continue packages_to_install.add(install_name) + for extra_key, extra_pkgs in project.get('extra_dependencies', {}).items(): + if dig(cfg, extra_key) is not NotFound: + packages_to_install.update(strings(extra_pkgs)) + wanted.remove(entry_name) for kind, wanted in wanted_plugins: From 5de1273259de58d335fa603e905aeaa992b0b828 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Tue, 6 Jun 2023 23:13:21 +0200 Subject: [PATCH 4/5] Avoid effects of YAML tags in get_deps --- mkdocs/commands/get_deps.py | 23 +++++++++++++++++++++-- mkdocs/utils/__init__.py | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/mkdocs/commands/get_deps.py b/mkdocs/commands/get_deps.py index 6cc07875..ac4cb186 100644 --- a/mkdocs/commands/get_deps.py +++ b/mkdocs/commands/get_deps.py @@ -4,19 +4,38 @@ import dataclasses import datetime import functools import logging +import sys from typing import Mapping, Sequence +if sys.version_info >= (3, 10): + from importlib.metadata import EntryPoint, entry_points +else: + from importlib_metadata import EntryPoint, entry_points + import yaml from mkdocs import utils from mkdocs.config.base import _open_config_file -from mkdocs.plugins import EntryPoint, entry_points from mkdocs.utils.cache import download_and_cache_url log = logging.getLogger(__name__) # Note: do not rely on functions in this module, it is not public API. + +class YamlLoader(yaml.SafeLoader): + pass + + +# Prevent errors from trying to access external modules which may not be installed yet. +YamlLoader.add_constructor("!ENV", lambda loader, node: None) # type: ignore +YamlLoader.add_multi_constructor( + "tag:yaml.org,2002:python/name:", lambda loader, suffix, node: None +) +YamlLoader.add_multi_constructor( + "tag:yaml.org,2002:python/object/apply:", lambda loader, suffix, node: None +) + NotFound = () @@ -76,7 +95,7 @@ def get_deps(projects_file_url: str, config_file_path: str | None = None) -> Non config_file_path: Non-default path to mkdocs.yml. """ with _open_config_file(config_file_path) as f: - cfg = utils.yaml_load(f) + cfg = utils.yaml_load(f, loader=YamlLoader) # type: ignore if all(c not in cfg for c in ('site_name', 'theme', 'plugins', 'markdown_extensions')): log.warning("The passed config file doesn't seem to be a mkdocs.yml config file") diff --git a/mkdocs/utils/__init__.py b/mkdocs/utils/__init__.py index 5ebf3731..e2319758 100644 --- a/mkdocs/utils/__init__.py +++ b/mkdocs/utils/__init__.py @@ -63,7 +63,7 @@ def get_yaml_loader(loader=yaml.Loader): return Loader -def yaml_load(source: IO | str, loader: type[yaml.Loader] | None = None) -> dict[str, Any]: +def yaml_load(source: IO | str, loader: type[yaml.BaseLoader] | None = None) -> dict[str, Any]: """Return dict of source YAML file using loader, recursively deep merging inherited parent.""" Loader = loader or get_yaml_loader() result = yaml.load(source, Loader=Loader) From 619f7cf8988960ada8018bdbcb3ed6f934e0dc1d Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Wed, 7 Jun 2023 00:13:19 +0200 Subject: [PATCH 5/5] Add tests for `get_deps` command + bugfix for namespacing --- mkdocs/commands/get_deps.py | 6 +- mkdocs/tests/get_deps_tests.py | 161 +++++++++++++++++++++++++ mkdocs/tests/integration.py | 4 +- mkdocs/tests/integration/projects.yaml | 82 +++++++++++++ 4 files changed, 250 insertions(+), 3 deletions(-) create mode 100644 mkdocs/tests/get_deps_tests.py create mode 100644 mkdocs/tests/integration/projects.yaml diff --git a/mkdocs/commands/get_deps.py b/mkdocs/commands/get_deps.py index ac4cb186..9c8cc436 100644 --- a/mkdocs/commands/get_deps.py +++ b/mkdocs/commands/get_deps.py @@ -125,14 +125,16 @@ def get_deps(projects_file_url: str, config_file_path: str | None = None) -> Non for kind, wanted in wanted_plugins: available = strings(project.get(kind.projects_key, ())) for entry_name in available: - if entry_name in wanted or ( - # Also check theme-namespaced plugin names against the current theme. + if ( # Also check theme-namespaced plugin names against the current theme. '/' in entry_name and theme is not None and kind.projects_key == 'mkdocs_plugin' and entry_name.startswith(f'{theme}/') and entry_name[len(theme) + 1 :] in wanted + and entry_name not in wanted ): + entry_name = entry_name[len(theme) + 1 :] + if entry_name in wanted: if 'pypi_id' in project: install_name = project['pypi_id'] elif 'github_id' in project: diff --git a/mkdocs/tests/get_deps_tests.py b/mkdocs/tests/get_deps_tests.py new file mode 100644 index 00000000..0b2153ad --- /dev/null +++ b/mkdocs/tests/get_deps_tests.py @@ -0,0 +1,161 @@ +import contextlib +import io +import os +import textwrap +import unittest + +from mkdocs.commands.get_deps import get_deps +from mkdocs.tests.base import tempdir + +_projects_file_path = os.path.join( + os.path.abspath(os.path.dirname(__file__)), 'integration', 'projects.yaml' +) + + +class TestGetDeps(unittest.TestCase): + @contextlib.contextmanager + def _assert_logs(self, expected): + with self.assertLogs('mkdocs.commands.get_deps') as cm: + yield + msgs = [f'{r.levelname}:{r.message}' for r in cm.records] + self.assertEqual('\n'.join(msgs), textwrap.dedent(expected).strip('\n')) + + @tempdir() + def _test_get_deps(self, tempdir, yml, expected): + if yml: + yml = 'site_name: Test\n' + textwrap.dedent(yml) + projects_path = os.path.join(tempdir, 'projects.yaml') + with open(projects_path, 'w') as f: + f.write(yml) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + get_deps(_projects_file_path, projects_path) + self.assertEqual(buf.getvalue().split(), expected) + + def test_empty_config(self): + expected_logs = "WARNING:The passed config file doesn't seem to be a mkdocs.yml config file" + with self._assert_logs(expected_logs): + self._test_get_deps('', []) + + def test_just_search(self): + cfg = ''' + plugins: [search] + ''' + self._test_get_deps(cfg, []) + + def test_mkdocs_config(self): + cfg = ''' + site_name: MkDocs + theme: + name: mkdocs + locale: en + markdown_extensions: + - toc: + permalink:  + - attr_list + - def_list + - tables + - pymdownx.highlight: + use_pygments: false + - pymdownx.snippets + - pymdownx.superfences + - callouts + - mdx_gh_links: + user: mkdocs + repo: mkdocs + - mkdocs-click + plugins: + - search + - redirects: + - autorefs + - literate-nav: + nav_file: README.md + implicit_index: true + - mkdocstrings: + handlers: + python: + options: + docstring_section_style: list + ''' + self._test_get_deps( + cfg, + [ + 'markdown-callouts', + 'mdx-gh-links', + 'mkdocs-autorefs', + 'mkdocs-click', + 'mkdocs-literate-nav', + 'mkdocs-redirects', + 'mkdocstrings', + 'mkdocstrings-python', + 'pymdown-extensions', + ], + ) + + def test_dict_keys_and_ignores_env(self): + cfg = ''' + theme: + name: material + plugins: + code-validator: + enabled: !ENV [LINT, false] + markdown_extensions: + pymdownx.emoji: + emoji_index: !!python/name:materialx.emoji.twemoji + emoji_generator: !!python/name:materialx.emoji.to_svg + ''' + self._test_get_deps(cfg, ['mkdocs-code-validator', 'mkdocs-material', 'pymdown-extensions']) + + def test_theme_precedence(self): + cfg = ''' + plugins: + - tags + theme: material + ''' + self._test_get_deps(cfg, ['mkdocs-material']) + + cfg = ''' + plugins: + - material/tags + ''' + self._test_get_deps(cfg, ['mkdocs-material']) + + cfg = ''' + plugins: + - tags + ''' + self._test_get_deps(cfg, ['mkdocs-plugin-tags']) + + def test_nonexistent(self): + cfg = ''' + plugins: + - taglttghhmdu + - syyisjupkbpo + - redirects + theme: qndyakplooyh + markdown_extensions: + - saqdhyndpvpa + ''' + expected_logs = """ + WARNING:Theme 'qndyakplooyh' is not provided by any registered project + WARNING:Plugin 'syyisjupkbpo' is not provided by any registered project + WARNING:Plugin 'taglttghhmdu' is not provided by any registered project + WARNING:Extension 'saqdhyndpvpa' is not provided by any registered project + """ + with self._assert_logs(expected_logs): + self._test_get_deps(cfg, ['mkdocs-redirects']) + + def test_git_and_shadowed(self): + cfg = ''' + theme: bootstrap4 + plugins: [blog] + ''' + self._test_get_deps( + cfg, ['git+https://github.com/andyoakley/mkdocs-blog', 'mkdocs-bootstrap4'] + ) + + def test_multi_theme(self): + cfg = ''' + theme: minty + ''' + self._test_get_deps(cfg, ['mkdocs-bootswatch']) diff --git a/mkdocs/tests/integration.py b/mkdocs/tests/integration.py index 8f3fdc37..a9569337 100644 --- a/mkdocs/tests/integration.py +++ b/mkdocs/tests/integration.py @@ -60,8 +60,10 @@ def main(output=None): log.debug("Building test projects.") for project in os.listdir(TEST_PROJECTS): - log.debug(f"Building test project: {project}") project_dir = os.path.join(TEST_PROJECTS, project) + if not os.path.isdir(project_dir): + continue + log.debug(f"Building test project: {project}") out = os.path.join(output, project) command = base_cmd + [out] subprocess.check_call(command, cwd=project_dir) diff --git a/mkdocs/tests/integration/projects.yaml b/mkdocs/tests/integration/projects.yaml new file mode 100644 index 00000000..4cca01e3 --- /dev/null +++ b/mkdocs/tests/integration/projects.yaml @@ -0,0 +1,82 @@ +# DO NOT UPDATE THIS FILE, only for tests. +# This is an intentionally small subset of https://github.com/mkdocs/catalog +projects: +- name: Material for MkDocs + mkdocs_theme: material + mkdocs_plugin: [material/info, material/offline, material/search, material/social, material/tags] + github_id: squidfunk/mkdocs-material + pypi_id: mkdocs-material +- name: Bootstrap4 + mkdocs_theme: bootstrap4 + github_id: byrnereese/mkdocs-bootstrap4 + pypi_id: mkdocs-bootstrap4 +- name: Bootstrap 4 + mkdocs_theme: bootstrap4 + shadowed: [mkdocs_theme] + github_id: LukeCarrier/mkdocs-theme-bootstrap4 + pypi_id: mkdocs-theme-bootstrap4 +- name: Bootswatch + mkdocs_theme: [cerulean, cosmo, cyborg, darkly, flatly, journal, litera, lumen, lux, materia, minty, pulse, sandstone, simplex, slate, solar, spacelab, superhero, united, yeti] + github_id: mkdocs/mkdocs-bootswatch + pypi_id: mkdocs-bootswatch +- name: mkdocstrings + mkdocs_plugin: mkdocstrings + extra_dependencies: + plugins.mkdocstrings.handlers.crystal: mkdocstrings-crystal + plugins.mkdocstrings.handlers.python: mkdocstrings-python + github_id: mkdocstrings/mkdocstrings + pypi_id: mkdocstrings +- name: mkdocs-click + markdown_extension: mkdocs-click + github_id: DataDog/mkdocs-click + pypi_id: mkdocs-click +- name: blog + mkdocs_plugin: blog + github_id: andyoakley/mkdocs-blog +- name: Blogs for MkDocs + shadowed: [mkdocs_plugin] + mkdocs_plugin: blog + github_id: fmaida/mkdocs-blog-plugin +- name: foo + homepage: foo + gitlab_id: bar/foo +- name: Termage + mkdocs_plugin: termage + github_id: bczsalba/Termage +- name: Github-Links + markdown_extension: mdx_gh_links + github_id: Python-Markdown/github-links + pypi_id: mdx-gh-links +- name: autorefs + mkdocs_plugin: autorefs + github_id: mkdocstrings/autorefs + pypi_id: mkdocs-autorefs +- name: mkdocs-redirects + mkdocs_plugin: redirects + github_id: mkdocs/mkdocs-redirects + pypi_id: mkdocs-redirects +- name: markdown-callouts + markdown_extension: callouts + github_id: oprypin/markdown-callouts + pypi_id: markdown-callouts +- name: PyMdown Extensions + markdown_extension: [pymdownx.arithmatex, pymdownx.b64, pymdownx.betterem, pymdownx.caret, pymdownx.critic, pymdownx.details, pymdownx.emoji, pymdownx.escapeall, pymdownx.extra, pymdownx.highlight, pymdownx.inlinehilite, pymdownx.keys, pymdownx.magiclink, pymdownx.mark, pymdownx.pathconverter, pymdownx.progressbar, pymdownx.saneheaders, pymdownx.smartsymbols, pymdownx.snippets, pymdownx.striphtml, pymdownx.superfences, pymdownx.tabbed, pymdownx.tasklist, pymdownx.tilde] + github_id: facelessuser/pymdown-extensions + pypi_id: pymdown-extensions +- name: literate-nav + mkdocs_plugin: literate-nav + github_id: oprypin/mkdocs-literate-nav + pypi_id: mkdocs-literate-nav +- name: mkdocs-code-validator + mkdocs_plugin: code-validator + github_id: oprypin/mkdocs-code-validator + pypi_id: mkdocs-code-validator +- name: tags + mkdocs_plugin: tags + description: Processes tags in yaml metadata + github_id: jldiaz/mkdocs-plugin-tags + pypi_id: mkdocs-plugin-tags +- name: tags + mkdocs_plugin: autotag + github_id: six-two/mkdocs-auto-tag-plugin + pypi_id: mkdocs-auto-tag-plugin