Serve excluded files in mkdocs serve + new not_in_nav config

* Docs that match any of the `not_in_nav` patterns will not produce warnings about never being included into the nav.

* Files that are excluded by `exclude_docs` will now be preserved in the `Files` collection and will even be rendered but only in `mkdocs serve` mode - such files will have a `DRAFT` notice prepended to the content.

docs/user-guide/configuration.md

@@ -310,6 +310,8 @@ exclude_docs: |
 
 This follows the [.gitignore pattern format](https://git-scm.com/docs/gitignore#_pattern_format).
 
+Note that `mkdocs serve` does *not* follow this setting and instead displays excluded documents but with a "DRAFT" mark.
+
 The following defaults are always implicitly prepended - to exclude dot-files (and directories) as well as the top-level `templates` directory:
 
 ```yaml
@@ -327,6 +329,31 @@ exclude_docs: |
     !.assets  # Don't exclude '.assets' although all other '.*' are excluded
 ```
 
+### not_in_nav
+
+NEW: **New in version 1.5.**
+
+NOTE: This option does *not* actually exclude anything from the nav.
+
+If you want to include some docs into the site but intentionally exclude them from the nav, normally MkDocs warns about this.
+
+Adding such patterns of files (relative to [`docs_dir`](#docs_dir)) into the `not_in_nav` config will prevent such warnings.
+
+Example:
+
+```yaml
+nav:
+    - Foo: foo.md
+    - Bar: bar.md
+
+not_in_nav: |
+    /private.md
+```
+
+As the previous option, this follows the .gitignore pattern format.
+
+NOTE: Adding a given file to [`exclude_docs`](#exclude_docs) takes precedence over and implies `not_in_nav`.
+
 ## Build directories
 
 ### theme

mkdocs/commands/build.py

@@ -14,7 +14,7 @@ import mkdocs
 from mkdocs import utils
 from mkdocs.config.defaults import MkDocsConfig
 from mkdocs.exceptions import Abort, BuildError
-from mkdocs.structure.files import File, Files, get_files
+from mkdocs.structure.files import File, Files, InclusionLevel, _set_exclusions, get_files
 from mkdocs.structure.nav import Navigation, get_navigation
 from mkdocs.structure.pages import Page
 
@@ -200,6 +200,7 @@ def _build_page(
     nav: Navigation,
     env: jinja2.Environment,
     dirty: bool = False,
+    excluded: bool = False,
 ) -> None:
     """Pass a Page to theme template and write output to site_dir."""
 
@@ -227,6 +228,13 @@ def _build_page(
             'page_context', context, page=page, config=config, nav=nav
         )
 
+        if excluded:
+            page.content = (
+                '<div class="mkdocs-draft-marker" title="This page will not be included into the built site.">'
+                'DRAFT'
+                '</div>' + (page.content or '')
+            )
+
         # Render the template.
         output = template.render(context)
 
@@ -263,6 +271,8 @@ def build(config: MkDocsConfig, live_server: bool = False, dirty: bool = False)
     if config.strict:
         logging.getLogger('mkdocs').addHandler(warning_counter)
 
+    inclusion = InclusionLevel.all if live_server else InclusionLevel.is_included
+
     try:
         start = time.monotonic()
 
@@ -295,6 +305,8 @@ def build(config: MkDocsConfig, live_server: bool = False, dirty: bool = False)
 
         # Run `files` plugin events.
         files = config.plugins.run_event('files', files, config=config)
+        # If plugins have added files but haven't set their inclusion level, calculate it again.
+        _set_exclusions(files._files, config)
 
         nav = get_navigation(files, config)
 
@@ -302,10 +314,20 @@ def build(config: MkDocsConfig, live_server: bool = False, dirty: bool = False)
         nav = config.plugins.run_event('nav', nav, config=config, files=files)
 
         log.debug("Reading markdown pages.")
-        for file in files.documentation_pages():
+        excluded = []
+        for file in files.documentation_pages(inclusion=inclusion):
             log.debug(f"Reading: {file.src_uri}")
+            if file.page is None and file.inclusion.is_excluded():
+                excluded.append(file.src_path)
+                Page(None, file, config)
             assert file.page is not None
             _populate_page(file.page, config, files, dirty)
+        if excluded:
+            log.info(
+                "The following pages are being built only for the preview "
+                "but will be excluded from `mkdocs build` per `exclude_docs`:\n  - "
+                + "\n  - ".join(excluded)
+            )
 
         # Run `env` plugin events.
         env = config.plugins.run_event('env', env, config=config, files=files)
@@ -314,7 +336,7 @@ def build(config: MkDocsConfig, live_server: bool = False, dirty: bool = False)
         # with lower precedence get written first so that files with higher precedence can overwrite them.
 
         log.debug("Copying static assets.")
-        files.copy_static_files(dirty=dirty)
+        files.copy_static_files(dirty=dirty, inclusion=inclusion)
 
         for template in config.theme.static_templates:
             _build_theme_template(template, env, files, config, nav)
@@ -323,10 +345,12 @@ def build(config: MkDocsConfig, live_server: bool = False, dirty: bool = False)
             _build_extra_template(template, files, config, nav)
 
         log.debug("Building markdown pages.")
-        doc_files = files.documentation_pages()
+        doc_files = files.documentation_pages(inclusion=inclusion)
         for file in doc_files:
             assert file.page is not None
-            _build_page(file.page, config, doc_files, nav, env, dirty)
+            _build_page(
+                file.page, config, doc_files, nav, env, dirty, excluded=file.inclusion.is_excluded()
+            )
 
         # Run `post_build` plugin events.
         config.plugins.run_event('post_build', config=config)

mkdocs/config/defaults.py

@@ -29,6 +29,9 @@ class MkDocsConfig(base.Config):
     exclude_docs = c.Optional(c.PathSpec())
     """Gitignore-like patterns of files (relative to docs dir) to exclude from the site."""
 
+    not_in_nav = c.Optional(c.PathSpec())
+    """Gitignore-like patterns of files (relative to docs dir) that are not intended to be in the nav."""
+
     site_url = c.Optional(c.URL(is_dir=True))
     """The full URL to where the documentation will be hosted."""
 

mkdocs/structure/files.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import enum
 import fnmatch
 import logging
 import os
@@ -7,7 +8,7 @@ import posixpath
 import shutil
 import warnings
 from pathlib import PurePath
-from typing import TYPE_CHECKING, Any, Iterable, Iterator, Mapping, Sequence
+from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, Mapping, Sequence
 from urllib.parse import quote as urlquote
 
 import jinja2.environment
@@ -25,6 +26,32 @@ if TYPE_CHECKING:
 log = logging.getLogger(__name__)
 
 
+class InclusionLevel(enum.Enum):
+    Excluded = -2
+    """The file is excluded from the final site, but will still be populated during `mkdocs serve`."""
+    NotInNav = -1
+    """The file is part of the site, but doesn't produce nav warnings."""
+    Undefined = 0
+    """Still needs to be computed based on the config. If the config doesn't kick in, acts the same as `included`."""
+    Included = 1
+    """The file is part of the site. Documentation pages that are omitted from the nav will produce warnings."""
+
+    def all(self):
+        return True
+
+    def is_included(self):
+        return self.value > self.Excluded.value
+
+    def is_excluded(self):
+        return self.value <= self.Excluded.value
+
+    def is_in_nav(self):
+        return self.value > self.NotInNav.value
+
+    def is_not_in_nav(self):
+        return self.value <= self.NotInNav.value
+
+
 class Files:
     """A collection of [File][mkdocs.structure.files.File] objects."""
 
@@ -71,15 +98,22 @@ class Files:
         self._src_uris = None
         self._files.remove(file)
 
-    def copy_static_files(self, dirty: bool = False) -> None:
+    def copy_static_files(
+        self,
+        dirty: bool = False,
+        *,
+        inclusion: Callable[[InclusionLevel], bool] = InclusionLevel.is_included,
+    ) -> None:
         """Copy static files from source to destination."""
         for file in self:
-            if not file.is_documentation_page():
+            if not file.is_documentation_page() and inclusion(file.inclusion):
                 file.copy_file(dirty)
 
-    def documentation_pages(self) -> Sequence[File]:
+    def documentation_pages(
+        self, *, inclusion: Callable[[InclusionLevel], bool] = InclusionLevel.is_included
+    ) -> Sequence[File]:
         """Return iterable of all Markdown page file objects."""
-        return [file for file in self if file.is_documentation_page()]
+        return [file for file in self if file.is_documentation_page() and inclusion(file.inclusion)]
 
     def static_pages(self) -> Sequence[File]:
         """Return iterable of all static page file objects."""
@@ -156,6 +190,9 @@ class File:
     url: str
     """The URI of the destination file relative to the destination directory as a string."""
 
+    inclusion: InclusionLevel
+    """Whether the file will be excluded from the built site."""
+
     @property
     def src_path(self) -> str:
         """Same as `src_uri` (and synchronized with it) but will use backslashes on Windows. Discouraged."""
@@ -184,6 +221,7 @@ class File:
         use_directory_urls: bool,
         *,
         dest_uri: str | None = None,
+        inclusion: InclusionLevel = InclusionLevel.Undefined,
     ) -> None:
         self.page = None
         self.src_path = path
@@ -194,6 +232,7 @@ class File:
         self.url = self._get_url(use_directory_urls)
         self.abs_src_path = os.path.normpath(os.path.join(src_dir, self.src_uri))
         self.abs_dest_path = os.path.normpath(os.path.join(dest_dir, self.dest_uri))
+        self.inclusion = inclusion
 
     def __eq__(self, other) -> bool:
         return (
@@ -280,16 +319,32 @@ class File:
 _default_exclude = pathspec.gitignore.GitIgnoreSpec.from_lines(['.*', '/templates/'])
 
 
+def _set_exclusions(files: Iterable[File], config: MkDocsConfig | Mapping[str, Any]) -> None:
+    """Re-calculate which files are excluded, based on the patterns in the config."""
+    exclude: pathspec.gitignore.GitIgnoreSpec | None = config.get('exclude_docs')
+    exclude = _default_exclude + exclude if exclude else _default_exclude
+    nav_exclude: pathspec.gitignore.GitIgnoreSpec | None = config.get('not_in_nav')
+
+    for file in files:
+        if file.inclusion == InclusionLevel.Undefined:
+            if exclude.match_file(file.src_uri):
+                file.inclusion = InclusionLevel.Excluded
+            elif nav_exclude and nav_exclude.match_file(file.src_uri):
+                file.inclusion = InclusionLevel.NotInNav
+            else:
+                file.inclusion = InclusionLevel.Included
+
+
 def get_files(config: MkDocsConfig | Mapping[str, Any]) -> Files:
     """Walk the `docs_dir` and return a Files collection."""
-    exclude = config.get('exclude_docs')
-    exclude = _default_exclude + exclude if exclude else _default_exclude
-    files = []
+    files: list[File] = []
+    conflicting_files: list[tuple[File, File]] = []
     for source_dir, dirnames, filenames in os.walk(config['docs_dir'], followlinks=True):
         relative_dir = os.path.relpath(source_dir, config['docs_dir'])
         dirnames.sort()
         filenames.sort(key=_file_sort_key)
 
+        files_by_dest: dict[str, File] = {}
         for filename in filenames:
             file = File(
                 os.path.join(relative_dir, filename),
@@ -297,26 +352,31 @@ def get_files(config: MkDocsConfig | Mapping[str, Any]) -> Files:
                 config['site_dir'],
                 config['use_directory_urls'],
             )
-            # Skip any excluded files
-            if exclude.match_file(file.src_uri):
-                continue
-            # Skip README.md if an index file also exists in dir
-            if filename == 'README.md' and 'index.md' in filenames:
-                log.warning(
-                    f"Excluding '{file.src_uri}' from the site because "
-                    f"it conflicts with 'index.md' in the same directory."
-                )
-                continue
+            # Skip README.md if an index file also exists in dir (part 1)
+            prev_file = files_by_dest.setdefault(file.dest_uri, file)
+            if prev_file is not file:
+                conflicting_files.append((prev_file, file))
             files.append(file)
+            prev_file = file
+
+    _set_exclusions(files, config)
+    # Skip README.md if an index file also exists in dir (part 2)
+    for a, b in conflicting_files:
+        if b.inclusion.is_included():
+            if a.inclusion.is_included():
+                log.warning(
+                    f"Excluding '{a.src_uri}' from the site because it conflicts with '{b.src_uri}'."
+                )
+            files.remove(a)
+        else:
+            files.remove(b)
 
     return Files(files)
 
 
 def _file_sort_key(f: str):
     """Always sort `index` or `README` as first filename in list."""
-    if os.path.splitext(f)[0] in ('index', 'README'):
-        return (0,)
-    return (1, f)
+    return (os.path.splitext(f)[0] not in ('index', 'README'), f)
 
 
 def _sort_files(filenames: Iterable[str]) -> list[str]:

mkdocs/structure/nav.py

@@ -147,7 +147,8 @@ class Link:
 
 def get_navigation(files: Files, config: MkDocsConfig | Mapping[str, Any]) -> Navigation:
     """Build site navigation from config and files."""
-    nav_config = config['nav'] or nest_paths(f.src_uri for f in files.documentation_pages())
+    documentation_pages = files.documentation_pages()
+    nav_config = config['nav'] or nest_paths(f.src_uri for f in documentation_pages)
     items = _data_to_navigation(nav_config, files, config)
     if not isinstance(items, list):
         items = [items]
@@ -159,19 +160,20 @@ def get_navigation(files: Files, config: MkDocsConfig | Mapping[str, Any]) -> Na
     _add_previous_and_next_links(pages)
     _add_parent_links(items)
 
-    missing_from_config = [file for file in files.documentation_pages() if file.page is None]
+    missing_from_config = []
+    for file in documentation_pages:
+        if file.page is None:
+            # Any documentation files not found in the nav should still have an associated page, so we
+            # create them here. The Page object will automatically be assigned to `file.page` during
+            # its creation (and this is the only way in which these page objects are accessible).
+            Page(None, file, config)
+            if file.inclusion.is_in_nav():
+                missing_from_config.append(file.src_path)
     if missing_from_config:
         log.info(
             'The following pages exist in the docs directory, but are not '
-            'included in the "nav" configuration:\n  - {}'.format(
-                '\n  - '.join(file.src_path for file in missing_from_config)
-            )
+            'included in the "nav" configuration:\n  - ' + '\n  - '.join(missing_from_config)
         )
-        # Any documentation files not found in the nav should still have an associated page, so we
-        # create them here. The Page object will automatically be assigned to `file.page` during
-        # its creation (and this is the only way in which these page objects are accessible).
-        for file in missing_from_config:
-            Page(None, file, config)
 
     links = _get_by_type(items, Link)
     for link in links:
@@ -184,11 +186,10 @@ def get_navigation(files: Files, config: MkDocsConfig | Mapping[str, Any]) -> Na
                 "configuration, which presumably points to an external resource."
             )
         else:
-            msg = (
+            log.warning(
                 f"A relative path to '{link.url}' is included in the 'nav' "
-                "configuration, which is not found in the documentation files"
+                "configuration, which is not found in the documentation files."
             )
-            log.warning(msg)
     return Navigation(items, pages)
 
 
@@ -210,6 +211,11 @@ def _data_to_navigation(data, files: Files, config: MkDocsConfig | Mapping[str,
     title, path = data if isinstance(data, tuple) else (None, data)
     file = files.get_file_from_path(path)
     if file:
+        if file.inclusion.is_excluded():
+            log.info(
+                f"A relative path to '{file.src_path}' is included in the 'nav' "
+                "configuration, but this file is excluded from the built site."
+            )
         return Page(title, file, config)
     return Link(title, path)
 

mkdocs/structure/pages.py

@@ -330,6 +330,11 @@ class _RelativePathTreeprocessor(Treeprocessor):
                 f"'{target_uri}' which is not found in the documentation files."
             )
             return url
+        if target_file.inclusion.is_excluded():
+            log.info(
+                f"Documentation file '{self.file.src_uri}' contains a link to "
+                f"'{target_uri}' which is excluded from the built site."
+            )
         path = target_file.url_relative_to(self.file)
         components = (scheme, netloc, path, query, fragment)
         return urlunsplit(components)

mkdocs/tests/structure/file_tests.py

@@ -60,7 +60,7 @@ class TestFiles(PathAssertionMixin, unittest.TestCase):
 
         self.assertEqual(
             _sort_files(['a.md', 'index.md', 'b.md', 'index.html']),
-            ['index.md', 'index.html', 'a.md', 'b.md'],
+            ['index.html', 'index.md', 'a.md', 'b.md'],
         )
 
         self.assertEqual(
@@ -631,9 +631,15 @@ class TestFiles(PathAssertionMixin, unittest.TestCase):
     def test_get_files(self, tdir):
         config = load_config(docs_dir=tdir, extra_css=['bar.css'], extra_javascript=['bar.js'])
         files = get_files(config)
-        expected = ['index.md', 'bar.css', 'bar.html', 'bar.jpg', 'bar.js', 'bar.md', 'readme.md']
         self.assertIsInstance(files, Files)
-        self.assertEqual([f.src_path for f in files], expected)
+        self.assertEqual(
+            [f.src_path for f in files if f.inclusion.is_included()],
+            ['index.md', 'bar.css', 'bar.html', 'bar.jpg', 'bar.js', 'bar.md', 'readme.md'],
+        )
+        self.assertEqual(
+            [f.src_path for f in files if f.inclusion.is_excluded()],
+            ['.dotfile', 'templates/foo.html'],
+        )
 
     @tempdir(
         files=[
@@ -644,9 +650,8 @@ class TestFiles(PathAssertionMixin, unittest.TestCase):
     def test_get_files_include_readme_without_index(self, tdir):
         config = load_config(docs_dir=tdir)
         files = get_files(config)
-        expected = ['README.md', 'foo.md']
         self.assertIsInstance(files, Files)
-        self.assertEqual([f.src_path for f in files], expected)
+        self.assertEqual([f.src_path for f in files], ['README.md', 'foo.md'])
 
     @tempdir(
         files=[
@@ -662,11 +667,10 @@ class TestFiles(PathAssertionMixin, unittest.TestCase):
         self.assertRegex(
             '\n'.join(cm.output),
             r"^WARNING:mkdocs.structure.files:"
-            r"Excluding 'README.md' from the site because it conflicts with 'index.md' in the same directory.$",
+            r"Excluding 'README.md' from the site because it conflicts with 'index.md'.$",
         )
-        expected = ['index.md', 'foo.md']
         self.assertIsInstance(files, Files)
-        self.assertEqual([f.src_path for f in files], expected)
+        self.assertEqual([f.src_path for f in files], ['index.md', 'foo.md'])
 
     @tempdir()
     @tempdir(files={'test.txt': 'source content'})

mkdocs/tests/structure/nav_tests.py

@@ -159,9 +159,9 @@ class SiteNavigationTests(unittest.TestCase):
             cm.output,
             [
                 "WARNING:mkdocs.structure.nav:A relative path to 'missing.html' is included "
-                "in the 'nav' configuration, which is not found in the documentation files",
+                "in the 'nav' configuration, which is not found in the documentation files.",
                 "WARNING:mkdocs.structure.nav:A relative path to 'example.com' is included "
-                "in the 'nav' configuration, which is not found in the documentation files",
+                "in the 'nav' configuration, which is not found in the documentation files.",
             ],
         )
         self.assertEqual(str(site_navigation).strip(), expected)