Use toc_tokens to generate the TOC

This patch improves the consistency of TOC levels, so now the level is always
equal to the N in the `<hN>` tag. It also allows users of the MkDocs theme to
set the navigation depth to show in the TOC panel (defaulting to 2).
Resolves #1910 and resolves #770.

docs/about/release-notes.md

@@ -96,6 +96,9 @@ do, adding `--strict`, `--theme`, `--theme-dir`, and `--site-dir`.
   theme (#1234).
 * Bugfix: Multi-row nav headers in the `mkdocs` theme no longer obscure the
   document content (#716).
+* Add support for `navigation_depth` theme option for the `mkdocs` theme (#1970).
+* `level` attribute in `page.toc` items is now 1-indexed to match the level in
+  `<hN>` tags (#1970).
 
 ## Version 1.0.4 (2018-09-07)
 

docs/user-guide/styling-your-docs.md

@@ -73,6 +73,9 @@ supports the following options:
 
     * __`search`__: Display the search modal. Default: `83` (s)
 
+* __`navigation_depth`__: The maximum depth of the navigation tree in the
+  sidebar. Default: `2`.
+
 * __`nav_style`__: This adjusts the visual style for the top navigation bar; by
   default, this is set to `primary` (the default), but it can also be set to
   `dark` or `light`.

mkdocs/contrib/search/search_index.py

@@ -27,7 +27,7 @@ class SearchIndex:
         and return the matched item in the TOC.
         """
         for toc_item in toc:
-            if toc_item.url[1:] == id_:
+            if toc_item.id == id_:
                 return toc_item
             toc_item_r = self._find_toc_by_id(toc_item.children, id_)
             if toc_item_r is not None:

mkdocs/structure/pages.py

@@ -181,7 +181,7 @@ class Page:
             extension_configs=config['mdx_configs'] or {}
         )
         self.content = md.convert(self.markdown)
-        self.toc = get_toc(getattr(md, 'toc', ''))
+        self.toc = get_toc(getattr(md, 'toc_tokens', []))
 
 
 class _RelativePathTreeprocessor(Treeprocessor):

mkdocs/structure/toc.py

@@ -1,16 +1,18 @@
 """
 Deals with generating the per-page table of contents.
 
-For the sake of simplicity we use an existing markdown extension to generate
-an HTML table of contents, and then parse that into the underlying data.
+For the sake of simplicity we use the Python-Markdown `toc` extension to
+generate a list of dicts for each toc item, and then store it as AnchorLinks to
+maintain compatibility with older versions of MkDocs.
 """
 
-from html.parser import HTMLParser
 
-
-def get_toc(toc_html):
-    items = _parse_html_table_of_contents(toc_html)
-    return TableOfContents(items)
+def get_toc(toc_tokens):
+    toc = [_parse_toc_token(i) for i in toc_tokens]
+    # For the table of contents, always mark the first element as active
+    if len(toc):
+        toc[0].active = True
+    return TableOfContents(toc)
 
 
 class TableOfContents:
@@ -34,10 +36,14 @@ class AnchorLink:
     """
     A single entry in the table of contents.
     """
-    def __init__(self, title, url, level):
-        self.title, self.url, self.level = title, url, level
+    def __init__(self, title, id, level):
+        self.title, self.id, self.level = title, id, level
         self.children = []
 
+    @property
+    def url(self):
+        return '#' + self.id
+
     def __str__(self):
         return self.indent_print()
 
@@ -49,79 +55,8 @@ class AnchorLink:
         return ret
 
 
-class _TOCParser(HTMLParser):
-    def __init__(self):
-        HTMLParser.__init__(self)
-        self.links = []
-
-        self.in_anchor = False
-        self.attrs = None
-        self.title = ''
-
-        # Prior to Python3.4 no convert_charrefs keyword existed.
-        # However, in Python3.5 the default was changed to True.
-        # We need the False behavior in all versions but can only
-        # set it if it exists.
-        if hasattr(self, 'convert_charrefs'):  # pragma: no cover
-            self.convert_charrefs = False
-
-    def handle_starttag(self, tag, attrs):
-        if not self.in_anchor:
-            if tag == 'a':
-                self.in_anchor = True
-                self.attrs = dict(attrs)
-
-    def handle_endtag(self, tag):
-        if tag == 'a':
-            self.in_anchor = False
-
-    def handle_data(self, data):
-        if self.in_anchor:
-            self.title += data
-
-    def handle_charref(self, ref):
-        self.handle_entityref("#" + ref)
-
-    def handle_entityref(self, ref):
-        self.handle_data("&%s;" % ref)
-
-
-def _parse_html_table_of_contents(html):
-    """
-    Given a table of contents string that has been automatically generated by
-    the markdown library, parse it into a tree of AnchorLink instances.
-
-    Returns a list of all the parent AnchorLink instances.
-    """
-    lines = html.splitlines()[2:-2]
-    ret, parents, level = [], [], 0
-    for line in lines:
-        parser = _TOCParser()
-        parser.feed(line)
-        if parser.title:
-            try:
-                href = parser.attrs['href']
-            except KeyError:
-                continue
-            title = parser.title
-            nav = AnchorLink(title, href, level)
-            # Add the item to its parent if required.  If it is a topmost
-            # item then instead append it to our return value.
-            if parents:
-                parents[-1].children.append(nav)
-            else:
-                ret.append(nav)
-            # If this item has children, store it as the current parent
-            if line.endswith('<ul>'):
-                level += 1
-                parents.append(nav)
-        elif line.startswith('</ul>'):
-            level -= 1
-            if parents:
-                parents.pop()
-
-    # For the table of contents, always mark the first element as active
-    if ret:
-        ret[0].active = True
-
-    return ret
+def _parse_toc_token(token):
+    anchor = AnchorLink(token['name'], token['id'], token['level'])
+    for i in token['children']:
+        anchor.children.append(_parse_toc_token(i))
+    return anchor

mkdocs/tests/base.py

@@ -16,7 +16,7 @@ def get_markdown_toc(markdown_source):
     """ Return TOC generated by Markdown parser from Markdown source text. """
     md = markdown.Markdown(extensions=['toc'])
     md.convert(markdown_source)
-    return md.toc
+    return md.toc_tokens
 
 
 def load_config(**cfg):

mkdocs/tests/config/config_tests.py

@@ -119,8 +119,9 @@ class ConfigTests(unittest.TestCase):
                         'highlightjs': True,
                         'hljs_style': 'github',
                         'hljs_languages': [],
-                        'shortcuts': {'help': 191, 'next': 78, 'previous': 80, 'search': 83},
-                        'nav_style': 'primary'
+                        'navigation_depth': 2,
+                        'nav_style': 'primary',
+                        'shortcuts': {'help': 191, 'next': 78, 'previous': 80, 'search': 83}
                     }
                 }, {
                     'dirs': [os.path.join(theme_dir, 'readthedocs'), mkdocs_templates_dir],
@@ -182,8 +183,9 @@ class ConfigTests(unittest.TestCase):
                         'highlightjs': True,
                         'hljs_style': 'github',
                         'hljs_languages': [],
-                        'shortcuts': {'help': 191, 'next': 78, 'previous': 80, 'search': 83},
-                        'nav_style': 'primary'
+                        'navigation_depth': 2,
+                        'nav_style': 'primary',
+                        'shortcuts': {'help': 191, 'next': 78, 'previous': 80, 'search': 83}
                     }
                 }
             )

mkdocs/tests/structure/toc_tests.py

@@ -7,23 +7,6 @@ from mkdocs.tests.base import dedent, get_markdown_toc
 
 class TableOfContentsTests(unittest.TestCase):
 
-    def test_html_toc(self):
-        html = dedent("""
-        <div class="toc">
-        <ul>
-        <li><a href="#foo">Heading 1</a></li>
-        <li><a href="#bar">Heading 2</a></li>
-        </ul>
-        </div>
-        """)
-        expected = dedent("""
-        Heading 1 - #foo
-        Heading 2 - #bar
-        """)
-        toc = get_toc(html)
-        self.assertEqual(str(toc).strip(), expected)
-        self.assertEqual(len(toc), 2)
-
     def test_indented_toc(self):
         md = dedent("""
         # Heading 1
@@ -163,20 +146,6 @@ class TableOfContentsTests(unittest.TestCase):
         self.assertEqual(str(toc).strip(), expected)
         self.assertEqual(len(toc), 1)
 
-    def test_skip_no_href(self):
-        html = dedent("""
-        <div class="toc">
-        <ul>
-        <li><a>Header 1</a></li>
-        <li><a href="#foo">Header 2</a></li>
-        </ul>
-        </div>
-        """)
-        expected = 'Header 2 - #foo'
-        toc = get_toc(html)
-        self.assertEqual(str(toc).strip(), expected)
-        self.assertEqual(len(toc), 1)
-
     def test_level(self):
         md = dedent("""
         # Heading 1
@@ -192,4 +161,4 @@ class TableOfContentsTests(unittest.TestCase):
                 yield item.level
                 yield from get_level_sequence(item.children)
 
-        self.assertEqual(tuple(get_level_sequence(toc)), (0, 1, 2, 2, 1))
+        self.assertEqual(tuple(get_level_sequence(toc)), (1, 2, 3, 3, 2))

mkdocs/tests/theme_tests.py

@@ -32,8 +32,9 @@ class ThemeTests(unittest.TestCase):
             'highlightjs': True,
             'hljs_style': 'github',
             'hljs_languages': [],
-            'shortcuts': {'help': 191, 'next': 78, 'previous': 80, 'search': 83},
-            'nav_style': 'primary'
+            'navigation_depth': 2,
+            'nav_style': 'primary',
+            'shortcuts': {'help': 191, 'next': 78, 'previous': 80, 'search': 83}
         })
 
     def test_custom_dir(self):

mkdocs/themes/mkdocs/css/base.css

@@ -33,10 +33,6 @@ body > .container {
     /* csslint ignore:end */
 }
 
-ul.nav .main {
-    font-weight: bold;
-}
-
 .source-links {
     float: right;
 }
@@ -168,7 +164,7 @@ footer {
 }
 
 /* First level of nav */
-.bs-sidenav {
+.bs-sidebar > .navbar-collapse > .nav {
     padding-top:    10px;
     padding-bottom: 10px;
     border-radius: 5px;
@@ -194,16 +190,16 @@ footer {
     border-right: 1px solid;
 }
 
-/* Nav: second level (shown on .active) */
-.bs-sidebar .nav .nav {
-    display: none; /* Hide by default, but at >768px, show it */
-    margin-bottom: 8px;
+.bs-sidebar .nav .nav .nav {
+  margin-left: 1em;
 }
+
+.bs-sidebar .nav > li > a {
+    font-weight: bold;
+}
+
 .bs-sidebar .nav .nav > li > a {
-    padding-top:    3px;
-    padding-bottom: 3px;
-    padding-left: 30px;
-    font-size: 90%;
+    font-weight: normal;
 }
 
 .headerlink {

mkdocs/themes/mkdocs/mkdocs_theme.yml

@@ -9,6 +9,8 @@ search_index_only: false
 highlightjs: true
 hljs_languages: []
 hljs_style: github
+
+navigation_depth: 2
 nav_style: primary
 
 shortcuts:

mkdocs/themes/mkdocs/toc.html

@@ -5,15 +5,21 @@
         </button>
     </div>
 
+    {% macro toc_item(item) %}
+        {%- if item.level <= config.theme.navigation_depth %}
+            <li class="nav-item" data-level="{{ item.level }}"><a href="{{ item.url }}" class="nav-link">{{ item.title }}</a>
+              <ul class="nav flex-column">
+                {%- for child in item.children %}
+                    {{- toc_item(child) }}
+                {%- endfor %}
+              </ul>
+            </li>
+        {%- endif %}
+    {%- endmacro %}
     <div id="toc-collapse" class="navbar-collapse collapse card bg-secondary">
-        <ul class="nav flex-column bs-sidenav">
-        {%- for toc_item in page.toc %}
-            <li class="nav-item main"><a href="{{ toc_item.url }}">{{ toc_item.title }}</a></li>
-            {%- for toc_item in toc_item.children %}
-                <li class="nav-item">
-                    <a href="{{ toc_item.url }}" class="nav-link{% if toc_item.active %} active{% endif %}">{{ toc_item.title }}</a>
-                </li>
-            {%- endfor %}
+        <ul class="nav flex-column">
+        {%- for item in page.toc %}
+            {{ toc_item(item) }}
         {%- endfor %}
         </ul>
     </div>

requirements/project-min.txt

@@ -1,7 +1,7 @@
 click==3.3
 Jinja2==2.10.1
 livereload==2.5.1
-Markdown==3.0.1
+Markdown==3.2.1
 PyYAML==3.13
 tornado==4.1
 mdx_gh_links>=0.2

requirements/project.txt

@@ -1,7 +1,7 @@
 click>=7.0
 Jinja2>=2.10.3
 livereload>=2.6.1
-Markdown>=3.1.1
+Markdown>=3.2.1
 PyYAML>=5.2
 tornado>=5.1.1
 mdx_gh_links>=0.2

setup.py

@@ -58,7 +58,7 @@ setup(
         'Jinja2>=2.10.1',
         'livereload>=2.5.1',
         'lunr[languages]>=0.5.2',
-        'Markdown>=3.0.1',
+        'Markdown>=3.2.1',
         'PyYAML>=3.10',
         'tornado>=5.0'
     ],