Follow Python-Markdown approach for getting the title from the first H1

This partly reverts changes in commit e755aaed7e as some of that functionality will be deprecated.
Several more edge cases are taken into account now.

Co-authored-by: Waylan Limberg <waylan.limberg@icloud.com>

mkdocs/structure/pages.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import copy
 import enum
 import logging
 import posixpath
@@ -20,6 +19,7 @@ from mkdocs import utils
 from mkdocs.structure import StructureItem
 from mkdocs.structure.toc import get_toc
 from mkdocs.utils import _removesuffix, get_build_date, get_markdown_title, meta, weak_property
+from mkdocs.utils.rendering import get_heading_text
 
 if TYPE_CHECKING:
     from xml.etree import ElementTree as etree
@@ -555,23 +555,13 @@ class _ExtractTitleTreeprocessor(markdown.treeprocessors.Treeprocessor):
     def run(self, root: etree.Element) -> etree.Element:
         for el in root:
             if el.tag == 'h1':
-                # Drop anchorlink from the element, if present.
-                if len(el) > 0 and el[-1].tag == 'a' and not (el[-1].tail or '').strip():
-                    el = copy.copy(el)
-                    del el[-1]
-                # Extract the text only, recursively.
-                title = ''.join(el.itertext())
-                # Unescape per Markdown implementation details.
-                title = markdown.extensions.toc.stashedHTML2text(
-                    title, self.md, strip_entities=False
-                )
-                self.title = title.strip()
+                self.title = get_heading_text(el, self.md)
             break
         return root
 
     def _register(self, md: markdown.Markdown) -> None:
         self.md = md
-        md.treeprocessors.register(self, "mkdocs_extract_title", priority=-1)  # After the end.
+        md.treeprocessors.register(self, "mkdocs_extract_title", priority=1)  # Close to the end.
 
 
 class _AbsoluteLinksValidationValue(enum.IntEnum):

mkdocs/tests/structure/page_tests.py

@@ -342,6 +342,12 @@ class PageTests(unittest.TestCase):
             expected='Welcome to MkDocs Setext',
         )
 
+    def test_page_title_from_markdown_with_email(self):
+        self._test_extract_title(
+            '''# <foo@example.org>''',
+            expected='&#102;&#111;&#111;&#64;&#101;&#120;&#97;&#109;&#112;&#108;&#101;&#46;&#111;&#114;&#103;',
+        )
+
     def test_page_title_from_markdown_stripped_anchorlinks(self):
         self._test_extract_title(
             self._SETEXT_CONTENT,
@@ -349,6 +355,24 @@ class PageTests(unittest.TestCase):
             expected='Welcome to MkDocs Setext',
         )
 
+    def test_page_title_from_markdown_strip_footnoteref(self):
+        foootnotes = '''\n\n[^1]: foo\n[^2]: bar'''
+        self._test_extract_title(
+            '''# Header[^1] foo[^2] bar''' + foootnotes,
+            extensions={'footnotes': {}},
+            expected='Header foo bar',
+        )
+        self._test_extract_title(
+            '''# *Header[^1]* *foo*[^2]''' + foootnotes,
+            extensions={'footnotes': {}},
+            expected='Header foo',
+        )
+        self._test_extract_title(
+            '''# *Header[^1][^2]s''' + foootnotes,
+            extensions={'footnotes': {}},
+            expected='*Headers',
+        )
+
     def test_page_title_from_markdown_strip_formatting(self):
         self._test_extract_title(
             '''# \\*Hello --- *beautiful* `wor<dl>`''',
@@ -356,11 +380,15 @@ class PageTests(unittest.TestCase):
             expected='*Hello &mdash; beautiful wor&lt;dl&gt;',
         )
 
+    def test_page_title_from_markdown_html_entity(self):
+        self._test_extract_title('''# Foo &lt; &amp; bar''', expected='Foo &lt; &amp; bar')
+        self._test_extract_title('''# Foo > & bar''', expected='Foo &gt; &amp; bar')
+
     def test_page_title_from_markdown_strip_raw_html(self):
-        self._test_extract_title(
-            '''# Hello <b>world</b>''',
-            expected='Hello world',
-        )
+        self._test_extract_title('''# Hello <b>world</b>''', expected='Hello world')
+
+    def test_page_title_from_markdown_strip_comments(self):
+        self._test_extract_title('''# foo <!-- comment with <em> --> bar''', expected='foo bar')
 
     def test_page_title_from_markdown_strip_image(self):
         self._test_extract_title(

mkdocs/utils/rendering.py

@@ -0,0 +1,79 @@
+import copy
+from typing import Callable
+from xml.etree import ElementTree as etree
+
+import markdown
+import markdown.treeprocessors
+
+# TODO: This will become unnecessary after min-versions have Markdown >=3.4
+_unescape: Callable[[str], str]
+try:
+    _unescape = markdown.treeprocessors.UnescapeTreeprocessor().unescape
+except AttributeError:
+    _unescape = lambda s: s
+
+# TODO: Most of this file will become unnecessary after https://github.com/Python-Markdown/markdown/pull/1441
+
+
+def get_heading_text(el: etree.Element, md: markdown.Markdown) -> str:
+    el = _remove_fnrefs(_remove_anchorlink(el))
+    return _strip_tags(_render_inner_html(el, md))
+
+
+def _strip_tags(text: str) -> str:
+    """Strip HTML tags and return plain text. Note: HTML entities are unaffected."""
+    # A comment could contain a tag, so strip comments first
+    while (start := text.find('<!--')) != -1 and (end := text.find('-->', start)) != -1:
+        text = text[:start] + text[end + 3 :]
+
+    while (start := text.find('<')) != -1 and (end := text.find('>', start)) != -1:
+        text = text[:start] + text[end + 1 :]
+
+    # Collapse whitespace
+    text = ' '.join(text.split())
+    return text
+
+
+def _render_inner_html(el: etree.Element, md: markdown.Markdown) -> str:
+    # The `UnescapeTreeprocessor` runs after `toc` extension so run here.
+    text = md.serializer(el)
+    text = _unescape(text)
+
+    # Strip parent tag
+    start = text.index('>') + 1
+    end = text.rindex('<')
+    text = text[start:end].strip()
+
+    for pp in md.postprocessors:
+        text = pp.run(text)
+    return text
+
+
+def _remove_anchorlink(el: etree.Element) -> etree.Element:
+    """Drop anchorlink from a copy of the element, if present."""
+    if len(el) > 0 and el[-1].tag == 'a' and el[-1].get('class') == 'headerlink':
+        el = copy.copy(el)
+        del el[-1]
+    return el
+
+
+def _remove_fnrefs(root: etree.Element) -> etree.Element:
+    """Remove footnote references from a copy of the element, if any are present."""
+    # If there are no `sup` elements, then nothing to do.
+    if next(root.iter('sup'), None) is None:
+        return root
+    root = copy.deepcopy(root)
+    # Find parent elements that contain `sup` elements.
+    for parent in root.iterfind('.//sup/..'):
+        carry_text = ""
+        for child in reversed(parent):  # Reversed for the ability to mutate during iteration.
+            # Remove matching footnote references but carry any `tail` text to preceding elements.
+            if child.tag == 'sup' and child.get('id', '').startswith('fnref'):
+                carry_text = (child.tail or "") + carry_text
+                parent.remove(child)
+            elif carry_text:
+                child.tail = (child.tail or "") + carry_text
+                carry_text = ""
+        if carry_text:
+            parent.text = (parent.text or "") + carry_text
+    return root

pyproject.toml

@@ -36,7 +36,7 @@ dependencies = [
     "click >=7.0",
     "Jinja2 >=2.11.1",
     "markupsafe >=2.0.1",
-    "Markdown >=3.4.1",
+    "Markdown >=3.3.6",
     "PyYAML >=5.1",
     "watchdog >=2.0",
     "ghp-import >=1.0",
@@ -57,7 +57,7 @@ min-versions = [
     "click ==7.0",
     "Jinja2 ==2.11.1",
     "markupsafe ==2.0.1",
-    "Markdown ==3.4.1",
+    "Markdown ==3.3.6",
     "PyYAML ==5.1",
     "watchdog ==2.0",
     "ghp-import ==1.0",