Add sanity tests (#1)

.github/workflows/ci.yaml

@@ -23,7 +23,7 @@ jobs:
 
     - name: Graft ansible-core
       run: |
-        python docs/bin/clone-core.py devel
+        python docs/bin/clone-core.py
 
     - name: Run docs-build Sanity
       run: |
@@ -47,8 +47,8 @@ jobs:
 
     - name: Graft ansible-core
       run: |
-        python docs/bin/clone-core.py devel
+        python docs/bin/clone-core.py
 
     - name: Run rstcheck Sanity
       run: |
-        python tests/sanity.py rstcheck
+        python tests/sanity.py rstcheck

.gitignore

@@ -124,3 +124,9 @@ test/units/.coverage.*
 /SYMLINK_CACHE.json
 changelogs/.plugin-cache.yaml
 .ansible-test-timeout.json
+# ignore copied in files
+/MANIFEST.in
+/pyproject.toml
+/requirements.txt
+/setup.cfg
+/setup.py

docs/ansible-core-branch.txt

@@ -0,0 +1 @@
+devel

docs/bin/clone-core.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+"""Clone relevant portions of ansible-core from ansible/ansible into the current source tree to facilitate building docs."""
+
+from __future__ import annotations
+
+import pathlib
+import shutil
+import subprocess
+import sys
+import tempfile
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent.parent
+
+
+def main() -> None:
+    keep_dirs = [
+        'bin',
+        'lib',
+        'packaging',
+        'test/lib',
+        'test/sanity',
+    ]
+
+    keep_files = [
+        'MANIFEST.in',
+        'pyproject.toml',
+        'requirements.txt',
+        'setup.cfg',
+        'setup.py',
+    ]
+
+    branch = (ROOT / 'docs' / 'ansible-core-branch.txt').read_text().strip()
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        subprocess.run(['git', 'clone', 'https://github.com/ansible/ansible', '--depth=1', '-b', branch, temp_dir], check=True)
+
+        for keep_dir in keep_dirs:
+            src = pathlib.Path(temp_dir, keep_dir)
+            dst = pathlib.Path.cwd() / keep_dir
+
+            print(f'Updating {keep_dir!r} ...', file=sys.stderr, flush=True)
+
+            if dst.exists():
+                shutil.rmtree(dst)
+
+            shutil.copytree(src, dst, symlinks=True)
+
+            (dst / '.gitignore').write_text('*')
+
+        for keep_file in keep_files:
+            src = pathlib.Path(temp_dir, keep_file)
+            dst = pathlib.Path.cwd() / keep_file
+
+            print(f'Updating {keep_file!r} ...', file=sys.stderr, flush=True)
+
+            shutil.copyfile(src, dst)
+
+
+if __name__ == '__main__':
+    main()

tests/checkers/docs-build.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def main():
+    base_dir = os.getcwd()
+
+    keep_dirs = [
+        'bin',
+        'docs',
+        'examples',
+        'hacking',
+        'lib',
+        'packaging',
+        'test/lib',
+        'test/sanity',
+    ]
+
+    keep_files = [
+        'MANIFEST.in',
+        'pyproject.toml',
+        'requirements.txt',
+        'setup.cfg',
+        'setup.py',
+    ]
+
+    # The tests write to the source tree, which isn't permitted for sanity tests.
+    # To work around this a temporary copy is used.
+
+    current_dir = os.getcwd()
+
+    with tempfile.TemporaryDirectory(prefix='docs-build-', suffix='-sanity') as temp_dir:
+        for keep_dir in keep_dirs:
+            shutil.copytree(os.path.join(base_dir, keep_dir), os.path.join(temp_dir, keep_dir), symlinks=True)
+
+        for keep_file in keep_files:
+            shutil.copy2(os.path.join(base_dir, keep_file), os.path.join(temp_dir, keep_file))
+
+        paths = os.environ['PATH'].split(os.pathsep)
+        paths = [f'{temp_dir}/bin' if path == f'{current_dir}/bin' else path for path in paths]
+
+        # Fix up the environment so everything runs from the temporary copy.
+        os.environ['PATH'] = os.pathsep.join(paths)
+        os.environ['PYTHONPATH'] = f'{temp_dir}/lib'
+        os.chdir(temp_dir)
+
+        run_test()
+
+
+def run_test():
+    base_dir = os.getcwd() + os.path.sep
+    docs_dir = os.path.abspath('docs/docsite')
+
+    cmd = ['make', 'core_singlehtmldocs']
+    sphinx = subprocess.run(cmd, stdin=subprocess.DEVNULL, capture_output=True, cwd=docs_dir, check=False, text=True)
+
+    stdout = sphinx.stdout
+    stderr = sphinx.stderr
+
+    if sphinx.returncode != 0:
+        sys.stderr.write("Command '%s' failed with status code: %d\n" % (' '.join(cmd), sphinx.returncode))
+
+        if stdout.strip():
+            stdout = simplify_stdout(stdout)
+
+            sys.stderr.write("--> Standard Output\n")
+            sys.stderr.write("%s\n" % stdout.strip())
+
+        if stderr.strip():
+            sys.stderr.write("--> Standard Error\n")
+            sys.stderr.write("%s\n" % stderr.strip())
+
+        sys.exit(1)
+
+    with open('docs/docsite/rst_warnings', 'r') as warnings_fd:
+        output = warnings_fd.read().strip()
+        lines = output.splitlines()
+
+    known_warnings = {
+        'block-quote-missing-blank-line': r'^Block quote ends without a blank line; unexpected unindent.$',
+        'literal-block-lex-error': r'^Could not lex literal_block as "[^"]*". Highlighting skipped.$',
+        'duplicate-label': r'^duplicate label ',
+        'undefined-label': r'undefined label: ',
+        'unknown-document': r'unknown document: ',
+        'toc-tree-missing-document': r'toctree contains reference to nonexisting document ',
+        'reference-target-not-found': r'[^ ]* reference target not found: ',
+        'not-in-toc-tree': r"document isn't included in any toctree$",
+        'unexpected-indentation': r'^Unexpected indentation.$',
+        'definition-list-missing-blank-line': r'^Definition list ends without a blank line; unexpected unindent.$',
+        'explicit-markup-missing-blank-line': r'Explicit markup ends without a blank line; unexpected unindent.$',
+        'toc-tree-glob-pattern-no-match': r"^toctree glob pattern '[^']*' didn't match any documents$",
+        'unknown-interpreted-text-role': '^Unknown interpreted text role "[^"]*".$',
+    }
+
+    for line in lines:
+        match = re.search('^(?P<path>[^:]+):((?P<line>[0-9]+):)?((?P<column>[0-9]+):)? (?P<level>WARNING|ERROR): (?P<message>.*)$', line)
+
+        if not match:
+            path = 'docs/docsite/rst/index.rst'
+            lineno = 0
+            column = 0
+            code = 'unknown'
+            message = line
+
+            # surface unknown lines while filtering out known lines to avoid excessive output
+            print('%s:%d:%d: %s: %s' % (path, lineno, column, code, message))
+            continue
+
+        path = match.group('path')
+        lineno = int(match.group('line') or 0)
+        column = int(match.group('column') or 0)
+        level = match.group('level').lower()
+        message = match.group('message')
+
+        path = os.path.abspath(path)
+
+        if path.startswith(base_dir):
+            path = path[len(base_dir):]
+
+        if path.startswith('rst/'):
+            path = 'docs/docsite/' + path  # fix up paths reported relative to `docs/docsite/`
+
+        if level == 'warning':
+            code = 'warning'
+
+            for label, pattern in known_warnings.items():
+                if re.search(pattern, message):
+                    code = label
+                    break
+        else:
+            code = 'error'
+
+        print('%s:%d:%d: %s: %s' % (path, lineno, column, code, message))
+
+
+def simplify_stdout(value):
+    """Simplify output by omitting earlier 'rendering: ...' messages."""
+    lines = value.strip().splitlines()
+
+    rendering = []
+    keep = []
+
+    def truncate_rendering():
+        """Keep last rendering line (if any) with a message about omitted lines as needed."""
+        if not rendering:
+            return
+
+        notice = rendering[-1]
+
+        if len(rendering) > 1:
+            notice += ' (%d previous rendering line(s) omitted)' % (len(rendering) - 1)
+
+        keep.append(notice)
+        # Could change to rendering.clear() if we do not support python2
+        rendering[:] = []
+
+    for line in lines:
+        if line.startswith('rendering: '):
+            rendering.append(line)
+            continue
+
+        truncate_rendering()
+        keep.append(line)
+
+    truncate_rendering()
+
+    result = '\n'.join(keep)
+
+    return result
+
+
+if __name__ == '__main__':
+    main()

tests/checkers/rstcheck.json

@@ -0,0 +1,5 @@
+{
+    "extensions": [
+        ".rst"
+    ]
+}

tests/checkers/rstcheck.py

@@ -0,0 +1,62 @@
+"""Sanity test using rstcheck and sphinx."""
+from __future__ import annotations
+
+import re
+import subprocess
+import sys
+
+
+def main():
+    paths = sys.argv[1:] or sys.stdin.read().splitlines()
+
+    encoding = 'utf-8'
+
+    ignore_substitutions = (
+        'br',
+    )
+
+    cmd = [
+        sys.executable,
+        '-c', 'import rstcheck; rstcheck.main();',
+        '--report', 'warning',
+        '--ignore-substitutions', ','.join(ignore_substitutions),
+    ] + paths
+
+    process = subprocess.run(cmd,
+                             stdin=subprocess.DEVNULL,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE,
+                             check=False,
+                             )
+
+    if process.stdout:
+        raise Exception(process.stdout)
+
+    pattern = re.compile(r'^(?P<path>[^:]*):(?P<line>[0-9]+): \((?P<level>INFO|WARNING|ERROR|SEVERE)/[0-4]\) (?P<message>.*)$')
+
+    results = parse_to_list_of_dict(pattern, process.stderr.decode(encoding))
+
+    for result in results:
+        print('%s:%s:%s: %s' % (result['path'], result['line'], 0, result['message']))
+
+
+def parse_to_list_of_dict(pattern, value):
+    matched = []
+    unmatched = []
+
+    for line in value.splitlines():
+        match = re.search(pattern, line)
+
+        if match:
+            matched.append(match.groupdict())
+        else:
+            unmatched.append(line)
+
+    if unmatched:
+        raise Exception('Pattern "%s" did not match values:\n%s' % (pattern, '\n'.join(unmatched)))
+
+    return matched
+
+
+if __name__ == '__main__':
+    main()

tests/requirements.txt

@@ -0,0 +1,8 @@
+jinja2
+pyyaml
+resolvelib < 1.1.0
+sphinx == 5.3.0
+sphinx-notfound-page
+sphinx-ansible-theme
+rstcheck < 6  # rstcheck 6.x has problem with rstcheck.core triggered by include files w/ sphinx directives https://github.com/rstcheck/rstcheck-core/issues/3
+antsibull-docs == 2.0.0  # currently approved version

tests/sanity.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+"""Simple test runner."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import pathlib
+import subprocess
+import sys
+
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('test', nargs='+')
+
+    args = parser.parse_args()
+    tests: list[str] = args.test
+    failed = False
+
+    for test in tests:
+        if run_test(test):
+            failed = True
+
+    if failed:
+        sys.exit(1)
+
+
+def run_test(name: str) -> bool:
+    print(f'Running {name!r} checker ...', file=sys.stderr, flush=True)
+
+    checker_path = ROOT / 'tests' / 'checkers' / f'{name}.py'
+    checker_json = checker_path.with_suffix('.json')
+
+    try:
+        config = json.loads(checker_json.read_text())
+    except FileNotFoundError:
+        config = {}
+
+    paths = []
+    extensions = set(config.get('extensions', []))
+
+    for root, dir_names, file_names in os.walk(ROOT / 'docs'):
+        for file_name in file_names:
+            path = os.path.join(root, file_name)
+            ext = os.path.splitext(path)[1]
+
+            if ext in extensions:
+                paths.append(path)
+
+    cmd = [sys.executable, checker_path] + paths
+
+    try:
+        result = subprocess.run(cmd, check=True, capture_output=True, text=True)
+    except subprocess.CalledProcessError as ex:
+        print(ex, file=sys.stderr, flush=True)
+        result = ex
+
+    sys.stdout.write(result.stdout)
+    sys.stderr.write(result.stderr)
+
+    return bool(result.stdout or result.stderr)
+
+
+if __name__ == '__main__':
+    main()