pkgs/nixos-render-docs: add prepend content via 'experimental-config'

This new mode is supposed to absorb content that is currently added via --infile and recursive {=include=}
As we restructure the nixpkgs manual we are going to add pages to this file
As a result we end up with a config file that describes the sidebar structure and what files map to which entry
This is needed for 'docs.nixos.org' navigation cutover; which will consume this file along with the .md files and render it into the portal
This commit is contained in:
Johannes Kirschbauer
2026-07-12 15:55:57 +02:00
parent 651840b86d
commit f34ca9940a
6 changed files with 390 additions and 29 deletions

View File

@@ -118,7 +118,7 @@ stdenvNoCC.mkDerivation (
--script ./anchor.min.js \
--script ./anchor-use.js \
--sidebar-depth 3 \
--nav ./nav.json \
--experimental-config ./nav.json \
--header ${./header.html}\
--no-navheader \
manual.md \

View File

@@ -1,3 +1,4 @@
{
"open": []
"open": [],
"items": []
}

View File

@@ -3,9 +3,9 @@ import hashlib
import html
import json
import re
import xml.sax.saxutils as xml
from abc import abstractmethod
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, ClassVar, Generic, NamedTuple, cast, get_args
@@ -44,6 +44,7 @@ class BaseConverter(Converter[md.TR], Generic[md.TR]):
self._current_type = ['book']
try:
tokens = self._parse(infile.read_text())
self._prepend_config(infile, tokens)
self._postprocess(infile, outfile, tokens)
converted = self._renderer.render(tokens)
outfile.write_text(converted)
@@ -53,6 +54,9 @@ class BaseConverter(Converter[md.TR], Generic[md.TR]):
def _postprocess(self, infile: Path, outfile: Path, tokens: Sequence[Token]) -> None:
pass
def _prepend_config(self, infile: Path, tokens: list[Token]) -> None:
pass
def _handle_headings(self, tokens: list[Token], *, src: str, on_heading: Callable[[Token,str],None]) -> None:
# Headings in a globally numbered order
# h1 to h6
@@ -232,6 +236,7 @@ class RendererMixin(Renderer):
'included_preface': lambda *args: self._included_thing("preface", *args),
'included_parts': lambda *args: self._included_thing("part", *args),
'included_appendix': lambda *args: self._included_thing("appendix", *args),
'included_content': lambda *args: self._included_thing("content", *args),
'included_options': self.included_options,
}
@@ -264,7 +269,6 @@ class HTMLParameters(NamedTuple):
# structural depth of the navigation sidebar tree
sidebar_depth: int
media_dir: Path
sidebar_open: frozenset[str] = frozenset()
header: Path | None = None
no_navheader: bool = False
@@ -273,15 +277,18 @@ class ManualHTMLRenderer(RendererMixin, HTMLRenderer):
_in_dir: Path
_html_params: HTMLParameters
_redirects: Redirects | None
_sidebar_open: frozenset[str]
def __init__(self, toplevel_tag: str, revision: str, html_params: HTMLParameters,
manpage_urls: Mapping[str, str], xref_targets: dict[str, XrefTarget],
redirects: Redirects | None, in_dir: Path, base_path: Path):
redirects: Redirects | None, in_dir: Path, base_path: Path,
sidebar_open: frozenset[str]):
super().__init__(toplevel_tag, revision, manpage_urls, xref_targets)
self._in_dir = in_dir
self._base_path = base_path.absolute()
self._html_params = html_params
self._redirects = redirects
self._sidebar_open = sidebar_open
def _pull_image(self, src: str) -> str:
src_path = Path(src)
@@ -542,7 +549,10 @@ document.addEventListener("DOMContentLoaded", createObserver);
link = f'<a href="{e.target.href()}">{e.target.toc_html}</a>'
cls = html.escape(e.kind, True)
if children:
open_attr = " open" if e.target.id in self._html_params.sidebar_open else ""
# a group without an 'id' in the config gets open_id "".
# the empty key then fails the truth test, so the group stays closed.
key = e.target.id if e.open_id is None else e.open_id
open_attr = " open" if key and key in self._sidebar_open else ""
items.append(
f'<li class="{cls}">'
f'<details{open_attr}><summary>{link}</summary>{children}</details>'
@@ -607,6 +617,25 @@ document.addEventListener("DOMContentLoaded", createObserver);
def _to_base26(n: int) -> str:
return (_to_base26(n // 26) if n > 26 else "") + chr(ord("A") + n % 26)
@dataclass
class ConfigLeaf:
file: str
label: str | None = None
@dataclass
class ConfigGroup:
label: str
children: list["ConfigNode"]
id: str | None = None
ConfigNode = ConfigLeaf | ConfigGroup
@dataclass
class ConfigManifest:
items: list[ConfigNode]
open: frozenset[str]
class HTMLConverter(BaseConverter[ManualHTMLRenderer]):
INCLUDE_ARGS_NS = "html"
INCLUDE_FRAGMENT_ALLOWED_ARGS = { 'into-file' }
@@ -618,24 +647,127 @@ class HTMLConverter(BaseConverter[ManualHTMLRenderer]):
_xref_targets: dict[str, XrefTarget]
_redirection_targets: set[str]
_appendix_count: int = 0
_config_path: Path | None
_config: ConfigManifest | None
def _next_appendix_id(self) -> str:
self._appendix_count += 1
return _to_base26(self._appendix_count - 1)
def __init__(self, revision: str, html_params: HTMLParameters, manpage_urls: Mapping[str, str], redirects: Redirects | None = None):
def __init__(
self,
revision: str,
html_params: HTMLParameters,
manpage_urls: Mapping[str, str],
redirects: Redirects | None = None,
config_path: Path | None = None
):
super().__init__()
self._revision, self._html_params, self._manpage_urls, self._redirects = revision, html_params, manpage_urls, redirects
self._config_path = config_path
self._config = None
self._xref_targets = {}
self._redirection_targets = set()
# renderer not set on purpose since it has a dependency on the output path!
def convert(self, infile: Path, outfile: Path) -> None:
try:
self._config = self._load_config()
except Exception as e:
raise RuntimeError(f"failed to load config '{self._config_path}'") from e
sidebar_open = self._config.open if self._config is not None else frozenset()
self._renderer = ManualHTMLRenderer(
'book', self._revision, self._html_params, self._manpage_urls, self._xref_targets,
self._redirects, infile.parent, outfile.parent)
self._redirects, infile.parent, outfile.parent, sidebar_open)
super().convert(infile, outfile)
def _load_config(self) -> ConfigManifest | None:
if self._config_path is None:
return None
src = self._config_path.read_text()
config = json.loads(src)
if not isinstance(config, dict) or not isinstance(config.get('items'), list):
raise SrcError(
src=src,
description=f"config {self._config_path}: expected a top-level object with an 'items' array")
open_ids = config.get('open', [])
if not isinstance(open_ids, list) or not all(isinstance(i, str) for i in open_ids):
raise SrcError(
src=src,
description=f"config {self._config_path}: 'open' must be an array of strings")
items = self._parse_config_nodes(config['items'], "config items", src)
return ConfigManifest(items=items, open=frozenset(open_ids))
def _parse_config_nodes(self, items: Any, where: str, src: str) -> list[ConfigNode]:
return [self._parse_config_node(item, f"{where}[{idx}]", src)
for idx, item in enumerate(items)]
def _parse_config_node(self, item: Any, where: str, src: str) -> ConfigNode:
if not isinstance(item, dict):
raise SrcError(src=src, description=f"{where}: expected an object, got {type(item).__name__}")
label = item.get('label')
if label is not None and (not isinstance(label, str) or not label):
raise SrcError(src=src, description=f"{where}: 'label' must be a non-empty string")
has_file, has_children = 'file' in item, 'children' in item
if has_file == has_children:
raise SrcError(
src=src,
description=f"{where}: must have exactly one of 'file' or 'children'")
if has_file:
if not isinstance(item['file'], str):
raise SrcError(src=src, description=f"{where}: 'file' must be a string")
return ConfigLeaf(file=item['file'], label=label)
children = item['children']
if not isinstance(children, list) or not children:
raise SrcError(src=src, description=f"{where}: 'children' must be a non-empty array")
if not isinstance(label, str) or not label:
raise SrcError(src=src, description=f"{where}: a group requires a non-empty 'label'")
gid = item.get('id')
if gid is not None and (not isinstance(gid, str) or not gid):
raise SrcError(src=src, description=f"{where}: 'id' must be a non-empty string")
return ConfigGroup(label=label, children=self._parse_config_nodes(children, where, src), id=gid)
def _prepend_config(self, infile: Path, tokens: list[Token]) -> None:
if self._config is None:
return
assert self._config_path is not None
include = self._build_config_include(self._config.items, self._config_path.resolve())
# tokens 0 to 5 hold the title h1 triple and the subtitle h2 triple.
# _render_book renders tokens[6:] as the body.
# a change to the preamble must change this index too.
tokens[6:6] = [include]
def _build_config_include(self, nodes: list[ConfigNode], config_file: Path) -> Token:
included = [self._build_config_item(node, config_file) for node in nodes]
token = Token('included_content', '', 0, map=[0, 1])
token.meta['included'] = included
token.meta['include-args'] = {}
return token
def _build_config_item(self, node: ConfigNode, config_file: Path) -> tuple[list[Token], Path]:
if isinstance(node, ConfigLeaf):
path = (config_file.parent / node.file).resolve()
leaf_src = path.read_text()
self._base_paths.append(path)
self._current_type.append('content')
try:
fragment = self._parse(leaf_src)
finally:
self._current_type.pop()
self._base_paths.pop()
if node.label is not None:
fragment[1].meta['toc-label'] = node.label
return fragment, path
# TocEntry._collect_entries reads nav-label and nav-id.
# it builds the sidebar group from them.
group = self._build_config_include(node.children, config_file)
group.meta['nav-label'] = node.label
if node.id is not None:
group.meta['nav-id'] = node.id
return [group], config_file
def _parse(self, src: str, *, auto_id_prefix: None | str = None) -> list[Token]:
tokens = super()._parse(src,auto_id_prefix=auto_id_prefix)
for token in tokens:
@@ -747,10 +879,12 @@ class HTMLConverter(BaseConverter[ManualHTMLRenderer]):
title_html = self._renderer.renderInline(inlines.children[0:1])
else:
toc_html, title = title_html, title_html
if (toc_label := inlines.meta.get('toc-label')) is not None:
toc_html = html.escape(cast(str, toc_label))
title_html = (
f"<em>{title_html}</em>"
if typ == 'chapter'
else title_html if typ in [ 'book', 'part' ]
else title_html if typ in [ 'book', 'part', 'content' ]
else f'the section called “{title_html}'
)
return XrefTarget(id, title_html, toc_html, re.sub('<.*?>', '', title), path, drop_fragment)
@@ -820,9 +954,14 @@ def _build_cli_html(p: argparse.ArgumentParser) -> None:
p.add_argument('--media-dir', default="media", type=Path)
p.add_argument('--redirects', type=Path)
p.add_argument('--sidebar-depth', default=2, type=int)
# nav metadata (JSON): {"open": ["anchor-id", ...]} selects which sidebar
# entries render expanded; omitted or absent means everything is collapsed.
p.add_argument('--nav', type=Path)
p.add_argument('--experimental-config', type=Path, help="""
JSON file of the following form
{ items: [ { file, label? } | { label, children: [...], id? } ], open: [ id, ... ] }
Files added through this flag prepend '--infile'
This flag will replace --infile and `{=include=}` directives in the future
""")
# Deprecated flags,
p.add_argument('--toc-depth', nargs='?', action=_DeprecatedDepthFlag, default=None)
p.add_argument('--chunk-toc-depth', nargs='?', action=_DeprecatedDepthFlag, default=None)
@@ -835,10 +974,6 @@ def _build_cli_html(p: argparse.ArgumentParser) -> None:
p.add_argument('outfile', type=Path)
def _run_cli_html(args: argparse.Namespace) -> None:
sidebar_open: frozenset[str] = frozenset()
if args.nav:
with open(args.nav) as nav_file:
sidebar_open = frozenset(json.load(nav_file).get("open", []))
with open(args.manpage_urls) as manpage_urls, open(Path(__file__).parent / "redirects.js") as redirects_script:
redirects = None
if args.redirects:
@@ -848,8 +983,8 @@ def _run_cli_html(args: argparse.Namespace) -> None:
md = HTMLConverter(
args.revision,
HTMLParameters(args.generator, args.stylesheet, args.script,
args.sidebar_depth, args.media_dir, sidebar_open, args.header, args.no_navheader),
json.load(manpage_urls), redirects)
args.sidebar_depth, args.media_dir, args.header, args.no_navheader),
json.load(manpage_urls), redirects, args.experimental_config)
md.convert(args.infile, args.outfile)
def build_cli(p: argparse.ArgumentParser) -> None:

View File

@@ -13,8 +13,10 @@ from .utils import Freezeable
# FragmentType is used to restrict structural include blocks.
FragmentType = Literal['preface', 'part', 'chapter', 'section', 'appendix']
# in the TOC all fragments are allowed, plus the all-encompassing book.
TocEntryType = Literal['book', 'preface', 'part', 'chapter', 'section', 'appendix', 'example', 'figure']
# the TOC allows every fragment type. it also allows the enclosing book.
# it allows 'example' and 'figure'.
# --experimental-config adds a generic 'content' kind.
TocEntryType = Literal['book', 'preface', 'part', 'chapter', 'section', 'appendix', 'example', 'figure', 'content']
def is_include(token: Token) -> bool:
return token.type == "fence" and token.info.startswith("{=include=} ")
@@ -145,6 +147,7 @@ class TocEntry(Freezeable):
next: TocEntry | None = None
children: list[TocEntry] = dc.field(default_factory=list)
starts_new_chunk: bool = False
open_id: str | None = None
@property
def root(self) -> TocEntry:
@@ -193,9 +196,21 @@ class TocEntry(Freezeable):
fragment_type_str = token.type[9:].removesuffix('s')
assert fragment_type_str in get_args(TocEntryType)
fragment_type = cast(TocEntryType, fragment_type_str)
for fragment, _path in included:
subentries = cls._collect_entries(xrefs, fragment, fragment_type)
entries[-1][1].children.append(subentries)
if (nav_label := token.meta.get('nav-label')) is not None:
children = [cls._collect_entries(xrefs, fragment, fragment_type)
for fragment, _path in included]
first = children[0].target
group = TocEntry(
fragment_type,
dc.replace(first, title_html=nav_label, toc_html=nav_label, title=nav_label),
children=children,
open_id=token.meta.get('nav-id', ''))
entries.append(('h1', group))
token.meta['TocEntry'] = group
else:
for fragment, _path in included:
subentries = cls._collect_entries(xrefs, fragment, fragment_type)
entries[-1][1].children.append(subentries)
elif token.type == 'heading_open' and (id := cast(str, token.attrs.get('id', ''))):
while len(entries) > 1 and entries[-1][0] >= token.tag:
entries[-2][1].children.append(entries.pop()[1])

View File

@@ -9,7 +9,7 @@ def render(tmp_path: Path, header: Path | None) -> str:
infile = tmp_path / "index.md"
infile.write_text(SAMPLE_BOOK)
outfile = tmp_path / "index.html"
params = HTMLParameters("", [], [], 2, sidebar_open = [], media_dir=tmp_path, header = header)
params = HTMLParameters("", [], [], 2, media_dir=tmp_path, header = header)
HTMLConverter("1.0.0", params, {}).convert(infile, outfile)
return outfile.read_text()

View File

@@ -1,7 +1,9 @@
import json
import xml.parsers.expat as expat
from html.entities import name2codepoint
from pathlib import Path
import pytest
from nixos_render_docs.manual import HTMLConverter, HTMLParameters
@@ -49,7 +51,7 @@ def _build(tmp_path: Path, sidebar_depth: int = 2, sidebar_open: frozenset[str]
out.mkdir(exist_ok=True)
conv = HTMLConverter(
"1.0.0",
HTMLParameters("test-gen", [], [], sidebar_depth, Path("media"), sidebar_open),
HTMLParameters("test-gen", [], [], sidebar_depth, Path("media")),
{},
)
conv.convert(tmp_path / "index.md", out / "index.html")
@@ -88,10 +90,42 @@ def test_sidebar_is_collapsible_tree(tmp_path: Path) -> None:
def test_nav_metadata_opens_selected_entries(tmp_path: Path) -> None:
# ids listed in the nav "open" set render as <details open>
html = _build(tmp_path, sidebar_depth=3, sidebar_open=frozenset({"chap-fpa"}))
assert '<details open><summary><a href="#chap-fpa"' in html
assert '<details><summary><a href="#part-builders"' in html
# the 'open' array expands a config group and an --infile heading.
html = _render_with_config(
tmp_path,
{
"items": [
{"label": "Guides", "id": "guides", "children": [
{"label": "Intro", "file": "intro.md"},
]},
],
"open": ["guides", "manual-chap"],
},
{"intro.md": "# Introduction {#intro}\n\nBody.\n"},
manual_chapter="# Manual chapter {#manual-chap}\n\n## Sub {#manual-sub}\n\nText.\n",
)
# the group opens by its own id. its link points at the child.
assert '<details open><summary><a href="#intro">Guides</a>' in html
assert '<details open><summary><a href="#manual-chap"' in html
def test_config_group_without_id_is_not_openable(tmp_path: Path) -> None:
# the config lists 'intro' in 'open'.
# the group has no id, so its key is empty. the group stays closed.
html = _render_with_config(
tmp_path,
{
"items": [
{"label": "Guides", "children": [
{"file": "intro.md"},
]},
],
"open": ["intro"],
},
{"intro.md": "# Introduction {#intro}\n\nBody.\n"},
)
assert '<details><summary><a href="#intro">Guides</a>' in html
assert "<details open>" not in html
def test_sidebar_depth_caps_the_tree(tmp_path: Path) -> None:
@@ -169,3 +203,179 @@ def test_output_is_well_formed_xhtml(tmp_path: Path) -> None:
assert {p.name for p in pages} == {"index.html", "chapter.html"}
for page in pages:
_parse_xhtml(page.read_text())
_DEFAULT_INDEX = (
"# Test manual {#book-test}\n\n"
"## Version 1\n\n"
"```{=include=} chapters\nmanual-chapter.md\n```\n"
)
_DEFAULT_MANUAL_CHAPTER = "# Manual chapter {#manual-chap}\n\nManual body.\n"
def _render_with_config(
tmp_path: Path,
config: object,
files: dict[str, str],
*,
index: str = _DEFAULT_INDEX,
manual_chapter: str = _DEFAULT_MANUAL_CHAPTER,
sidebar_depth: int = 6,
) -> str:
(tmp_path / "config.json").write_text(json.dumps(config))
(tmp_path / "manual-chapter.md").write_text(manual_chapter)
for name, content in files.items():
path = tmp_path / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content)
(tmp_path / "index.md").write_text(index)
out = tmp_path / "out"
out.mkdir(exist_ok=True)
conv = HTMLConverter(
"1.0.0",
HTMLParameters("test-gen", [], [], sidebar_depth, Path("media")),
{},
config_path=tmp_path / "config.json",
)
conv.convert(tmp_path / "index.md", out / "index.html")
return (out / "index.html").read_text()
def test_config_content_renders_before_manual(tmp_path: Path) -> None:
html = _render_with_config(
tmp_path,
{"items": [{"label": "Intro", "file": "intro.md"}]},
{"intro.md": "# Introduction {#intro}\n\nConfig intro body.\n"},
)
assert "Config intro body." in html
assert "Manual body." in html
assert html.index("Config intro body.") < html.index("Manual body.")
def test_config_grouping_node_is_sidebar_only(tmp_path: Path) -> None:
html = _render_with_config(
tmp_path,
{
"items": [
{
"label": "Guides", "children": [
{"label": "Install", "file": "install.md"},
]
}
]
},
{"install.md": "# Install {#install}\n\nInstall body.\n"},
)
# the group adds no heading and no anchor to the body.
assert "Guides</h2>" not in html
assert 'id="guides"' not in html
# the child renders as a page.
assert '<h2 id="install" class="title" >Install' in html
assert "Install body." in html
# the sidebar nests the child under the group.
assert '<details><summary><a href="#install">Guides</a></summary>' in html
assert '<a href="#install">Install</a>' in html
def test_config_leaf_derives_id_and_label_from_file(tmp_path: Path) -> None:
# the sidebar link reuses the heading id. the label reuses the heading title.
html = _render_with_config(
tmp_path,
{"items": [{"file": "intro.md"}]},
{"intro.md": "# Introduction {#intro}\n\nBody.\n"},
)
assert '<h2 id="intro" class="title" >Introduction' in html
assert '<a href="#intro">Introduction</a>' in html
def test_config_leaf_label_differs_from_file_title(tmp_path: Path) -> None:
html = _render_with_config(
tmp_path,
{
"items": [
{"label": "Sidebar Label", "file": "leaf1.md"}
]
},
{"leaf1.md": "# File Title {#leaf1}\n\nBody.\n"},
)
assert '<h2 id="leaf1" class="title" >File Title' in html
assert '<a href="#leaf1">Sidebar Label</a>' in html
def test_config_tree_nests_arbitrarily_deep(tmp_path: Path) -> None:
html = _render_with_config(
tmp_path,
{
"items": [
{"label": "A", "children": [
{"label": "B", "children": [
{"label": "C", "file": "c.md"},
]},
]}
]
},
{"c.md": "# C File {#leaf-c}\n\nDeep body.\n"},
)
# only the leaf renders a heading. the groups A and B add no anchor.
assert '<h2 id="leaf-c" class="title"' in html
assert 'id="ga"' not in html and 'id="gb"' not in html
# the sidebar nests the groups.
# every group links to its first descendant page, #leaf-c.
assert '<details><summary><a href="#leaf-c">A</a></summary>' in html
assert '<details><summary><a href="#leaf-c">B</a></summary>' in html
assert '<a href="#leaf-c">C</a></li>' in html
assert html.index('>A</a>') < html.index('>B</a>') < html.index('>C</a>')
def test_config_id_is_cross_referenceable(tmp_path: Path) -> None:
html = _render_with_config(
tmp_path,
{
"items": [{"label": "Intro", "file": "intro.md"}]
},
{"intro.md": "# Introduction {#intro}\n\nBody.\n"},
manual_chapter="# Manual chapter {#manual-chap}\n\nSee [](#intro).\n",
)
assert '<a class="xref" href="#intro"' in html
assert ">Introduction</a>" in html
def test_config_duplicate_id_fails(tmp_path: Path) -> None:
with pytest.raises(RuntimeError) as excinfo:
_render_with_config(
tmp_path,
{
"items": [{"label": "Dup", "file": "dup.md"}]
},
{"dup.md": "# Dup file {#dup}\n\nBody.\n"},
manual_chapter="# Manual {#dup}\n\nBody.\n",
)
assert "duplicate id" in str(excinfo.value.__cause__)
def test_config_malformed_node_fails(tmp_path: Path) -> None:
with pytest.raises(RuntimeError) as excinfo:
_render_with_config(
tmp_path,
{"items": [{"label": "Bad", "file": "x.md", "children": []}]},
{},
)
assert "exactly one of" in str(excinfo.value.__cause__)
with pytest.raises(RuntimeError) as excinfo:
_render_with_config(
tmp_path,
{"items": [{"file": 123}]},
{},
)
assert "'file' must be a string" in str(excinfo.value.__cause__)
with pytest.raises(RuntimeError) as excinfo:
_render_with_config(
tmp_path,
{"items": [{"children": [{"file": "x.md"}]}]},
{"x.md": "# X {#x}\n\nB.\n"},
)
assert "a group requires a non-empty 'label'" in str(excinfo.value.__cause__)