diff --git a/doc/doc-support/package.nix b/doc/doc-support/package.nix index 728bbe334501..7a0993cfcea5 100644 --- a/doc/doc-support/package.nix +++ b/doc/doc-support/package.nix @@ -118,7 +118,7 @@ stdenvNoCC.mkDerivation ( --script ./anchor.min.js \ --script ./anchor-use.js \ --sidebar-depth 3 \ - --nav ./nav.json \ + --experimental-config ./nav.json \ --header ${./header.html}\ --no-navheader \ manual.md \ diff --git a/doc/nav.json b/doc/nav.json index 15d4926421cf..8c564c94294d 100644 --- a/doc/nav.json +++ b/doc/nav.json @@ -1,3 +1,4 @@ { - "open": [] + "open": [], + "items": [] } diff --git a/pkgs/by-name/ni/nixos-render-docs/src/nixos_render_docs/manual.py b/pkgs/by-name/ni/nixos-render-docs/src/nixos_render_docs/manual.py index 3f61608d60fa..551465ad4881 100644 --- a/pkgs/by-name/ni/nixos-render-docs/src/nixos_render_docs/manual.py +++ b/pkgs/by-name/ni/nixos-render-docs/src/nixos_render_docs/manual.py @@ -3,9 +3,9 @@ import hashlib import html import json import re -import xml.sax.saxutils as xml from abc import abstractmethod from collections.abc import Mapping, Sequence +from dataclasses import dataclass from pathlib import Path from typing import Any, Callable, ClassVar, Generic, NamedTuple, cast, get_args @@ -44,6 +44,7 @@ class BaseConverter(Converter[md.TR], Generic[md.TR]): self._current_type = ['book'] try: tokens = self._parse(infile.read_text()) + self._prepend_config(infile, tokens) self._postprocess(infile, outfile, tokens) converted = self._renderer.render(tokens) outfile.write_text(converted) @@ -53,6 +54,9 @@ class BaseConverter(Converter[md.TR], Generic[md.TR]): def _postprocess(self, infile: Path, outfile: Path, tokens: Sequence[Token]) -> None: pass + def _prepend_config(self, infile: Path, tokens: list[Token]) -> None: + pass + def _handle_headings(self, tokens: list[Token], *, src: str, on_heading: Callable[[Token,str],None]) -> None: # Headings in a globally numbered order # h1 to h6 @@ -232,6 +236,7 @@ class RendererMixin(Renderer): 'included_preface': lambda *args: self._included_thing("preface", *args), 'included_parts': lambda *args: self._included_thing("part", *args), 'included_appendix': lambda *args: self._included_thing("appendix", *args), + 'included_content': lambda *args: self._included_thing("content", *args), 'included_options': self.included_options, } @@ -264,7 +269,6 @@ class HTMLParameters(NamedTuple): # structural depth of the navigation sidebar tree sidebar_depth: int media_dir: Path - sidebar_open: frozenset[str] = frozenset() header: Path | None = None no_navheader: bool = False @@ -273,15 +277,18 @@ class ManualHTMLRenderer(RendererMixin, HTMLRenderer): _in_dir: Path _html_params: HTMLParameters _redirects: Redirects | None + _sidebar_open: frozenset[str] def __init__(self, toplevel_tag: str, revision: str, html_params: HTMLParameters, manpage_urls: Mapping[str, str], xref_targets: dict[str, XrefTarget], - redirects: Redirects | None, in_dir: Path, base_path: Path): + redirects: Redirects | None, in_dir: Path, base_path: Path, + sidebar_open: frozenset[str]): super().__init__(toplevel_tag, revision, manpage_urls, xref_targets) self._in_dir = in_dir self._base_path = base_path.absolute() self._html_params = html_params self._redirects = redirects + self._sidebar_open = sidebar_open def _pull_image(self, src: str) -> str: src_path = Path(src) @@ -542,7 +549,10 @@ document.addEventListener("DOMContentLoaded", createObserver); link = f'{e.target.toc_html}' cls = html.escape(e.kind, True) if children: - open_attr = " open" if e.target.id in self._html_params.sidebar_open else "" + # a group without an 'id' in the config gets open_id "". + # the empty key then fails the truth test, so the group stays closed. + key = e.target.id if e.open_id is None else e.open_id + open_attr = " open" if key and key in self._sidebar_open else "" items.append( f'
  • ' f'{link}{children}' @@ -607,6 +617,25 @@ document.addEventListener("DOMContentLoaded", createObserver); def _to_base26(n: int) -> str: return (_to_base26(n // 26) if n > 26 else "") + chr(ord("A") + n % 26) +@dataclass +class ConfigLeaf: + file: str + label: str | None = None + +@dataclass +class ConfigGroup: + label: str + children: list["ConfigNode"] + id: str | None = None + +ConfigNode = ConfigLeaf | ConfigGroup + +@dataclass +class ConfigManifest: + items: list[ConfigNode] + open: frozenset[str] + + class HTMLConverter(BaseConverter[ManualHTMLRenderer]): INCLUDE_ARGS_NS = "html" INCLUDE_FRAGMENT_ALLOWED_ARGS = { 'into-file' } @@ -618,24 +647,127 @@ class HTMLConverter(BaseConverter[ManualHTMLRenderer]): _xref_targets: dict[str, XrefTarget] _redirection_targets: set[str] _appendix_count: int = 0 + _config_path: Path | None + _config: ConfigManifest | None def _next_appendix_id(self) -> str: self._appendix_count += 1 return _to_base26(self._appendix_count - 1) - def __init__(self, revision: str, html_params: HTMLParameters, manpage_urls: Mapping[str, str], redirects: Redirects | None = None): + def __init__( + self, + revision: str, + html_params: HTMLParameters, + manpage_urls: Mapping[str, str], + redirects: Redirects | None = None, + config_path: Path | None = None + ): super().__init__() self._revision, self._html_params, self._manpage_urls, self._redirects = revision, html_params, manpage_urls, redirects + self._config_path = config_path + self._config = None self._xref_targets = {} self._redirection_targets = set() # renderer not set on purpose since it has a dependency on the output path! def convert(self, infile: Path, outfile: Path) -> None: + try: + self._config = self._load_config() + except Exception as e: + raise RuntimeError(f"failed to load config '{self._config_path}'") from e + sidebar_open = self._config.open if self._config is not None else frozenset() self._renderer = ManualHTMLRenderer( 'book', self._revision, self._html_params, self._manpage_urls, self._xref_targets, - self._redirects, infile.parent, outfile.parent) + self._redirects, infile.parent, outfile.parent, sidebar_open) super().convert(infile, outfile) + def _load_config(self) -> ConfigManifest | None: + if self._config_path is None: + return None + src = self._config_path.read_text() + config = json.loads(src) + if not isinstance(config, dict) or not isinstance(config.get('items'), list): + raise SrcError( + src=src, + description=f"config {self._config_path}: expected a top-level object with an 'items' array") + open_ids = config.get('open', []) + if not isinstance(open_ids, list) or not all(isinstance(i, str) for i in open_ids): + raise SrcError( + src=src, + description=f"config {self._config_path}: 'open' must be an array of strings") + items = self._parse_config_nodes(config['items'], "config items", src) + return ConfigManifest(items=items, open=frozenset(open_ids)) + + def _parse_config_nodes(self, items: Any, where: str, src: str) -> list[ConfigNode]: + return [self._parse_config_node(item, f"{where}[{idx}]", src) + for idx, item in enumerate(items)] + + def _parse_config_node(self, item: Any, where: str, src: str) -> ConfigNode: + if not isinstance(item, dict): + raise SrcError(src=src, description=f"{where}: expected an object, got {type(item).__name__}") + label = item.get('label') + if label is not None and (not isinstance(label, str) or not label): + raise SrcError(src=src, description=f"{where}: 'label' must be a non-empty string") + has_file, has_children = 'file' in item, 'children' in item + if has_file == has_children: + raise SrcError( + src=src, + description=f"{where}: must have exactly one of 'file' or 'children'") + if has_file: + if not isinstance(item['file'], str): + raise SrcError(src=src, description=f"{where}: 'file' must be a string") + return ConfigLeaf(file=item['file'], label=label) + children = item['children'] + if not isinstance(children, list) or not children: + raise SrcError(src=src, description=f"{where}: 'children' must be a non-empty array") + if not isinstance(label, str) or not label: + raise SrcError(src=src, description=f"{where}: a group requires a non-empty 'label'") + gid = item.get('id') + if gid is not None and (not isinstance(gid, str) or not gid): + raise SrcError(src=src, description=f"{where}: 'id' must be a non-empty string") + return ConfigGroup(label=label, children=self._parse_config_nodes(children, where, src), id=gid) + + def _prepend_config(self, infile: Path, tokens: list[Token]) -> None: + if self._config is None: + return + assert self._config_path is not None + include = self._build_config_include(self._config.items, self._config_path.resolve()) + # tokens 0 to 5 hold the title h1 triple and the subtitle h2 triple. + # _render_book renders tokens[6:] as the body. + # a change to the preamble must change this index too. + tokens[6:6] = [include] + + def _build_config_include(self, nodes: list[ConfigNode], config_file: Path) -> Token: + included = [self._build_config_item(node, config_file) for node in nodes] + token = Token('included_content', '', 0, map=[0, 1]) + token.meta['included'] = included + token.meta['include-args'] = {} + return token + + def _build_config_item(self, node: ConfigNode, config_file: Path) -> tuple[list[Token], Path]: + if isinstance(node, ConfigLeaf): + path = (config_file.parent / node.file).resolve() + leaf_src = path.read_text() + self._base_paths.append(path) + self._current_type.append('content') + try: + fragment = self._parse(leaf_src) + finally: + self._current_type.pop() + self._base_paths.pop() + if node.label is not None: + fragment[1].meta['toc-label'] = node.label + return fragment, path + + # TocEntry._collect_entries reads nav-label and nav-id. + # it builds the sidebar group from them. + group = self._build_config_include(node.children, config_file) + group.meta['nav-label'] = node.label + if node.id is not None: + group.meta['nav-id'] = node.id + + return [group], config_file + def _parse(self, src: str, *, auto_id_prefix: None | str = None) -> list[Token]: tokens = super()._parse(src,auto_id_prefix=auto_id_prefix) for token in tokens: @@ -747,10 +879,12 @@ class HTMLConverter(BaseConverter[ManualHTMLRenderer]): title_html = self._renderer.renderInline(inlines.children[0:1]) else: toc_html, title = title_html, title_html + if (toc_label := inlines.meta.get('toc-label')) is not None: + toc_html = html.escape(cast(str, toc_label)) title_html = ( f"{title_html}" if typ == 'chapter' - else title_html if typ in [ 'book', 'part' ] + else title_html if typ in [ 'book', 'part', 'content' ] else f'the section called “{title_html}”' ) return XrefTarget(id, title_html, toc_html, re.sub('<.*?>', '', title), path, drop_fragment) @@ -820,9 +954,14 @@ def _build_cli_html(p: argparse.ArgumentParser) -> None: p.add_argument('--media-dir', default="media", type=Path) p.add_argument('--redirects', type=Path) p.add_argument('--sidebar-depth', default=2, type=int) - # nav metadata (JSON): {"open": ["anchor-id", ...]} selects which sidebar - # entries render expanded; omitted or absent means everything is collapsed. - p.add_argument('--nav', type=Path) + p.add_argument('--experimental-config', type=Path, help=""" +JSON file of the following form +{ items: [ { file, label? } | { label, children: [...], id? } ], open: [ id, ... ] } + +Files added through this flag prepend '--infile' + +This flag will replace --infile and `{=include=}` directives in the future +""") # Deprecated flags, p.add_argument('--toc-depth', nargs='?', action=_DeprecatedDepthFlag, default=None) p.add_argument('--chunk-toc-depth', nargs='?', action=_DeprecatedDepthFlag, default=None) @@ -835,10 +974,6 @@ def _build_cli_html(p: argparse.ArgumentParser) -> None: p.add_argument('outfile', type=Path) def _run_cli_html(args: argparse.Namespace) -> None: - sidebar_open: frozenset[str] = frozenset() - if args.nav: - with open(args.nav) as nav_file: - sidebar_open = frozenset(json.load(nav_file).get("open", [])) with open(args.manpage_urls) as manpage_urls, open(Path(__file__).parent / "redirects.js") as redirects_script: redirects = None if args.redirects: @@ -848,8 +983,8 @@ def _run_cli_html(args: argparse.Namespace) -> None: md = HTMLConverter( args.revision, HTMLParameters(args.generator, args.stylesheet, args.script, - args.sidebar_depth, args.media_dir, sidebar_open, args.header, args.no_navheader), - json.load(manpage_urls), redirects) + args.sidebar_depth, args.media_dir, args.header, args.no_navheader), + json.load(manpage_urls), redirects, args.experimental_config) md.convert(args.infile, args.outfile) def build_cli(p: argparse.ArgumentParser) -> None: diff --git a/pkgs/by-name/ni/nixos-render-docs/src/nixos_render_docs/manual_structure.py b/pkgs/by-name/ni/nixos-render-docs/src/nixos_render_docs/manual_structure.py index 859ff5197d17..204b5aa9b933 100644 --- a/pkgs/by-name/ni/nixos-render-docs/src/nixos_render_docs/manual_structure.py +++ b/pkgs/by-name/ni/nixos-render-docs/src/nixos_render_docs/manual_structure.py @@ -13,8 +13,10 @@ from .utils import Freezeable # FragmentType is used to restrict structural include blocks. FragmentType = Literal['preface', 'part', 'chapter', 'section', 'appendix'] -# in the TOC all fragments are allowed, plus the all-encompassing book. -TocEntryType = Literal['book', 'preface', 'part', 'chapter', 'section', 'appendix', 'example', 'figure'] +# the TOC allows every fragment type. it also allows the enclosing book. +# it allows 'example' and 'figure'. +# --experimental-config adds a generic 'content' kind. +TocEntryType = Literal['book', 'preface', 'part', 'chapter', 'section', 'appendix', 'example', 'figure', 'content'] def is_include(token: Token) -> bool: return token.type == "fence" and token.info.startswith("{=include=} ") @@ -145,6 +147,7 @@ class TocEntry(Freezeable): next: TocEntry | None = None children: list[TocEntry] = dc.field(default_factory=list) starts_new_chunk: bool = False + open_id: str | None = None @property def root(self) -> TocEntry: @@ -193,9 +196,21 @@ class TocEntry(Freezeable): fragment_type_str = token.type[9:].removesuffix('s') assert fragment_type_str in get_args(TocEntryType) fragment_type = cast(TocEntryType, fragment_type_str) - for fragment, _path in included: - subentries = cls._collect_entries(xrefs, fragment, fragment_type) - entries[-1][1].children.append(subentries) + if (nav_label := token.meta.get('nav-label')) is not None: + children = [cls._collect_entries(xrefs, fragment, fragment_type) + for fragment, _path in included] + first = children[0].target + group = TocEntry( + fragment_type, + dc.replace(first, title_html=nav_label, toc_html=nav_label, title=nav_label), + children=children, + open_id=token.meta.get('nav-id', '')) + entries.append(('h1', group)) + token.meta['TocEntry'] = group + else: + for fragment, _path in included: + subentries = cls._collect_entries(xrefs, fragment, fragment_type) + entries[-1][1].children.append(subentries) elif token.type == 'heading_open' and (id := cast(str, token.attrs.get('id', ''))): while len(entries) > 1 and entries[-1][0] >= token.tag: entries[-2][1].children.append(entries.pop()[1]) diff --git a/pkgs/by-name/ni/nixos-render-docs/src/tests/test_html_header.py b/pkgs/by-name/ni/nixos-render-docs/src/tests/test_html_header.py index 7111959bea55..b0af6d8c4b99 100644 --- a/pkgs/by-name/ni/nixos-render-docs/src/tests/test_html_header.py +++ b/pkgs/by-name/ni/nixos-render-docs/src/tests/test_html_header.py @@ -9,7 +9,7 @@ def render(tmp_path: Path, header: Path | None) -> str: infile = tmp_path / "index.md" infile.write_text(SAMPLE_BOOK) outfile = tmp_path / "index.html" - params = HTMLParameters("", [], [], 2, sidebar_open = [], media_dir=tmp_path, header = header) + params = HTMLParameters("", [], [], 2, media_dir=tmp_path, header = header) HTMLConverter("1.0.0", params, {}).convert(infile, outfile) return outfile.read_text() diff --git a/pkgs/by-name/ni/nixos-render-docs/src/tests/test_manual.py b/pkgs/by-name/ni/nixos-render-docs/src/tests/test_manual.py index 45bce4cb6e78..5da16ab7659b 100644 --- a/pkgs/by-name/ni/nixos-render-docs/src/tests/test_manual.py +++ b/pkgs/by-name/ni/nixos-render-docs/src/tests/test_manual.py @@ -1,7 +1,9 @@ +import json import xml.parsers.expat as expat from html.entities import name2codepoint from pathlib import Path +import pytest from nixos_render_docs.manual import HTMLConverter, HTMLParameters @@ -49,7 +51,7 @@ def _build(tmp_path: Path, sidebar_depth: int = 2, sidebar_open: frozenset[str] out.mkdir(exist_ok=True) conv = HTMLConverter( "1.0.0", - HTMLParameters("test-gen", [], [], sidebar_depth, Path("media"), sidebar_open), + HTMLParameters("test-gen", [], [], sidebar_depth, Path("media")), {}, ) conv.convert(tmp_path / "index.md", out / "index.html") @@ -88,10 +90,42 @@ def test_sidebar_is_collapsible_tree(tmp_path: Path) -> None: def test_nav_metadata_opens_selected_entries(tmp_path: Path) -> None: - # ids listed in the nav "open" set render as
    - html = _build(tmp_path, sidebar_depth=3, sidebar_open=frozenset({"chap-fpa"})) - assert '
    Guides' in html + assert '
    None: + # the config lists 'intro' in 'open'. + # the group has no id, so its key is empty. the group stays closed. + html = _render_with_config( + tmp_path, + { + "items": [ + {"label": "Guides", "children": [ + {"file": "intro.md"}, + ]}, + ], + "open": ["intro"], + }, + {"intro.md": "# Introduction {#intro}\n\nBody.\n"}, + ) + assert '
    Guides' in html + assert "
    " not in html def test_sidebar_depth_caps_the_tree(tmp_path: Path) -> None: @@ -169,3 +203,179 @@ def test_output_is_well_formed_xhtml(tmp_path: Path) -> None: assert {p.name for p in pages} == {"index.html", "chapter.html"} for page in pages: _parse_xhtml(page.read_text()) + + +_DEFAULT_INDEX = ( + "# Test manual {#book-test}\n\n" + "## Version 1\n\n" + "```{=include=} chapters\nmanual-chapter.md\n```\n" +) +_DEFAULT_MANUAL_CHAPTER = "# Manual chapter {#manual-chap}\n\nManual body.\n" + + +def _render_with_config( + tmp_path: Path, + config: object, + files: dict[str, str], + *, + index: str = _DEFAULT_INDEX, + manual_chapter: str = _DEFAULT_MANUAL_CHAPTER, + sidebar_depth: int = 6, +) -> str: + (tmp_path / "config.json").write_text(json.dumps(config)) + (tmp_path / "manual-chapter.md").write_text(manual_chapter) + for name, content in files.items(): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + (tmp_path / "index.md").write_text(index) + out = tmp_path / "out" + out.mkdir(exist_ok=True) + + conv = HTMLConverter( + "1.0.0", + HTMLParameters("test-gen", [], [], sidebar_depth, Path("media")), + {}, + config_path=tmp_path / "config.json", + ) + conv.convert(tmp_path / "index.md", out / "index.html") + return (out / "index.html").read_text() + + +def test_config_content_renders_before_manual(tmp_path: Path) -> None: + html = _render_with_config( + tmp_path, + {"items": [{"label": "Intro", "file": "intro.md"}]}, + {"intro.md": "# Introduction {#intro}\n\nConfig intro body.\n"}, + ) + assert "Config intro body." in html + assert "Manual body." in html + assert html.index("Config intro body.") < html.index("Manual body.") + + +def test_config_grouping_node_is_sidebar_only(tmp_path: Path) -> None: + html = _render_with_config( + tmp_path, + { + "items": [ + { + "label": "Guides", "children": [ + {"label": "Install", "file": "install.md"}, + ] + } + ] + }, + {"install.md": "# Install {#install}\n\nInstall body.\n"}, + ) + # the group adds no heading and no anchor to the body. + assert "Guides" not in html + assert 'id="guides"' not in html + # the child renders as a page. + assert '

    Install' in html + assert "Install body." in html + # the sidebar nests the child under the group. + assert '
    Guides' in html + assert 'Install' in html + + +def test_config_leaf_derives_id_and_label_from_file(tmp_path: Path) -> None: + # the sidebar link reuses the heading id. the label reuses the heading title. + html = _render_with_config( + tmp_path, + {"items": [{"file": "intro.md"}]}, + {"intro.md": "# Introduction {#intro}\n\nBody.\n"}, + ) + assert '

    Introduction' in html + assert 'Introduction' in html + + +def test_config_leaf_label_differs_from_file_title(tmp_path: Path) -> None: + html = _render_with_config( + tmp_path, + { + "items": [ + {"label": "Sidebar Label", "file": "leaf1.md"} + ] + }, + {"leaf1.md": "# File Title {#leaf1}\n\nBody.\n"}, + ) + assert '

    File Title' in html + assert 'Sidebar Label' in html + + +def test_config_tree_nests_arbitrarily_deep(tmp_path: Path) -> None: + html = _render_with_config( + tmp_path, + { + "items": [ + {"label": "A", "children": [ + {"label": "B", "children": [ + {"label": "C", "file": "c.md"}, + ]}, + ]} + ] + }, + {"c.md": "# C File {#leaf-c}\n\nDeep body.\n"}, + ) + # only the leaf renders a heading. the groups A and B add no anchor. + assert '

    A' in html + assert '
    B' in html + assert 'C

  • ' in html + assert html.index('>A') < html.index('>B') < html.index('>C') + + +def test_config_id_is_cross_referenceable(tmp_path: Path) -> None: + html = _render_with_config( + tmp_path, + { + "items": [{"label": "Intro", "file": "intro.md"}] + }, + {"intro.md": "# Introduction {#intro}\n\nBody.\n"}, + manual_chapter="# Manual chapter {#manual-chap}\n\nSee [](#intro).\n", + ) + assert 'Introduction" in html + + +def test_config_duplicate_id_fails(tmp_path: Path) -> None: + with pytest.raises(RuntimeError) as excinfo: + _render_with_config( + tmp_path, + { + "items": [{"label": "Dup", "file": "dup.md"}] + }, + {"dup.md": "# Dup file {#dup}\n\nBody.\n"}, + manual_chapter="# Manual {#dup}\n\nBody.\n", + ) + assert "duplicate id" in str(excinfo.value.__cause__) + + +def test_config_malformed_node_fails(tmp_path: Path) -> None: + with pytest.raises(RuntimeError) as excinfo: + _render_with_config( + tmp_path, + {"items": [{"label": "Bad", "file": "x.md", "children": []}]}, + {}, + ) + assert "exactly one of" in str(excinfo.value.__cause__) + + with pytest.raises(RuntimeError) as excinfo: + _render_with_config( + tmp_path, + {"items": [{"file": 123}]}, + {}, + ) + assert "'file' must be a string" in str(excinfo.value.__cause__) + + with pytest.raises(RuntimeError) as excinfo: + _render_with_config( + tmp_path, + {"items": [{"children": [{"file": "x.md"}]}]}, + {"x.md": "# X {#x}\n\nB.\n"}, + ) + assert "a group requires a non-empty 'label'" in str(excinfo.value.__cause__)