nathan/nixpkgs
1
0
Fork 0
mirror of https://github.com/NixOS/nixpkgs.git synced 2026-06-05 21:03:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

nixos/etc: clear stale opaque markers from mutable overlay upperdir (#507963)

Browse Source
This commit is contained in:
Ramses
2026-05-13 16:36:09 +00:00
committed by GitHub
parent 4de46ba6f7 1c7c4d2291
commit 5d8dc420a0
10 changed files with 251 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
nixos/modules/system/etc/etc-activation.nix
View File

@@ -1,7 +1,6 @@
{
config,
lib,
pkgs,
...
}:
@@ -51,6 +50,9 @@
];
boot.initrd.systemd = {
storePaths = lib.mkIf config.system.etc.overlay.mutable [
"${config.system.nixos-init.package}/bin/clear-etc-opaque"
];
mounts = [
{
where = "/run/nixos-etc-metadata";
@@ -131,13 +133,20 @@
before = [ "initrd-fs.target" ];
unitConfig = {
DefaultDependencies = false;
RequiresMountsFor = "/sysroot";
RequiresMountsFor = [
"/sysroot"
# Needed so we can clear stale opaque markers from the
# upperdir based on the contents of the new metadata layer
# before the overlay is mounted.
"/run/nixos-etc-metadata"
];
};
serviceConfig = {
Type = "oneshot";
ExecStart = ''
/bin/mkdir -p -m 0755 /sysroot/.rw-etc/upper /sysroot/.rw-etc/work
'';
ExecStart = [
"/bin/mkdir -p -m 0755 /sysroot/.rw-etc/upper /sysroot/.rw-etc/work"
"${config.system.nixos-init.package}/bin/clear-etc-opaque /run/nixos-etc-metadata /sysroot/.rw-etc/upper"
];
};
};
})

7
nixos/modules/system/etc/etc.nix
View File

@@ -285,6 +285,13 @@ in
tmpMetadataMount=$(TMPDIR="/run" mktemp --directory -t nixos-etc-metadata.XXXXXXXXXX)
mount --type erofs --options ro,nodev,nosuid ${config.system.build.etcMetadataImage} "$tmpMetadataMount"
${lib.optionalString config.system.etc.overlay.mutable ''
# Clear stale opaque markers from the upperdir so that lowerdir
# entries added by the new generation are not hidden.
# See https://github.com/NixOS/nixpkgs/issues/505475
${config.system.nixos-init.package}/bin/clear-etc-opaque "$tmpMetadataMount" /.rw-etc/upper
''}
# There was no previous /etc mounted. This happens when we're called
# directly without an initrd, like with nixos-enter.
if ! mountpoint -q /etc; then

40
nixos/tests/activation/etc-overlay-mutable.nix
View File

@@ -27,6 +27,12 @@
specialisation.new-generation.configuration = {
environment.etc."newgen".text = "newgen";
# Regression test for https://github.com/NixOS/nixpkgs/issues/505475:
# A symlink in a subdirectory that does not exist in the base generation's
# lowerdir. If something creates that subdirectory at runtime before
# switching (e.g. stage-2-init.sh creating /etc/nixos), overlayfs makes it
# opaque, hiding lowerdir content added by the new generation.
environment.etc."nixos/newlink".source = pkgs.emptyDirectory;
};
specialisation.newer-generation.configuration = {
environment.etc."newergen".text = "newergen";
@@ -53,6 +59,13 @@
machine.succeed("stat --format '%F' /etc/modetest2 | tee /dev/stderr | grep -Eq '^regular file$'")
machine.succeed("stat --format '%a' /etc/modetest2 | tee /dev/stderr | grep -Eq '^300$'")
with subtest("/etc/nixos created by stage-2-init is opaque in upperdir"):
# stage-2-init.sh unconditionally runs `install -d /etc/nixos`. Since
# /nixos is not in the lowerdir, overlayfs creates it as an opaque dir
# in the upperdir. Verify this precondition for the regression test below.
machine.succeed("test -d /.rw-etc/upper/nixos")
print(machine.succeed("getfattr -h -d -m 'trusted.overlay' /.rw-etc/upper/nixos 2>&1 || true"))
with subtest("switching to the same generation"):
machine.succeed("/run/current-system/bin/switch-to-configuration test")
@@ -77,6 +90,15 @@
assert machine.succeed("cat /etc/newgen") == "newgen"
assert machine.succeed("cat /etc/mutable") == "mutable"
# Regression test for https://github.com/NixOS/nixpkgs/issues/505475:
# The opaque /etc/nixos in the upperdir (created by stage-2-init.sh
# before /nixos existed in the lowerdir) must not hide lowerdir entries
# added by the new generation. The activation script must have cleared
# the stale opaque marker.
print(machine.succeed("ls -la /etc/nixos/"))
machine.succeed("test -L /etc/nixos/newlink")
machine.fail("getfattr -h -n trusted.overlay.opaque /.rw-etc/upper/nixos")
print(machine.succeed("findmnt /etc/mountpoint"))
print(machine.succeed("stat /etc/mountpoint/extra-file"))
print(machine.succeed("findmnt /etc/filemount"))
@@ -93,5 +115,23 @@
numOfMetaMounts = len(metaMounts.splitlines())
assert numOfTmpMounts == 0, f"Found {numOfTmpMounts} remaining tmpmounts"
assert numOfMetaMounts == 1, f"Found {numOfMetaMounts} remaining metamounts"
with subtest("stale opaque markers are cleared by initrd on boot (NixOS/nixpkgs#505475)"):
# Simulate the bug precondition: an opaque /pam.d in the upperdir.
# /pam.d is guaranteed to exist as a directory in the metadata layer.
machine.succeed("mkdir -p /.rw-etc/upper/pam.d")
machine.succeed("setfattr -h -n trusted.overlay.opaque -v y /.rw-etc/upper/pam.d")
machine.succeed("getfattr -h -n trusted.overlay.opaque /.rw-etc/upper/pam.d")
# Also create a non-opaque upperdir directory that exists in the
# metadata layer, to ensure clear-etc-opaque tolerates the
# already-clear case.
machine.succeed("mkdir -p /.rw-etc/upper/systemd")
# Reboot and verify the initrd rw-etc service cleared the opaque marker.
machine.shutdown()
machine.start()
machine.wait_for_unit("multi-user.target")
machine.fail("getfattr -h -n trusted.overlay.opaque /.rw-etc/upper/pam.d")
machine.succeed("test -e /etc/pam.d/login")
'';
}

11
pkgs/by-name/ni/nixos-init/Cargo.lock generated
View File

@@ -142,6 +142,7 @@ dependencies = [
"serde",
"serde_json",
"tempfile",
"xattr",
]
[[package]]
@@ -506,3 +507,13 @@ checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
dependencies = [
"bitflags",
]
[[package]]
name = "xattr"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
dependencies = [
"libc",
"rustix",
]

1
pkgs/by-name/ni/nixos-init/Cargo.toml
View File

@@ -11,6 +11,7 @@ pathrs = "0.2.2"
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.145"
bootspec = "2.0.0"
xattr = "1.6.1"
[dev-dependencies]
tempfile = "3.20.0"

3
pkgs/by-name/ni/nixos-init/README.md
View File

@@ -52,6 +52,9 @@ closure. Currently nixos-init comes in at ~500 KiB.
- `find-etc`: Finds the `/etc` paths in `/sysroot` so that the initrd doesn't
directly depend on the toplevel, reducing the need to rebuild the initrd on
every generation.
- `clear-etc-opaque`: Clears stale `trusted.overlay.opaque` xattrs from the
mutable `/etc` overlay's upperdir before it is mounted, so that lowerdir
entries added by a new generation are not hidden.
- `resolve-in-root`: Figures out the canonical path inside a chroot.
## Future

1
pkgs/by-name/ni/nixos-init/package.nix
View File

@@ -47,6 +47,7 @@ rustPlatform.buildRustPackage (finalAttrs: {
binaries = [
"initrd-init"
"find-etc"
"clear-etc-opaque"
"resolve-in-root"
"env-generator"
];

170
pkgs/by-name/ni/nixos-init/src/etc_overlay.rs Normal file
View File

@@ -0,0 +1,170 @@
use std::{
env, fs,
path::{Path, PathBuf},
};
use anyhow::{Context, Result, bail};
const OVERLAY_OPAQUE_XATTR: &str = "trusted.overlay.opaque";
/// Entrypoint for the `clear-etc-opaque` binary.
///
/// When a directory is created in the mutable `/etc` overlay that does not yet
/// exist in the lowerdir, overlayfs marks it opaque in the upperdir. This is
/// correct at creation time, but becomes stale when a later generation adds
/// entries under that same directory to the metadata layer: the opaque marker
/// hides them.
///
/// This walks the (newly mounted) metadata layer and removes
/// `trusted.overlay.opaque` from any upperdir directory that now has a
/// directory counterpart in the lowerdir, turning it back into a merged view.
/// Files the user placed in the upperdir remain visible (upperdir wins
/// per-entry) and individual whiteouts are preserved; only the blanket hiding
/// of lowerdir content is undone.
///
/// See <https://github.com/NixOS/nixpkgs/issues/505475>.
///
/// Usage: `clear-etc-opaque <metadata-mount> <upperdir>`
pub fn clear_etc_opaque() -> Result<()> {
let args: Vec<String> = env::args().collect();
if args.len() != 3 {
bail!("Usage: {} <metadata-mount> <upperdir>", args[0]);
}
let metadata_mount = PathBuf::from(&args[1]);
let upperdir = PathBuf::from(&args[2]);
if !upperdir.is_dir() {
// Nothing to clear (e.g. first boot before the upperdir is created).
log::info!(
"Upperdir {} does not exist, nothing to clear.",
upperdir.display()
);
return Ok(());
}
clear_opaque_markers(&metadata_mount, &metadata_mount, &upperdir)
}
/// Recursively walk `current` (a subtree of `metadata_root`) and clear the
/// opaque xattr from the corresponding directory in `upperdir`.
fn clear_opaque_markers(metadata_root: &Path, current: &Path, upperdir: &Path) -> Result<()> {
let entries = fs::read_dir(current)
.with_context(|| format!("Failed to read directory {}", current.display()))?;
for entry in entries {
let entry =
entry.with_context(|| format!("Failed to read entry in {}", current.display()))?;
// Use the entry's own type info (no symlink following) so we only
// recurse into real directories of the metadata image.
if !entry
.file_type()
.with_context(|| format!("Failed to stat {}", entry.path().display()))?
.is_dir()
{
continue;
}
let path = entry.path();
let rel = path
.strip_prefix(metadata_root)
.context("Failed to strip metadata root prefix")?;
let target = upperdir.join(rel);
// Only act on real directories in the upperdir; an opaque marker on a
// non-directory would be meaningless and we must not follow symlinks
// out of the upperdir.
match fs::symlink_metadata(&target) {
Ok(meta) if meta.is_dir() => {
remove_opaque_xattr(&target);
// Only recurse when the upperdir also has this directory:
// deeper lowerdir directories without an upperdir counterpart
// cannot carry stale markers.
clear_opaque_markers(metadata_root, &path, upperdir)?;
}
// Missing or not a directory: nothing to do for this subtree.
_ => {}
}
}
Ok(())
}
/// Remove the `trusted.overlay.opaque` xattr from `path` if present.
fn remove_opaque_xattr(path: &Path) {
// Check first instead of removing unconditionally: lremovexattr(2) reports
// a missing attribute as ENODATA, which std does not map to a stable
// io::ErrorKind, so distinguishing it from real errors is awkward.
match xattr::get(path, OVERLAY_OPAQUE_XATTR) {
Ok(None) => return,
Ok(Some(_)) => {}
Err(err) => {
log::warn!(
"Failed to read {OVERLAY_OPAQUE_XATTR} on {}: {err}.",
path.display()
);
return;
}
}
match xattr::remove(path, OVERLAY_OPAQUE_XATTR) {
Ok(()) => {
log::info!("Cleared stale opaque marker from {}.", path.display());
}
Err(err) => {
// Don't abort the boot over this; the worst case is that some
// declaratively-managed /etc entries stay hidden, which is what
// would happen anyway without this fixup.
log::warn!(
"Failed to remove {OVERLAY_OPAQUE_XATTR} from {}: {err}.",
path.display()
);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
#[test]
fn clears_opaque_only_for_matching_dirs() -> Result<()> {
if !xattr::SUPPORTED_PLATFORM {
return Ok(());
}
let metadata = tempdir()?;
let upper = tempdir()?;
// lowerdir gained nixos/sub in the new generation.
fs::create_dir_all(metadata.path().join("nixos/sub"))?;
// upperdir has an opaque nixos/ from before.
fs::create_dir_all(upper.path().join("nixos"))?;
// upperdir directory without a lowerdir counterpart: we only clear
// markers where the lowerdir has a matching directory, so this one
// must stay opaque.
fs::create_dir_all(upper.path().join("only-upper"))?;
// The build sandbox usually lacks CAP_SYS_ADMIN, so trusted.* xattrs
// cannot be set. Skip in that case rather than fail the build.
if xattr::set(upper.path().join("nixos"), OVERLAY_OPAQUE_XATTR, b"y").is_err() {
eprintln!("skipping: cannot set trusted.* xattrs in this environment");
return Ok(());
}
xattr::set(upper.path().join("only-upper"), OVERLAY_OPAQUE_XATTR, b"y")?;
clear_opaque_markers(metadata.path(), metadata.path(), upper.path())?;
assert!(xattr::get(upper.path().join("nixos"), OVERLAY_OPAQUE_XATTR)?.is_none());
assert_eq!(
xattr::get(upper.path().join("only-upper"), OVERLAY_OPAQUE_XATTR)?.as_deref(),
Some(b"y".as_slice())
);
Ok(())
}
}

2
pkgs/by-name/ni/nixos-init/src/lib.rs
View File

@@ -1,6 +1,7 @@
mod activate;
mod config;
mod env_generator;
mod etc_overlay;
mod find_etc;
mod fs;
mod init;
@@ -16,6 +17,7 @@ use anyhow::{Context, Result, bail};
pub use crate::{
activate::activate,
env_generator::env_generator,
etc_overlay::clear_etc_opaque,
find_etc::find_etc,
init::init,
initrd_init::initrd_init,

3
pkgs/by-name/ni/nixos-init/src/main.rs
View File

@@ -2,7 +2,7 @@ use std::{env, io::Write, process::ExitCode};
use log::Level;
use nixos_init::{env_generator, find_etc, initrd_init, resolve_in_root};
use nixos_init::{clear_etc_opaque, env_generator, find_etc, initrd_init, resolve_in_root};
fn main() -> ExitCode {
let arg0 = env::args()
@@ -12,6 +12,7 @@ fn main() -> ExitCode {
setup_logger();
let entrypoint = match arg0.as_str() {
"clear-etc-opaque" => clear_etc_opaque,
"find-etc" => find_etc,
"resolve-in-root" => resolve_in_root,
"initrd-init" => initrd_init,