~talya/nix-rosetta-builder

https://github.com/cpick/nix-rosetta-builder ·

👀

fork

r/o

services.logind.extraConfig -> services.logind.settings.Login. 1c2148c1 parent 8d9a12b0

authored by Talya Connor committed by ~talya

nix-rosetta-builder / module.nix

{

# configuration

image,

linuxSystem,

{

config,

lib,

pkgs,

...

let

inherit (lib)

boolToString

escapeShellArg

mkAfter

mkBefore

mkDefault

mkEnableOption

mkForce

mkIf

mkMerge

mkOption

optionalAttrs

optionalString

types

;

{

options.nix-rosetta-builder = {

enable = mkEnableOption "Nix Rosetta Linux builder";

potentiallyInsecureExtraNixosModule = mkOption {

type = types.attrs;

default = { };

description = ''

Extra NixOS configuration module to pass to the VM.

The VM's default configuration allows it to be securely used as a builder. Some extra

configuration changes may endager this security and allow compromised deriviations into the

host's Nix store. Care should be taken to think through the implications of any extra

configuration changes using this option. When in doubt, please open a GitHub issue to

discuss (additional, restricted options can be added to support safe configurations).

'';

};

cores = mkOption {

type = types.int;

default = 8;

description = ''

The number of CPU cores allocated to the VM.

This also sets the maximum number of jobs allowed for the

builder in the `nix.buildMachines` specification.

'';

};

diskSize = mkOption {

type = types.str;

default = "100GiB";

description = ''

The size of the disk image for the VM.

'';

};

memory = mkOption {

type = types.str;

default = "6GiB";

description = ''

The amount of memory to allocate to the VM.

'';

example = "8GiB";

};

onDemand = mkOption {

type = types.bool;

default = false;

description = ''

By default, the VM will run all the time as a daemon in the background. This allows Linux

builds to start right away, but means the VM is always consuming RAM (and a bit of CPU).

Alternatively, this option will cause the VM to run only "on-demand": when not in use the VM

will not be running. Any Linux build will cause it to automatically start up

(blocking/pausing the build for several seconds until the VM boots) and after a period of

time/hours without any active Linux builds, the VM will power itself off.

'';

};

onDemandLingerMinutes = mkOption {

type = types.ints.positive;

default = 180;

description = ''

If onDemand=true, this specifies the number of minutes of inactivity before the VM will

power itself off.

'';

};

permitNonRootSshAccess = mkOption {

type = types.bool;

default = false;

description = ''

100

Allow regular, non-root users to SSH into the VM with `ssh rosetta-builder`.

101

102

By default, regular users can `nix build` using the VM without any extra permissions (since

103

it's configured as a remote builder), but they can only SSH directly into it with

104

`sudo ssh rosetta-builder`.

'';

};

port = mkOption {

type = types.int;

# `nix.linux-builder` uses 31022:

112

# https://github.com/LnL7/nix-darwin/blob/a35b08d09efda83625bef267eb24347b446c80b8/modules/nix/linux-builder.nix#L199

113

# Use a similar, but different one:

default = 31122;

description = ''

The SSH port used by the VM.

'';

};

rosettaAot = mkOption {

type = types.bool;

default = false;

description = ''

Whether to enable the rosettad AOT translation daemon.

126

127

Some binaries crash when run with this enabled.

'';

};

};

config =

let

inherit (import ./constants.nix)

name

linuxHostName

linuxUser

sshKeyType

sshHostPrivateKeyFileName

140

sshHostPublicKeyFileName

141

sshUserPrivateKeyFileName

142

sshUserPublicKeyFileName

143

;

144

145

debugInsecurely = false; # enable root access in VM and debug logging

146

147

imageWithFinalConfig = image.override {

148

inherit debugInsecurely;

149

onDemand = cfg.onDemand;

150

onDemandLingerMinutes = cfg.onDemandLingerMinutes;

151

potentiallyInsecureExtraNixosModule = cfg.potentiallyInsecureExtraNixosModule;

152

withRosettaAot = cfg.rosettaAot;

153

};

154

155

cfg = config.nix-rosetta-builder;

156

daemonName = "${name}d";

157

daemonSocketName = "Listener";

158

159

# `sysadminctl -h` says role account UIDs (no mention of service accounts or GIDs) should be

160

# in the 200-400 range `mkuser`s README.md mentions the same:

161

# https://github.com/freegeek-pdx/mkuser/blob/b7a7900d2e6ef01dfafad1ba085c94f7302677d9/README.md?plain=1#L413-L437

162

# Determinate's `nix-installer` (and, I believe, current versions of the official one) uses a

163

# variable number starting at 350 and up:

164

# https://github.com/DeterminateSystems/nix-installer/blob/6beefac4d23bd9a0b74b6758f148aa24d6df3ca9/README.md?plain=1#L511-L514

165

# Meanwhile, new macOS versions are installing accounts that encroach from below.

166

# Try to fit in between:

167

darwinGid = 349;

168

darwinUid = darwinGid;

169

170

darwinGroup = builtins.replaceStrings [ "-" ] [ "" ] name; # keep in sync with `name`s format

171

darwinUser = "_${darwinGroup}";

172

linuxSshdKeysDirName = "linux-sshd-keys";

173

174

sshGlobalKnownHostsFileName = "ssh_known_hosts";

175

sshHost = name; # no prefix because it's user visible (in `sudo ssh '${sshHost}'`)

176

sshHostKeyAlias = "${sshHost}-key";

177

workingDirPath = "/var/lib/${name}";

178

179

gidSh = escapeShellArg (toString darwinGid);

180

groupSh = escapeShellArg darwinGroup;

181

groupPathSh = escapeShellArg "/Groups/${darwinGroup}";

182

183

uidSh = escapeShellArg (toString darwinUid);

184

userSh = escapeShellArg darwinUser;

185

userPathSh = escapeShellArg "/Users/${darwinUser}";

186

187

workingDirPathSh = escapeShellArg workingDirPath;

188

189

vmYaml = (pkgs.formats.yaml { }).generate "${name}.yaml" {

190

# Prevent ~200MiB unused nerdctl-full*.tar.gz download

191

# https://github.com/lima-vm/lima/blob/0e931107cadbcb6dbc7bbb25626f66cdbca1f040/pkg/instance/start.go#L43

192

containerd.user = false;

cpus = cfg.cores;

disk = cfg.diskSize;

images = [

{

# extension must match `imageFormat`

201

location = "${imageWithFinalConfig}/${imageWithFinalConfig.passthru.filePath}";

}

];

memory = cfg.memory;

mounts = [

{

# order must match `sshdKeysVirtiofsTag`s suffix

210

location = "${workingDirPath}/${linuxSshdKeysDirName}";

}

];

rosetta.enabled = true;

215

216

ssh = {

217

launchdSocketName = optionalString cfg.onDemand daemonSocketName;

218

localPort = cfg.port;

};

};

mkMerge [

(mkIf (!cfg.enable) {

224

# This `postActivation` was chosen in particiular because it's one of the system level (as

225

# opposed to user level) ones that's been set aside for customization:

226

# https://github.com/LnL7/nix-darwin/blob/a35b08d09efda83625bef267eb24347b446c80b8/modules/system/activation-scripts.nix#L121-L125

227

# And of those, it's the one that's executed after `activationScripts.launchd` which stops

228

# the VM:

229

# https://github.com/LnL7/nix-darwin/blob/a35b08d09efda83625bef267eb24347b446c80b8/modules/system/activation-scripts.nix#L58-L66

230

# https://github.com/LnL7/nix-darwin/blob/a35b08d09efda83625bef267eb24347b446c80b8/modules/system/activation-scripts.nix#L66-L75

231

system.activationScripts.postActivation.text =

232

# apply "before" to work cooperatively with any other modules using this activation script

233

mkBefore ''

234

if [ -d ${workingDirPathSh} ] ; then

235

printf >&2 'removing working directory %s...\n' ${workingDirPathSh}

236

rm -rf ${workingDirPathSh}

237

238

239

if uid="$(id -u ${userSh} 2>'/dev/null')" ; then

240

if [ "$uid" -ne ${uidSh} ] ; then

241

printf >&2 \

242

'\e[1;31merror: existing user: %s has unexpected UID: %s\e[0m\n' \

${userSh} \

"$uid"

exit 1

printf >&2 'deleting user %s...\n' ${userSh}

248

dscl . -delete ${userPathSh}

unset 'uid'

if primaryGroupId="$(dscl . -read ${groupPathSh} 'PrimaryGroupID' 2>'/dev/null')" ; then

253

if [[ "$primaryGroupId" != *\ ${gidSh} ]] ; then

254

printf >&2 \

255

'\e[1;31merror: existing group: %s has unexpected %s\e[0m\n' \

${groupSh} \

"$primaryGroupId"

exit 1

printf >&2 'deleting group %s...\n' ${groupSh}

261

dscl . -delete ${groupPathSh}

262

263

unset 'primaryGroupId'

'';

})

(mkIf cfg.enable {

environment.etc."ssh/ssh_config.d/100-${sshHost}.conf".text = ''

268

Host "${sshHost}"

269

GlobalKnownHostsFile "${workingDirPath}/${sshGlobalKnownHostsFileName}"

270

Hostname localhost

271

HostKeyAlias "${sshHostKeyAlias}"

272

Port "${toString cfg.port}"

273

StrictHostKeyChecking yes

274

User "${linuxUser}"

275

IdentityFile "${workingDirPath}/${sshUserPrivateKeyFileName}"

276

'';

277

278

launchd.daemons."${daemonName}" = {

path = [

pkgs.coreutils

pkgs.diffutils

pkgs.findutils

pkgs.gnugrep

(pkgs.lima.overrideAttrs (old: {

285

src = pkgs.fetchFromGitHub {

286

owner = "cpick";

287

repo = "lima";

288

rev = "afbfdfb8dd5fa370547b7fc64a16ce2a354b1ff0";

289

hash = "sha256-tCildZJp6ls+WxRAbkoeLRb4WdroBYn/gvE5Vb8Hm5A=";

290

};

291

292

vendorHash = "sha256-I84971WovhJL/VO/Ycu12qa9lDL3F9USxlt9rXcsnTU=";

}))

pkgs.openssh

# Lima calls `sw_vers` which is not packaged in Nix:

297

# https://github.com/lima-vm/lima/blob/0e931107cadbcb6dbc7bbb25626f66cdbca1f040/pkg/osutil/osversion_darwin.go#L13

298

# If the call fails it will not use the Virtualization framework bakend (by default? among

# other things?).

"/usr/bin"

];

script =

let

darwinUserSh = escapeShellArg darwinUser;

306

linuxHostNameSh = escapeShellArg linuxHostName;

307

linuxSshdKeysDirNameSh = escapeShellArg linuxSshdKeysDirName;

308

sshGlobalKnownHostsFileNameSh = escapeShellArg sshGlobalKnownHostsFileName;

309

sshHostKeyAliasSh = escapeShellArg sshHostKeyAlias;

310

sshHostPrivateKeyFileNameSh = escapeShellArg sshHostPrivateKeyFileName;

311

sshHostPublicKeyFileNameSh = escapeShellArg sshHostPublicKeyFileName;

312

sshKeyTypeSh = escapeShellArg sshKeyType;

313

sshUserPrivateKeyFileNameSh = escapeShellArg sshUserPrivateKeyFileName;

314

sshUserPublicKeyFileNameSh = escapeShellArg sshUserPublicKeyFileName;

315

vmNameSh = escapeShellArg "${name}-vm";

316

vmYamlSh = escapeShellArg vmYaml;

set -e

set -u

umask 'g-w,o='

chmod 'g-w,o=x' .

# must be idempotent in the face of partial failues

326

# the `find` test must fail if the user private key was readable but should no longer be

327

cmp -s ${vmYamlSh} .lima/${vmNameSh}/lima.yaml && \

328

limactl list -q 2>'/dev/null' | grep -q ${vmNameSh} && \

329

find ${sshUserPrivateKeyFileNameSh} \

330

-perm '-go=r' -exec ${boolToString cfg.permitNonRootSshAccess} '{}' '+' \

331

2>'/dev/null' && \

332

true || {

333

rm -f ${sshUserPrivateKeyFileNameSh} ${sshUserPublicKeyFileNameSh}

334

ssh-keygen \

335

-C ${darwinUserSh}@darwin -f ${sshUserPrivateKeyFileNameSh} -N "" -t ${sshKeyTypeSh}

336

337

rm -f ${sshHostPrivateKeyFileNameSh} ${sshHostPublicKeyFileNameSh}

338

ssh-keygen \

339

-C root@${linuxHostNameSh} -f ${sshHostPrivateKeyFileNameSh} -N "" -t ${sshKeyTypeSh}

340

341

mkdir -p ${linuxSshdKeysDirNameSh}

342

mv \

343

${sshUserPublicKeyFileNameSh} ${sshHostPrivateKeyFileNameSh} \

344

${linuxSshdKeysDirNameSh}

345

346

echo ${sshHostKeyAliasSh} "$(cat ${sshHostPublicKeyFileNameSh})" \

347

>${sshGlobalKnownHostsFileNameSh}

348

349

limactl delete --force ${vmNameSh}

350

351

# must be last so `limactl list` only now succeeds

352

limactl create --name=${vmNameSh} ${vmYamlSh}

353

}

354

355

# outside the block so both new and old installations end up with the same permissions

356

chmod 'go+r' ${sshGlobalKnownHostsFileNameSh}

357

358

# outside the block so non-root access may be enabled without recreating VM

359

${optionalString cfg.permitNonRootSshAccess ''

360

chmod 'go+r' ${sshUserPrivateKeyFileNameSh}

361

''}

362

363

exec limactl start ${optionalString debugInsecurely "--debug"} --foreground ${vmNameSh}

'';

serviceConfig =

{

KeepAlive = !cfg.onDemand;

369

370

Sockets."${daemonSocketName}" = optionalAttrs cfg.onDemand {

371

SockFamily = "IPv4";

372

SockNodeName = "localhost";

373

SockServiceName = toString cfg.port;

374

};

375

376

UserName = darwinUser;

377

WorkingDirectory = workingDirPath;

378

}

379

// optionalAttrs debugInsecurely {

380

StandardErrorPath = "/tmp/${daemonName}.err.log";

381

StandardOutPath = "/tmp/${daemonName}.out.log";

};

};

nix = {

buildMachines = [

{

hostName = sshHost;

maxJobs = cfg.cores;

protocol = "ssh-ng";

supportedFeatures = [

"benchmark"

"big-parallel"

"kvm"

"nixos-test"

];

systems = [

linuxSystem

"x86_64-linux"

];

}

];

distributedBuilds = mkForce true;

405

settings.builders-use-substitutes = mkDefault true;

406

};

407

408

# `users.users` cannot create a service account and cannot create an empty home directory so do

409

# it manually in an activation script. This `extraActivation` was chosen in particiular because

410

# it's one of the system level (as opposed to user level) ones that's been set aside for

411

# customization:

412

# https://github.com/LnL7/nix-darwin/blob/a35b08d09efda83625bef267eb24347b446c80b8/modules/system/activation-scripts.nix#L121-L125

413

# And of those, it's the one that's executed latest but still before

414

# `activationScripts.launchd` which needs the group, user, and directory in place:

415

# https://github.com/LnL7/nix-darwin/blob/a35b08d09efda83625bef267eb24347b446c80b8/modules/system/activation-scripts.nix#L58-L66

416

system.activationScripts.extraActivation.text =

417

# apply "after" to work cooperatively with any other modules using this activation script

418

mkAfter ''

419

printf >&2 'setting up group %s...\n' ${groupSh}

420

421

if ! primaryGroupId="$(dscl . -read ${groupPathSh} 'PrimaryGroupID' 2>'/dev/null')" ; then

422

printf >&2 'creating group %s...\n' ${groupSh}

423

dscl . -create ${groupPathSh} 'PrimaryGroupID' ${gidSh}

424

elif [[ "$primaryGroupId" != *\ ${gidSh} ]] ; then

425

printf >&2 \

426

'\e[1;31merror: existing group: %s has unexpected %s\e[0m\n' \

${groupSh} \

"$primaryGroupId"

exit 1

unset 'primaryGroupId'

432

433

434

printf >&2 'setting up user %s...\n' ${userSh}

435

436

if ! uid="$(id -u ${userSh} 2>'/dev/null')" ; then

437

printf >&2 'creating user %s...\n' ${userSh}

438

dscl . -create ${userPathSh}

439

dscl . -create ${userPathSh} 'PrimaryGroupID' ${gidSh}

440

dscl . -create ${userPathSh} 'NFSHomeDirectory' ${workingDirPathSh}

441

dscl . -create ${userPathSh} 'UserShell' '/usr/bin/false'

442

dscl . -create ${userPathSh} 'IsHidden' 1

443

dscl . -create ${userPathSh} 'UniqueID' ${uidSh} # must be last so `id` only now succeeds

444

elif [ "$uid" -ne ${uidSh} ] ; then

445

printf >&2 \

446

'\e[1;31merror: existing user: %s has unexpected UID: %s\e[0m\n' \

${userSh} \

"$uid"

exit 1

unset 'uid'

printf >&2 'setting up working directory %s...\n' ${workingDirPathSh}

455

mkdir -p ${workingDirPathSh}

456

chown ${userSh}:${groupSh} ${workingDirPathSh}

'';

})

];

}