1
{
2
description = "Lima-based, Rosetta 2-enabled, Apple silicon (macOS/Darwin)-hosted Linux builder";
3
4
inputs = {
5
nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
6
nixos-generators = {
7
url = "github:nix-community/nixos-generators";
8
inputs.nixpkgs.follows = "nixpkgs";
9
};
10
};
11
12
outputs = { self, nixos-generators, nixpkgs }:
13
let
14
darwinSystem = "aarch64-darwin";
15
linuxSystem = builtins.replaceStrings [ "darwin" ] [ "linux" ] darwinSystem;
16
lib = nixpkgs.lib;
17
18
name = "rosetta-builder"; # update `darwinGroup` if adding or removing special characters
19
linuxHostName = name; # no prefix because it's user visible (on prompt when `ssh`d in)
20
linuxUser = "builder"; # follow linux-builder/darwin-builder precedent
21
22
sshKeyType = "ed25519";
23
sshHostPrivateKeyFileName = "ssh_host_${sshKeyType}_key";
24
sshHostPublicKeyFileName = "${sshHostPrivateKeyFileName}.pub";
25
sshUserPrivateKeyFileName = "ssh_user_${sshKeyType}_key";
26
sshUserPublicKeyFileName = "${sshUserPrivateKeyFileName}.pub";
27
28
debug = false; # enable root access in VM and debug logging
29
30
in {
31
packages."${linuxSystem}".default = nixos-generators.nixosGenerate (
32
let
33
imageFormat = "qcow-efi"; # must match `vmYaml.images.location`s extension
34
pkgs = nixpkgs.legacyPackages."${linuxSystem}";
35
36
sshdKeys = "sshd-keys";
37
sshDirPath = "/etc/ssh";
38
sshHostPrivateKeyFilePath = "${sshDirPath}/${sshHostPrivateKeyFileName}";
39
40
in {
41
format = imageFormat;
42
43
modules = [ {
44
boot = {
45
kernelParams = [ "console=tty0" ];
46
47
loader = {
48
efi.canTouchEfiVariables = true;
49
systemd-boot.enable = true;
50
};
51
};
52
53
documentation.enable = false;
54
55
fileSystems = {
56
"/".options = [ "discard" "noatime" ];
57
"/boot".options = [ "discard" "noatime" "umask=0077" ];
58
};
59
60
networking.hostName = linuxHostName;
61
62
nix = {
63
channel.enable = false;
64
registry.nixpkgs.flake = nixpkgs;
65
66
settings = {
67
auto-optimise-store = true;
68
experimental-features = [ "flakes" "nix-command" ];
69
min-free = "5G";
70
max-free = "7G";
71
trusted-users = [ linuxUser ];
72
};
73
};
74
75
security = {
76
sudo = {
77
enable = debug;
78
wheelNeedsPassword = !debug;
79
};
80
};
81
82
services = {
83
getty = lib.optionalAttrs debug { autologinUser = linuxUser; };
84
85
openssh = {
86
enable = true;
87
hostKeys = []; # disable automatic host key generation
88
89
settings = {
90
HostKey = sshHostPrivateKeyFilePath;
91
PasswordAuthentication = false;
92
};
93
};
94
};
95
96
system = {
97
disableInstallerTools = true;
98
stateVersion = "24.05";
99
};
100
101
# macOS' Virtualization framework's virtiofs implementation will grant any guest user access
102
# to mounted files; they always appear to be owned by the effective UID and so access cannot
103
# be restricted.
104
# To protect the guest's SSH host key, the VM is configured to prevent any logins (via
105
# console, SSH, etc) by default. This service then runs before sshd, mounts virtiofs,
106
# copies the keys to local files (with appropriate ownership and permissions), and unmounts
107
# the filesystem before allowing SSH to start.
108
# Once SSH has been allowed to start (and given the guest user a chance to log in), the
109
# virtiofs must never be mounted again (as the user could have left some process active to
110
# read its secrets). This is prevented by `unitconfig.ConditionPathExists` below.
111
systemd.services."${sshdKeys}" =
112
let
113
# Lima labels its virtiofs folder mounts counting up:
114
# https://github.com/lima-vm/lima/blob/0e931107cadbcb6dbc7bbb25626f66cdbca1f040/pkg/vz/vm_darwin.go#L568
115
# So this suffix must match `vmYaml.mounts.location`s order:
116
sshdKeysVirtiofsTag = "mount0";
117
118
sshdKeysDirPath = "/var/${sshdKeys}";
119
sshAuthorizedKeysUserFilePath = "${sshDirPath}/authorized_keys.d/${linuxUser}";
120
sshdService = "sshd.service";
121
122
in {
123
before = [ sshdService ];
124
description = "Install sshd's host and authorized keys";
125
enableStrictShellChecks = true;
126
path = [ pkgs.mount pkgs.umount ];
127
requiredBy = [ sshdService ];
128
129
script =
130
let
131
sshAuthorizedKeysUserFilePathSh = lib.escapeShellArg sshAuthorizedKeysUserFilePath;
132
sshAuthorizedKeysUserTmpFilePathSh =
133
lib.escapeShellArg "${sshAuthorizedKeysUserFilePath}.tmp";
134
sshHostPrivateKeyFileNameSh = lib.escapeShellArg sshHostPrivateKeyFileName;
135
sshHostPrivateKeyFilePathSh = lib.escapeShellArg sshHostPrivateKeyFilePath;
136
sshUserPublicKeyFileNameSh = lib.escapeShellArg sshUserPublicKeyFileName;
137
sshdKeysDirPathSh = lib.escapeShellArg sshdKeysDirPath;
138
sshdKeysVirtiofsTagSh = lib.escapeShellArg sshdKeysVirtiofsTag;
139
140
in ''
141
# must be idempotent in the face of partial failues
142
143
mkdir -p ${sshdKeysDirPathSh}
144
mount \
145
-t 'virtiofs' \
146
-o 'nodev,noexec,nosuid,ro' \
147
${sshdKeysVirtiofsTagSh} \
148
${sshdKeysDirPathSh}
149
150
mkdir -p "$(dirname ${sshHostPrivateKeyFilePathSh})"
151
(
152
umask 'go='
153
cp ${sshdKeysDirPathSh}/${sshHostPrivateKeyFileNameSh} ${sshHostPrivateKeyFilePathSh}
154
)
155
156
mkdir -p "$(dirname ${sshAuthorizedKeysUserTmpFilePathSh})"
157
cp \
158
${sshdKeysDirPathSh}/${sshUserPublicKeyFileNameSh} \
159
${sshAuthorizedKeysUserTmpFilePathSh}
160
chmod 'a+r' ${sshAuthorizedKeysUserTmpFilePathSh}
161
162
umount ${sshdKeysDirPathSh}
163
rmdir ${sshdKeysDirPathSh}
164
165
# must be last so only now `unitConfig.ConditionPathExists` triggers
166
mv ${sshAuthorizedKeysUserTmpFilePathSh} ${sshAuthorizedKeysUserFilePathSh}
167
'';
168
169
serviceConfig.Type = "oneshot";
170
171
# see comments on this service and in its `script`
172
unitConfig.ConditionPathExists = "!${sshAuthorizedKeysUserFilePath}";
173
};
174
175
users = {
176
# console and (initial) SSH logins are purposely disabled
177
# see: `systemd.services."${sshdKeys}"`
178
allowNoPasswordLogin = true;
179
180
mutableUsers = false;
181
182
users."${linuxUser}" = {
183
isNormalUser = true;
184
extraGroups = lib.optionals debug [ "wheel" ];
185
};
186
};
187
188
virtualisation.rosetta = {
189
enable = true;
190
191
# Lima's virtiofs label for rosetta:
192
# https://github.com/lima-vm/lima/blob/0e931107cadbcb6dbc7bbb25626f66cdbca1f040/pkg/vz/rosetta_directory_share_arm64.go#L15
193
mountTag = "vz-rosetta";
194
};
195
} ];
196
197
system = linuxSystem;
198
});
199
200
devShells."${darwinSystem}".default =
201
let
202
pkgs = nixpkgs.legacyPackages."${darwinSystem}";
203
in pkgs.mkShell {
204
packages = [ pkgs.lima ];
205
};
206
207
darwinModules.default = { lib, pkgs, ... }:
208
let
209
cores = 8;
210
daemonName = "${name}d";
211
212
# `sysadminctl -h` says role account UIDs (no mention of service accounts or GIDs) should be
213
# in the 200-400 range `mkuser`s README.md mentions the same:
214
# https://github.com/freegeek-pdx/mkuser/blob/b7a7900d2e6ef01dfafad1ba085c94f7302677d9/README.md?plain=1#L413-L437
215
# Determinate's `nix-installer` (and, I believe, current versions of the official one) uses a
216
# variable number starting at 350 and up:
217
# https://github.com/DeterminateSystems/nix-installer/blob/6beefac4d23bd9a0b74b6758f148aa24d6df3ca9/README.md?plain=1#L511-L514
218
# Meanwhile, new macOS versions are installing accounts that encroach from below.
219
# Try to fit in between:
220
darwinGid = 349;
221
darwinUid = darwinGid;
222
223
darwinGroup = builtins.replaceStrings [ "-" ] [ "" ] name; # keep in sync with `name`s format
224
darwinUser = "_${darwinGroup}";
225
linuxSshdKeysDirName = "linux-sshd-keys";
226
227
# `nix.linux-builder` uses 31022:
228
# https://github.com/LnL7/nix-darwin/blob/a35b08d09efda83625bef267eb24347b446c80b8/modules/nix/linux-builder.nix#L199
229
# Use a similar, but different one:
230
port = 31122;
231
232
sshGlobalKnownHostsFileName = "ssh_known_hosts";
233
sshHost = name; # no prefix because it's user visible (in `sudo ssh '${sshHost}'`)
234
sshHostKeyAlias = "${sshHost}-key";
235
workingDirPath = "/var/lib/${name}";
236
237
vmYaml = (pkgs.formats.yaml {}).generate "${name}.yaml" {
238
# Prevent ~200MiB unused nerdctl-full*.tar.gz download
239
# https://github.com/lima-vm/lima/blob/0e931107cadbcb6dbc7bbb25626f66cdbca1f040/pkg/instance/start.go#L43
240
containerd.user = false;
241
242
cpus = cores;
243
244
images = [{
245
# extension must match `imageFormat`
246
location = "${self.packages."${linuxSystem}".default}/nixos.qcow2";
247
}];
248
249
memory = "6GiB";
250
251
mounts = [{
252
# order must match `sshdKeysVirtiofsTag`s suffix
253
location = "${workingDirPath}/${linuxSshdKeysDirName}";
254
}];
255
256
rosetta.enabled = true;
257
ssh.localPort = port;
258
};
259
260
in {
261
environment.etc."ssh/ssh_config.d/100-${sshHost}.conf".text = ''
262
Host "${sshHost}"
263
GlobalKnownHostsFile "${workingDirPath}/${sshGlobalKnownHostsFileName}"
264
Hostname localhost
265
HostKeyAlias "${sshHostKeyAlias}"
266
Port "${toString port}"
267
StrictHostKeyChecking yes
268
User "${linuxUser}"
269
IdentityFile "${workingDirPath}/${sshUserPrivateKeyFileName}"
270
'';
271
272
launchd.daemons."${daemonName}" = {
273
path = [
274
pkgs.coreutils
275
pkgs.gnugrep
276
pkgs.lima
277
pkgs.openssh
278
279
# Lima calls `sw_vers` which is not packaged in Nix:
280
# https://github.com/lima-vm/lima/blob/0e931107cadbcb6dbc7bbb25626f66cdbca1f040/pkg/osutil/osversion_darwin.go#L13
281
# If the call fails it will not use the Virtualization framework bakend (by default? among
282
# other things?).
283
"/usr/bin/"
284
];
285
286
script =
287
let
288
darwinUserSh = lib.escapeShellArg darwinUser;
289
linuxHostNameSh = lib.escapeShellArg linuxHostName;
290
linuxSshdKeysDirNameSh = lib.escapeShellArg linuxSshdKeysDirName;
291
sshGlobalKnownHostsFileNameSh = lib.escapeShellArg sshGlobalKnownHostsFileName;
292
sshHostKeyAliasSh = lib.escapeShellArg sshHostKeyAlias;
293
sshHostPrivateKeyFileNameSh = lib.escapeShellArg sshHostPrivateKeyFileName;
294
sshHostPublicKeyFileNameSh = lib.escapeShellArg sshHostPublicKeyFileName;
295
sshKeyTypeSh = lib.escapeShellArg sshKeyType;
296
sshUserPrivateKeyFileNameSh = lib.escapeShellArg sshUserPrivateKeyFileName;
297
sshUserPublicKeyFileNameSh = lib.escapeShellArg sshUserPublicKeyFileName;
298
vmNameSh = lib.escapeShellArg "${name}-vm";
299
vmYamlSh = lib.escapeShellArg vmYaml;
300
301
in ''
302
set -e
303
set -u
304
305
umask 'g-w,o='
306
chmod 'g-w,o=' .
307
308
# must be idempotent in the face of partial failues
309
limactl list -q 2>'/dev/null' | grep -q ${vmNameSh} || {
310
yes | ssh-keygen \
311
-C ${darwinUserSh}@darwin -f ${sshUserPrivateKeyFileNameSh} -N "" -t ${sshKeyTypeSh}
312
yes | ssh-keygen \
313
-C root@${linuxHostNameSh} -f ${sshHostPrivateKeyFileNameSh} -N "" -t ${sshKeyTypeSh}
314
315
mkdir -p ${linuxSshdKeysDirNameSh}
316
mv \
317
${sshUserPublicKeyFileNameSh} ${sshHostPrivateKeyFileNameSh} ${linuxSshdKeysDirNameSh}
318
319
echo ${sshHostKeyAliasSh} "$(cat ${sshHostPublicKeyFileNameSh})" \
320
>${sshGlobalKnownHostsFileNameSh}
321
322
# must be last so `limactl list` only now succeeds
323
limactl create --name=${vmNameSh} ${vmYamlSh}
324
}
325
326
exec limactl start ${lib.optionalString debug "--debug"} --foreground ${vmNameSh}
327
'';
328
329
serviceConfig = {
330
KeepAlive = true;
331
RunAtLoad = true;
332
UserName = darwinUser;
333
WorkingDirectory = workingDirPath;
334
} // lib.optionalAttrs debug {
335
StandardErrorPath = "/tmp/${daemonName}.err.log";
336
StandardOutPath = "/tmp/${daemonName}.out.log";
337
};
338
};
339
340
nix = {
341
buildMachines = [{
342
hostName = sshHost;
343
maxJobs = cores;
344
protocol = "ssh-ng";
345
supportedFeatures = [ "benchmark" "big-parallel" "kvm" ];
346
systems = [ linuxSystem "x86_64-linux" ];
347
}];
348
349
distributedBuilds = true;
350
settings.builders-use-substitutes = true;
351
};
352
353
# `users.users` cannot create a service account and cannot create an empty home directory so do it
354
# manually in an activation script. This `extraActivation` was chosen in particiular because it's one of the system level (as opposed to user level) ones that's been set aside for customization:
355
# https://github.com/LnL7/nix-darwin/blob/a35b08d09efda83625bef267eb24347b446c80b8/modules/system/activation-scripts.nix#L121-L125
356
# And of those, it's the one that's executed latest but still before
357
# `activationScripts.launchd` which needs the group, user, and directory in place:
358
# https://github.com/LnL7/nix-darwin/blob/a35b08d09efda83625bef267eb24347b446c80b8/modules/system/activation-scripts.nix#L58-L66
359
system.activationScripts.extraActivation.text =
360
let
361
gidSh = lib.escapeShellArg (toString darwinGid);
362
groupSh = lib.escapeShellArg darwinGroup;
363
groupPathSh = lib.escapeShellArg "/Groups/${darwinGroup}";
364
365
uidSh = lib.escapeShellArg (toString darwinUid);
366
userSh = lib.escapeShellArg darwinUser;
367
userPathSh = lib.escapeShellArg "/Users/${darwinUser}";
368
369
workingDirPathSh = lib.escapeShellArg workingDirPath;
370
371
# apply "after" to work cooperatively with any other modules using this activation script
372
in lib.mkAfter ''
373
printf >&2 'setting up group %s...\n' ${groupSh}
374
375
if ! primaryGroupId="$(dscl . -read ${groupPathSh} 'PrimaryGroupID' 2>'/dev/null')" ; then
376
printf >&2 'creating group %s...\n' ${groupSh}
377
dscl . -create ${groupPathSh} 'PrimaryGroupID' ${gidSh}
378
elif [[ "$primaryGroupId" != *\ ${gidSh} ]] ; then
379
printf >&2 \
380
'\e[1;31merror: existing group: %s has unexpected %s\e[0m\n' \
381
${groupSh} \
382
"$primaryGroupId"
383
exit 1
384
fi
385
unset 'primaryGroupId'
386
387
388
printf >&2 'setting up user %s...\n' ${userSh}
389
390
if ! uid="$(id -u ${userSh} 2>'/dev/null')" ; then
391
printf >&2 'creating user %s...\n' ${userSh}
392
dscl . -create ${userPathSh}
393
dscl . -create ${userPathSh} 'PrimaryGroupID' ${gidSh}
394
dscl . -create ${userPathSh} 'NFSHomeDirectory' ${workingDirPathSh}
395
dscl . -create ${userPathSh} 'UserShell' '/usr/bin/false'
396
dscl . -create ${userPathSh} 'IsHidden' 1
397
dscl . -create ${userPathSh} 'UniqueID' ${uidSh} # must be last so `id` only now succeeds
398
elif [ "$uid" -ne ${uidSh} ] ; then
399
printf >&2 \
400
'\e[1;31merror: existing user: %s has unexpected UID: %s\e[0m\n' \
401
${userSh} \
402
"$uid"
403
exit 1
404
fi
405
unset 'uid'
406
407
408
printf >&2 'setting up working directory %s...\n' ${workingDirPathSh}
409
mkdir -p ${workingDirPathSh}
410
chown ${userSh}:${groupSh} ${workingDirPathSh}
411
'';
412
413
};
414
};
415
}
416
