1
{
2
description = "Lima-based, Rosetta 2-enabled, Apple silicon (macOS/Darwin)-hosted Linux builder";
3
4
inputs = {
5
nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
6
nixos-generators = {
7
url = "github:nix-community/nixos-generators";
8
inputs.nixpkgs.follows = "nixpkgs";
9
};
10
};
11
12
outputs = { self, nixos-generators, nixpkgs }:
13
let
14
darwinSystem = "aarch64-darwin";
15
linuxSystem = builtins.replaceStrings [ "darwin" ] [ "linux" ] darwinSystem;
16
lib = nixpkgs.lib;
17
18
name = "rosetta-builder"; # update `darwinGroup` if adding or removing special characters
19
linuxHostName = name; # no prefix because it's user visible (on prompt when `ssh`d in)
20
linuxUser = "builder"; # follow linux-builder/darwin-builder precedent
21
22
sshKeyType = "ed25519";
23
sshHostPrivateKeyFileName = "ssh_host_${sshKeyType}_key";
24
sshHostPublicKeyFileName = "${sshHostPrivateKeyFileName}.pub";
25
sshUserPrivateKeyFileName = "ssh_user_${sshKeyType}_key";
26
sshUserPublicKeyFileName = "${sshUserPrivateKeyFileName}.pub";
27
28
debug = false; # enable root access in VM and debug logging
29
30
in {
31
packages."${linuxSystem}".default = nixos-generators.nixosGenerate (
32
let
33
imageFormat = "qcow-efi"; # must match `vmYaml.images.location`s extension
34
pkgs = nixpkgs.legacyPackages."${linuxSystem}";
35
36
sshdKeys = "sshd-keys";
37
sshDirPath = "/etc/ssh";
38
sshHostPrivateKeyFilePath = "${sshDirPath}/${sshHostPrivateKeyFileName}";
39
40
in {
41
format = imageFormat;
42
43
modules = [ {
44
boot = {
45
kernelParams = [ "console=tty0" ];
46
47
loader = {
48
efi.canTouchEfiVariables = true;
49
systemd-boot.enable = true;
50
};
51
};
52
53
documentation.enable = false;
54
55
fileSystems = {
56
"/".options = [ "discard" "noatime" ];
57
"/boot".options = [ "discard" "noatime" "umask=0077" ];
58
};
59
60
networking.hostName = linuxHostName;
61
62
nix = {
63
channel.enable = false;
64
registry.nixpkgs.flake = nixpkgs;
65
66
settings = {
67
auto-optimise-store = true;
68
experimental-features = [ "flakes" "nix-command" ];
69
min-free = "5G";
70
max-free = "7G";
71
trusted-users = [ linuxUser ];
72
};
73
};
74
75
security = {
76
polkit = lib.optionalAttrs debug {
77
enable = true;
78
extraConfig = ''
79
polkit.addRule(function(action, subject) {
80
if (
81
(
82
action.id === "org.freedesktop.login1.power-off"
83
|| action.id === "org.freedesktop.login1.reboot"
84
)
85
&& subject.user === "${linuxUser}"
86
) {
87
return "yes";
88
} else {
89
return "no";
90
}
91
})
92
'';
93
};
94
95
sudo = {
96
enable = debug;
97
wheelNeedsPassword = !debug;
98
};
99
};
100
101
services = {
102
getty = lib.optionalAttrs debug { autologinUser = linuxUser; };
103
104
openssh = {
105
enable = true;
106
hostKeys = []; # disable automatic host key generation
107
108
settings = {
109
HostKey = sshHostPrivateKeyFilePath;
110
PasswordAuthentication = false;
111
};
112
};
113
};
114
115
system = {
116
disableInstallerTools = true;
117
stateVersion = "24.05";
118
};
119
120
systemd.services."${sshdKeys}" =
121
let
122
sshdKeysVirtiofsTag = "mount0"; # suffix must match `vmYaml.mounts.location`s order
123
sshdKeysDirPath = "/var/${sshdKeys}";
124
sshAuthorizedKeysUserFilePath = "${sshDirPath}/authorized_keys.d/${linuxUser}";
125
sshdService = "sshd.service";
126
127
in {
128
before = [ sshdService ];
129
description = "Install sshd's host and authorized keys";
130
enableStrictShellChecks = true;
131
path = [ pkgs.mount pkgs.umount ];
132
requiredBy = [ sshdService ];
133
134
# must be idempotent in the face of partial failues
135
script = ''
136
mkdir -p '${sshdKeysDirPath}'
137
mount \
138
-t 'virtiofs' \
139
-o 'nodev,noexec,nosuid,ro' \
140
'${sshdKeysVirtiofsTag}' \
141
'${sshdKeysDirPath}'
142
143
mkdir -p "$(dirname '${sshHostPrivateKeyFilePath}')"
144
(
145
umask 'go='
146
cp '${sshdKeysDirPath}/${sshHostPrivateKeyFileName}' '${sshHostPrivateKeyFilePath}'
147
)
148
149
mkdir -p "$(dirname '${sshAuthorizedKeysUserFilePath}')"
150
cp '${sshdKeysDirPath}/${sshUserPublicKeyFileName}' '${sshAuthorizedKeysUserFilePath}'
151
chmod 'a+r' '${sshAuthorizedKeysUserFilePath}'
152
153
umount '${sshdKeysDirPath}'
154
rmdir '${sshdKeysDirPath}'
155
'';
156
157
serviceConfig.Type = "oneshot";
158
unitConfig.ConditionPathExists = "!${sshAuthorizedKeysUserFilePath}";
159
};
160
161
users = {
162
allowNoPasswordLogin = true;
163
mutableUsers = false;
164
165
users."${linuxUser}" = {
166
isNormalUser = true;
167
extraGroups = lib.optionals debug [ "wheel" ];
168
};
169
};
170
171
virtualisation.rosetta = {
172
enable = true;
173
mountTag = "vz-rosetta";
174
};
175
} ];
176
177
system = linuxSystem;
178
});
179
180
devShells."${darwinSystem}".default =
181
let
182
pkgs = nixpkgs.legacyPackages."${darwinSystem}";
183
in pkgs.mkShell {
184
packages = [ pkgs.lima ];
185
};
186
187
darwinModules.default = { lib, pkgs, ... }:
188
let
189
cores = 8;
190
daemonName = "${name}d";
191
darwinGid = 349;
192
darwinGroup = builtins.replaceStrings [ "-" ] [ "" ] name; # keep in sync with `name`s format
193
darwinUid = darwinGid;
194
darwinUser = "_${darwinGroup}";
195
linuxSshdKeysDirName = "linux-sshd-keys";
196
port = 31122;
197
sshGlobalKnownHostsFileName = "ssh_known_hosts";
198
sshHost = name; # no prefix because it's user visible (in `sudo ssh '${sshHost}'`)
199
sshHostKeyAlias = "${sshHost}-key";
200
workingDirPath = "/var/lib/${name}";
201
202
vmYaml = (pkgs.formats.yaml {}).generate "${name}.yaml" {
203
containerd.user = false;
204
cpus = cores;
205
206
images = [{
207
# extension must match `imageFormat`
208
location = "${self.packages."${linuxSystem}".default}/nixos.qcow2";
209
}];
210
211
memory = "6GiB";
212
213
mounts = [{
214
# order must match `sshdKeysVirtiofsTag`s suffix
215
location = "${workingDirPath}/${linuxSshdKeysDirName}";
216
}];
217
218
rosetta.enabled = true;
219
ssh.localPort = port;
220
};
221
222
in {
223
environment.etc."ssh/ssh_config.d/100-${sshHost}.conf".text = ''
224
Host "${sshHost}"
225
GlobalKnownHostsFile "${workingDirPath}/${sshGlobalKnownHostsFileName}"
226
Hostname localhost
227
HostKeyAlias "${sshHostKeyAlias}"
228
Port "${toString port}"
229
StrictHostKeyChecking yes
230
User "${linuxUser}"
231
IdentityFile "${workingDirPath}/${sshUserPrivateKeyFileName}"
232
'';
233
234
users = {
235
knownUsers = [ darwinUser ];
236
237
users."${darwinUser}" = {
238
gid = darwinGid;
239
home = workingDirPath;
240
isHidden = true;
241
uid = darwinUid;
242
};
243
};
244
245
launchd.daemons."${daemonName}" = {
246
path = [ pkgs.coreutils pkgs.gnugrep pkgs.lima pkgs.openssh "/usr/bin/" ];
247
248
script =
249
let
250
vmName = "${name}-vm";
251
252
in ''
253
set -e
254
set -u
255
256
umask 'g-w,o='
257
chmod 'g-w,o=' .
258
259
# must be idempotent in the face of partial failues
260
limactl list -q 2>'/dev/null' | grep -q '${vmName}' || {
261
yes | ssh-keygen \
262
-C '${darwinUser}@darwin' -f '${sshUserPrivateKeyFileName}' -N "" -t '${sshKeyType}'
263
yes | ssh-keygen \
264
-C 'root@${linuxHostName}' -f '${sshHostPrivateKeyFileName}' -N "" -t '${sshKeyType}'
265
266
mkdir -p '${linuxSshdKeysDirName}'
267
mv \
268
'${sshUserPublicKeyFileName}' '${sshHostPrivateKeyFileName}' '${linuxSshdKeysDirName}'
269
270
echo "${sshHostKeyAlias} $(cat '${sshHostPublicKeyFileName}')" \
271
>'${sshGlobalKnownHostsFileName}'
272
273
limactl create --name='${vmName}' '${vmYaml}'
274
}
275
276
exec limactl start ${lib.optionalString debug "--debug"} --foreground '${vmName}'
277
'';
278
279
serviceConfig = {
280
KeepAlive = true;
281
RunAtLoad = true;
282
UserName = darwinUser;
283
WorkingDirectory = workingDirPath;
284
} // lib.optionalAttrs debug {
285
StandardErrorPath = "/tmp/${daemonName}.err.log";
286
StandardOutPath = "/tmp/${daemonName}.out.log";
287
};
288
};
289
290
nix = {
291
buildMachines = [{
292
hostName = sshHost;
293
maxJobs = cores;
294
protocol = "ssh-ng";
295
supportedFeatures = [ "benchmark" "big-parallel" "kvm" ];
296
systems = [ linuxSystem "x86_64-linux" ];
297
}];
298
299
distributedBuilds = true;
300
settings.builders-use-substitutes = true;
301
};
302
303
system.activationScripts.extraActivation.text =
304
let
305
gidSh = lib.escapeShellArg (toString darwinGid);
306
groupSh = lib.escapeShellArg darwinGroup;
307
groupPathSh = lib.escapeShellArg "/Groups/${darwinGroup}";
308
userSh = lib.escapeShellArg darwinUser;
309
workingDirPathSh = lib.escapeShellArg workingDirPath;
310
311
in lib.mkAfter ''
312
printf >&2 'setting up group %s...\n' ${groupSh}
313
314
if ! primaryGroupId="$(dscl . -read ${groupPathSh} 'PrimaryGroupID' 2>'/dev/null')" ; then
315
printf >&2 'creating group %s...\n' ${groupSh}
316
dscl . -create ${groupPathSh} 'PrimaryGroupID' ${gidSh}
317
elif [[ "$primaryGroupId" != *\ ${gidSh} ]] ; then
318
printf >&2 \
319
'\e[1;31merror: existing group: %s has unexpected %s\e[0m\n' \
320
${groupSh} \
321
"$primaryGroupId"
322
exit 1
323
fi
324
unset 'primaryGroupId'
325
326
327
printf >&2 'setting up working directory %s...\n' ${workingDirPathSh}
328
mkdir -p ${workingDirPathSh}
329
chown ${userSh}:${groupSh} ${workingDirPathSh}
330
'';
331
332
};
333
};
334
}
335
