Serge Bazanski | ef3aab6 | 2022-11-18 14:39:45 +0000 | [diff] [blame] | 1 | # Vendored from nixpkgs git 44ad80ab1036c5cc83ada4bfa451dac9939f2a10 |
| 2 | # Copyright (c) 2003-2023 Eelco Dolstra and the Nixpkgs/NixOS contributors |
| 3 | # SPDX-License-Identifier: MIT |
| 4 | |
| 5 | { config, lib, pkgs, ... }: |
| 6 | |
| 7 | with lib; |
| 8 | |
| 9 | let |
| 10 | top = config.services.kubernetes; |
| 11 | cfg = top.apiserver; |
| 12 | |
| 13 | isRBACEnabled = elem "RBAC" cfg.authorizationMode; |
| 14 | |
| 15 | apiserverServiceIP = (concatStringsSep "." ( |
| 16 | take 3 (splitString "." cfg.serviceClusterIpRange |
| 17 | )) + ".1"); |
| 18 | in |
| 19 | { |
| 20 | |
| 21 | imports = [ |
| 22 | (mkRenamedOptionModule [ "services" "kubernetes" "apiserver" "admissionControl" ] [ "services" "kubernetes" "apiserver" "enableAdmissionPlugins" ]) |
| 23 | (mkRenamedOptionModule [ "services" "kubernetes" "apiserver" "address" ] ["services" "kubernetes" "apiserver" "bindAddress"]) |
| 24 | (mkRenamedOptionModule [ "services" "kubernetes" "apiserver" "port" ] ["services" "kubernetes" "apiserver" "insecurePort"]) |
| 25 | (mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "publicAddress" ] "") |
| 26 | (mkRenamedOptionModule [ "services" "kubernetes" "etcd" "servers" ] [ "services" "kubernetes" "apiserver" "etcd" "servers" ]) |
| 27 | (mkRenamedOptionModule [ "services" "kubernetes" "etcd" "keyFile" ] [ "services" "kubernetes" "apiserver" "etcd" "keyFile" ]) |
| 28 | (mkRenamedOptionModule [ "services" "kubernetes" "etcd" "certFile" ] [ "services" "kubernetes" "apiserver" "etcd" "certFile" ]) |
| 29 | (mkRenamedOptionModule [ "services" "kubernetes" "etcd" "caFile" ] [ "services" "kubernetes" "apiserver" "etcd" "caFile" ]) |
| 30 | ]; |
| 31 | |
| 32 | ###### interface |
| 33 | options.services.kubernetes.apiserver = with lib.types; { |
| 34 | |
| 35 | advertiseAddress = mkOption { |
| 36 | description = '' |
| 37 | Kubernetes apiserver IP address on which to advertise the apiserver |
| 38 | to members of the cluster. This address must be reachable by the rest |
| 39 | of the cluster. |
| 40 | ''; |
| 41 | default = null; |
| 42 | type = nullOr str; |
| 43 | }; |
| 44 | |
| 45 | allowPrivileged = mkOption { |
| 46 | description = "Whether to allow privileged containers on Kubernetes."; |
| 47 | default = false; |
| 48 | type = bool; |
| 49 | }; |
| 50 | |
| 51 | authorizationMode = mkOption { |
| 52 | description = '' |
| 53 | Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/Webhook/RBAC/Node). See |
| 54 | <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authorization/"/> |
| 55 | ''; |
| 56 | default = ["RBAC" "Node"]; # Enabling RBAC by default, although kubernetes default is AllowAllow |
| 57 | type = listOf (enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "Webhook" "RBAC" "Node"]); |
| 58 | }; |
| 59 | |
| 60 | authorizationPolicy = mkOption { |
| 61 | description = '' |
| 62 | Kubernetes apiserver authorization policy file. See |
| 63 | <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authorization/"/> |
| 64 | ''; |
| 65 | default = []; |
| 66 | type = listOf attrs; |
| 67 | }; |
| 68 | |
| 69 | basicAuthFile = mkOption { |
| 70 | description = '' |
| 71 | Kubernetes apiserver basic authentication file. See |
| 72 | <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authentication"/> |
| 73 | ''; |
| 74 | default = null; |
| 75 | type = nullOr path; |
| 76 | }; |
| 77 | |
| 78 | bindAddress = mkOption { |
| 79 | description = '' |
| 80 | The IP address on which to listen for the --secure-port port. |
| 81 | The associated interface(s) must be reachable by the rest |
| 82 | of the cluster, and by CLI/web clients. |
| 83 | ''; |
| 84 | default = "0.0.0.0"; |
| 85 | type = str; |
| 86 | }; |
| 87 | |
| 88 | clientCaFile = mkOption { |
| 89 | description = "Kubernetes apiserver CA file for client auth."; |
| 90 | default = top.caFile; |
| 91 | type = nullOr path; |
| 92 | }; |
| 93 | |
| 94 | disableAdmissionPlugins = mkOption { |
| 95 | description = '' |
| 96 | Kubernetes admission control plugins to disable. See |
| 97 | <link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/> |
| 98 | ''; |
| 99 | default = []; |
| 100 | type = listOf str; |
| 101 | }; |
| 102 | |
| 103 | enable = mkEnableOption "Kubernetes apiserver"; |
| 104 | |
| 105 | enableAdmissionPlugins = mkOption { |
| 106 | description = '' |
| 107 | Kubernetes admission control plugins to enable. See |
| 108 | <link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/> |
| 109 | ''; |
| 110 | default = [ |
| 111 | "NamespaceLifecycle" "LimitRanger" "ServiceAccount" |
| 112 | "ResourceQuota" "DefaultStorageClass" "DefaultTolerationSeconds" |
| 113 | "NodeRestriction" |
| 114 | ]; |
| 115 | example = [ |
| 116 | "NamespaceLifecycle" "NamespaceExists" "LimitRanger" |
| 117 | "SecurityContextDeny" "ServiceAccount" "ResourceQuota" |
| 118 | "PodSecurityPolicy" "NodeRestriction" "DefaultStorageClass" |
| 119 | ]; |
| 120 | type = listOf str; |
| 121 | }; |
| 122 | |
| 123 | etcd = { |
| 124 | servers = mkOption { |
| 125 | description = "List of etcd servers."; |
| 126 | default = ["http://127.0.0.1:2379"]; |
| 127 | type = types.listOf types.str; |
| 128 | }; |
| 129 | |
| 130 | keyFile = mkOption { |
| 131 | description = "Etcd key file."; |
| 132 | default = null; |
| 133 | type = types.nullOr types.path; |
| 134 | }; |
| 135 | |
| 136 | certFile = mkOption { |
| 137 | description = "Etcd cert file."; |
| 138 | default = null; |
| 139 | type = types.nullOr types.path; |
| 140 | }; |
| 141 | |
| 142 | caFile = mkOption { |
| 143 | description = "Etcd ca file."; |
| 144 | default = top.caFile; |
| 145 | type = types.nullOr types.path; |
| 146 | }; |
| 147 | }; |
| 148 | |
| 149 | extraOpts = mkOption { |
| 150 | description = "Kubernetes apiserver extra command line options."; |
| 151 | default = ""; |
| 152 | type = str; |
| 153 | }; |
| 154 | |
| 155 | extraSANs = mkOption { |
| 156 | description = "Extra x509 Subject Alternative Names to be added to the kubernetes apiserver tls cert."; |
| 157 | default = []; |
| 158 | type = listOf str; |
| 159 | }; |
| 160 | |
| 161 | featureGates = mkOption { |
| 162 | description = "List set of feature gates"; |
| 163 | default = top.featureGates; |
| 164 | type = listOf str; |
| 165 | }; |
| 166 | |
| 167 | insecureBindAddress = mkOption { |
| 168 | description = "The IP address on which to serve the --insecure-port."; |
| 169 | default = "127.0.0.1"; |
| 170 | type = str; |
| 171 | }; |
| 172 | |
| 173 | insecurePort = mkOption { |
| 174 | description = "Kubernetes apiserver insecure listening port. (0 = disabled)"; |
| 175 | default = 0; |
| 176 | type = int; |
| 177 | }; |
| 178 | |
| 179 | kubeletClientCaFile = mkOption { |
| 180 | description = "Path to a cert file for connecting to kubelet."; |
| 181 | default = top.caFile; |
| 182 | type = nullOr path; |
| 183 | }; |
| 184 | |
| 185 | kubeletClientCertFile = mkOption { |
| 186 | description = "Client certificate to use for connections to kubelet."; |
| 187 | default = null; |
| 188 | type = nullOr path; |
| 189 | }; |
| 190 | |
| 191 | kubeletClientKeyFile = mkOption { |
| 192 | description = "Key to use for connections to kubelet."; |
| 193 | default = null; |
| 194 | type = nullOr path; |
| 195 | }; |
| 196 | |
| 197 | kubeletHttps = mkOption { |
| 198 | description = "Whether to use https for connections to kubelet."; |
| 199 | default = true; |
| 200 | type = bool; |
| 201 | }; |
| 202 | |
| 203 | preferredAddressTypes = mkOption { |
| 204 | description = "List of the preferred NodeAddressTypes to use for kubelet connections."; |
| 205 | type = nullOr str; |
| 206 | default = null; |
| 207 | }; |
| 208 | |
| 209 | proxyClientCertFile = mkOption { |
| 210 | description = "Client certificate to use for connections to proxy."; |
| 211 | default = null; |
| 212 | type = nullOr path; |
| 213 | }; |
| 214 | |
| 215 | proxyClientKeyFile = mkOption { |
| 216 | description = "Key to use for connections to proxy."; |
| 217 | default = null; |
| 218 | type = nullOr path; |
| 219 | }; |
| 220 | |
| 221 | runtimeConfig = mkOption { |
| 222 | description = '' |
| 223 | Api runtime configuration. See |
| 224 | <link xlink:href="https://kubernetes.io/docs/tasks/administer-cluster/cluster-management/"/> |
| 225 | ''; |
| 226 | default = "authentication.k8s.io/v1beta1=true"; |
| 227 | example = "api/all=false,api/v1=true"; |
| 228 | type = str; |
| 229 | }; |
| 230 | |
| 231 | storageBackend = mkOption { |
| 232 | description = '' |
| 233 | Kubernetes apiserver storage backend. |
| 234 | ''; |
| 235 | default = "etcd3"; |
| 236 | type = enum ["etcd2" "etcd3"]; |
| 237 | }; |
| 238 | |
| 239 | securePort = mkOption { |
| 240 | description = "Kubernetes apiserver secure port."; |
| 241 | default = 6443; |
| 242 | type = int; |
| 243 | }; |
| 244 | |
| 245 | serviceAccountKeyFile = mkOption { |
| 246 | description = '' |
| 247 | Kubernetes apiserver PEM-encoded x509 RSA private or public key file, |
| 248 | used to verify ServiceAccount tokens. By default tls private key file |
| 249 | is used. |
| 250 | ''; |
| 251 | default = null; |
| 252 | type = nullOr path; |
| 253 | }; |
| 254 | |
| 255 | serviceClusterIpRange = mkOption { |
| 256 | description = '' |
| 257 | A CIDR notation IP range from which to assign service cluster IPs. |
| 258 | This must not overlap with any IP ranges assigned to nodes for pods. |
| 259 | ''; |
| 260 | default = "10.0.0.0/24"; |
| 261 | type = str; |
| 262 | }; |
| 263 | |
| 264 | tlsCertFile = mkOption { |
| 265 | description = "Kubernetes apiserver certificate file."; |
| 266 | default = null; |
| 267 | type = nullOr path; |
| 268 | }; |
| 269 | |
| 270 | tlsKeyFile = mkOption { |
| 271 | description = "Kubernetes apiserver private key file."; |
| 272 | default = null; |
| 273 | type = nullOr path; |
| 274 | }; |
| 275 | |
| 276 | tokenAuthFile = mkOption { |
| 277 | description = '' |
| 278 | Kubernetes apiserver token authentication file. See |
| 279 | <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authentication"/> |
| 280 | ''; |
| 281 | default = null; |
| 282 | type = nullOr path; |
| 283 | }; |
| 284 | |
| 285 | verbosity = mkOption { |
| 286 | description = '' |
| 287 | Optional glog verbosity level for logging statements. See |
| 288 | <link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/> |
| 289 | ''; |
| 290 | default = null; |
| 291 | type = nullOr int; |
| 292 | }; |
| 293 | |
| 294 | webhookConfig = mkOption { |
| 295 | description = '' |
| 296 | Kubernetes apiserver Webhook config file. It uses the kubeconfig file format. |
| 297 | See <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/webhook/"/> |
| 298 | ''; |
| 299 | default = null; |
| 300 | type = nullOr path; |
| 301 | }; |
| 302 | |
| 303 | }; |
| 304 | |
| 305 | |
| 306 | ###### implementation |
| 307 | config = mkMerge [ |
| 308 | |
| 309 | (mkIf cfg.enable { |
| 310 | systemd.services.kube-apiserver = { |
| 311 | description = "Kubernetes APIServer Service"; |
| 312 | wantedBy = [ "kubernetes.target" ]; |
| 313 | after = [ "network.target" ]; |
| 314 | serviceConfig = { |
| 315 | Slice = "kubernetes.slice"; |
| 316 | ExecStart = ''${top.package}/bin/kube-apiserver \ |
| 317 | --allow-privileged=${boolToString cfg.allowPrivileged} \ |
| 318 | --authorization-mode=${concatStringsSep "," cfg.authorizationMode} \ |
| 319 | ${optionalString (elem "ABAC" cfg.authorizationMode) |
| 320 | "--authorization-policy-file=${ |
| 321 | pkgs.writeText "kube-auth-policy.jsonl" |
| 322 | (concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.authorizationPolicy) |
| 323 | }" |
| 324 | } \ |
| 325 | ${optionalString (elem "Webhook" cfg.authorizationMode) |
| 326 | "--authorization-webhook-config-file=${cfg.webhookConfig}" |
| 327 | } \ |
| 328 | --bind-address=${cfg.bindAddress} \ |
| 329 | ${optionalString (cfg.advertiseAddress != null) |
| 330 | "--advertise-address=${cfg.advertiseAddress}"} \ |
| 331 | ${optionalString (cfg.clientCaFile != null) |
| 332 | "--client-ca-file=${cfg.clientCaFile}"} \ |
| 333 | --disable-admission-plugins=${concatStringsSep "," cfg.disableAdmissionPlugins} \ |
| 334 | --enable-admission-plugins=${concatStringsSep "," cfg.enableAdmissionPlugins} \ |
| 335 | --etcd-servers=${concatStringsSep "," cfg.etcd.servers} \ |
| 336 | ${optionalString (cfg.etcd.caFile != null) |
| 337 | "--etcd-cafile=${cfg.etcd.caFile}"} \ |
| 338 | ${optionalString (cfg.etcd.certFile != null) |
| 339 | "--etcd-certfile=${cfg.etcd.certFile}"} \ |
| 340 | ${optionalString (cfg.etcd.keyFile != null) |
| 341 | "--etcd-keyfile=${cfg.etcd.keyFile}"} \ |
| 342 | ${optionalString (cfg.featureGates != []) |
| 343 | "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ |
| 344 | ${optionalString (cfg.basicAuthFile != null) |
| 345 | "--basic-auth-file=${cfg.basicAuthFile}"} \ |
| 346 | --kubelet-https=${boolToString cfg.kubeletHttps} \ |
| 347 | ${optionalString (cfg.kubeletClientCaFile != null) |
| 348 | "--kubelet-certificate-authority=${cfg.kubeletClientCaFile}"} \ |
| 349 | ${optionalString (cfg.kubeletClientCertFile != null) |
| 350 | "--kubelet-client-certificate=${cfg.kubeletClientCertFile}"} \ |
| 351 | ${optionalString (cfg.kubeletClientKeyFile != null) |
| 352 | "--kubelet-client-key=${cfg.kubeletClientKeyFile}"} \ |
| 353 | ${optionalString (cfg.preferredAddressTypes != null) |
| 354 | "--kubelet-preferred-address-types=${cfg.preferredAddressTypes}"} \ |
| 355 | ${optionalString (cfg.proxyClientCertFile != null) |
| 356 | "--proxy-client-cert-file=${cfg.proxyClientCertFile}"} \ |
| 357 | ${optionalString (cfg.proxyClientKeyFile != null) |
| 358 | "--proxy-client-key-file=${cfg.proxyClientKeyFile}"} \ |
| 359 | --insecure-bind-address=${cfg.insecureBindAddress} \ |
| 360 | --insecure-port=${toString cfg.insecurePort} \ |
| 361 | ${optionalString (cfg.runtimeConfig != "") |
| 362 | "--runtime-config=${cfg.runtimeConfig}"} \ |
| 363 | --secure-port=${toString cfg.securePort} \ |
| 364 | ${optionalString (cfg.serviceAccountKeyFile!=null) |
| 365 | "--service-account-key-file=${cfg.serviceAccountKeyFile}"} \ |
| 366 | --service-cluster-ip-range=${cfg.serviceClusterIpRange} \ |
| 367 | --storage-backend=${cfg.storageBackend} \ |
| 368 | ${optionalString (cfg.tlsCertFile != null) |
| 369 | "--tls-cert-file=${cfg.tlsCertFile}"} \ |
| 370 | ${optionalString (cfg.tlsKeyFile != null) |
| 371 | "--tls-private-key-file=${cfg.tlsKeyFile}"} \ |
| 372 | ${optionalString (cfg.tokenAuthFile != null) |
| 373 | "--token-auth-file=${cfg.tokenAuthFile}"} \ |
| 374 | ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ |
| 375 | ${cfg.extraOpts} |
| 376 | ''; |
| 377 | WorkingDirectory = top.dataDir; |
| 378 | User = "kubernetes"; |
| 379 | Group = "kubernetes"; |
| 380 | AmbientCapabilities = "cap_net_bind_service"; |
| 381 | Restart = "on-failure"; |
| 382 | RestartSec = 5; |
| 383 | }; |
| 384 | }; |
| 385 | |
| 386 | services.etcd = { |
| 387 | clientCertAuth = mkDefault true; |
| 388 | peerClientCertAuth = mkDefault true; |
| 389 | listenClientUrls = mkDefault ["https://0.0.0.0:2379"]; |
| 390 | listenPeerUrls = mkDefault ["https://0.0.0.0:2380"]; |
| 391 | advertiseClientUrls = mkDefault ["https://${top.masterAddress}:2379"]; |
| 392 | initialCluster = mkDefault ["${top.masterAddress}=https://${top.masterAddress}:2380"]; |
| 393 | name = mkDefault top.masterAddress; |
| 394 | initialAdvertisePeerUrls = mkDefault ["https://${top.masterAddress}:2380"]; |
| 395 | }; |
| 396 | |
| 397 | services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled { |
| 398 | |
| 399 | apiserver-kubelet-api-admin-crb = { |
| 400 | apiVersion = "rbac.authorization.k8s.io/v1"; |
| 401 | kind = "ClusterRoleBinding"; |
| 402 | metadata = { |
| 403 | name = "system:kube-apiserver:kubelet-api-admin"; |
| 404 | }; |
| 405 | roleRef = { |
| 406 | apiGroup = "rbac.authorization.k8s.io"; |
| 407 | kind = "ClusterRole"; |
| 408 | name = "system:kubelet-api-admin"; |
| 409 | }; |
| 410 | subjects = [{ |
| 411 | kind = "User"; |
| 412 | name = "system:kube-apiserver"; |
| 413 | }]; |
| 414 | }; |
| 415 | |
| 416 | }; |
| 417 | |
| 418 | services.kubernetes.pki.certs = with top.lib; { |
| 419 | apiServer = mkCert { |
| 420 | name = "kube-apiserver"; |
| 421 | CN = "kubernetes"; |
| 422 | hosts = [ |
| 423 | "kubernetes.default.svc" |
| 424 | "kubernetes.default.svc.${top.addons.dns.clusterDomain}" |
| 425 | cfg.advertiseAddress |
| 426 | top.masterAddress |
| 427 | apiserverServiceIP |
| 428 | "127.0.0.1" |
| 429 | ] ++ cfg.extraSANs; |
| 430 | action = "systemctl restart kube-apiserver.service"; |
| 431 | }; |
| 432 | apiserverProxyClient = mkCert { |
| 433 | name = "kube-apiserver-proxy-client"; |
| 434 | CN = "front-proxy-client"; |
| 435 | action = "systemctl restart kube-apiserver.service"; |
| 436 | }; |
| 437 | apiserverKubeletClient = mkCert { |
| 438 | name = "kube-apiserver-kubelet-client"; |
| 439 | CN = "system:kube-apiserver"; |
| 440 | action = "systemctl restart kube-apiserver.service"; |
| 441 | }; |
| 442 | apiserverEtcdClient = mkCert { |
| 443 | name = "kube-apiserver-etcd-client"; |
| 444 | CN = "etcd-client"; |
| 445 | action = "systemctl restart kube-apiserver.service"; |
| 446 | }; |
| 447 | clusterAdmin = mkCert { |
| 448 | name = "cluster-admin"; |
| 449 | CN = "cluster-admin"; |
| 450 | fields = { |
| 451 | O = "system:masters"; |
| 452 | }; |
| 453 | privateKeyOwner = "root"; |
| 454 | }; |
| 455 | etcd = mkCert { |
| 456 | name = "etcd"; |
| 457 | CN = top.masterAddress; |
| 458 | hosts = [ |
| 459 | "etcd.local" |
| 460 | "etcd.${top.addons.dns.clusterDomain}" |
| 461 | top.masterAddress |
| 462 | cfg.advertiseAddress |
| 463 | ]; |
| 464 | privateKeyOwner = "etcd"; |
| 465 | action = "systemctl restart etcd.service"; |
| 466 | }; |
| 467 | }; |
| 468 | |
| 469 | }) |
| 470 | |
| 471 | ]; |
| 472 | |
| 473 | } |