prod{access,vider}: implement
Prodaccess/Prodvider allow issuing short-lived certificates for all SSO
users to access the kubernetes cluster.
Currently, all users get a personal-$username namespace in which they
have adminitrative rights. Otherwise, they get no access.
In addition, we define a static CRB to allow some admins access to
everything. In the future, this will be more granular.
We also update relevant documentation.
Change-Id: Ia18594eea8a9e5efbb3e9a25a04a28bbd6a42153
diff --git a/cluster/kube/cluster.jsonnet b/cluster/kube/cluster.jsonnet
index dab37a8..1226354 100644
--- a/cluster/kube/cluster.jsonnet
+++ b/cluster/kube/cluster.jsonnet
@@ -1,6 +1,7 @@
# Top level cluster configuration.
local kube = import "../../kube/kube.libsonnet";
+local policies = import "../../kube/policies.libsonnet";
local calico = import "lib/calico.libsonnet";
local certmanager = import "lib/cert-manager.libsonnet";
@@ -9,6 +10,7 @@
local metallb = import "lib/metallb.libsonnet";
local metrics = import "lib/metrics.libsonnet";
local nginx = import "lib/nginx.libsonnet";
+local prodvider = import "lib/prodvider.libsonnet";
local registry = import "lib/registry.libsonnet";
local rook = import "lib/rook.libsonnet";
@@ -30,7 +32,7 @@
"rbac.authorization.kubernetes.io/autoupdate": "true",
},
labels+: {
- "kubernets.io/bootstrapping": "rbac-defaults",
+ "kubernetes.io/bootstrapping": "rbac-defaults",
},
},
rules: [
@@ -57,6 +59,96 @@
],
},
+ // This ClusteRole is bound to all humans that log in via prodaccess/prodvider/SSO.
+ // It should allow viewing of non-sensitive data for debugability and openness.
+ crViewer: kube.ClusterRole("system:viewer") {
+ rules: [
+ {
+ apiGroups: [""],
+ resources: [
+ "nodes",
+ "namespaces",
+ "pods",
+ "configmaps",
+ "services",
+ ],
+ verbs: ["list"],
+ },
+ {
+ apiGroups: ["metrics.k8s.io"],
+ resources: [
+ "nodes",
+ "pods",
+ ],
+ verbs: ["list"],
+ },
+ {
+ apiGroups: ["apps"],
+ resources: [
+ "statefulsets",
+ ],
+ verbs: ["list"],
+ },
+ {
+ apiGroups: ["extensions"],
+ resources: [
+ "deployments",
+ "ingresses",
+ ],
+ verbs: ["list"],
+ }
+ ],
+ },
+ // This ClusterRole is applied (scoped to personal namespace) to all humans.
+ crFullInNamespace: kube.ClusterRole("system:admin-namespace") {
+ rules: [
+ {
+ apiGroups: ["*"],
+ resources: ["*"],
+ verbs: ["*"],
+ },
+ ],
+ },
+ // This ClusterRoleBindings allows root access to cluster admins.
+ crbAdmins: kube.ClusterRoleBinding("system:admins") {
+ roleRef: {
+ apiGroup: "rbac.authorization.k8s.io",
+ kind: "ClusterRole",
+ name: "cluster-admin",
+ },
+ subjects: [
+ {
+ apiGroup: "rbac.authorization.k8s.io",
+ kind: "User",
+ name: user + "@hackerspace.pl",
+ } for user in [
+ "q3k",
+ "implr",
+ "informatic",
+ ]
+ ],
+ },
+
+ podSecurityPolicies: policies.Cluster {},
+
+ allowInsecureNamespaces: [
+ policies.AllowNamespaceInsecure("kube-system"),
+ # TODO(q3k): fix this?
+ policies.AllowNamespaceInsecure("ceph-waw2"),
+ ],
+
+ // Allow all service accounts (thus all controllers) to create secure pods.
+ crbAllowServiceAccountsSecure: kube.ClusterRoleBinding("policy:allow-all-secure") {
+ roleRef_: cluster.podSecurityPolicies.secureRole,
+ subjects: [
+ {
+ kind: "Group",
+ apiGroup: "rbac.authorization.k8s.io",
+ name: "system:serviceaccounts",
+ }
+ ],
+ },
+
// Calico network fabric
calico: calico.Environment {},
// CoreDNS for this cluster.
@@ -106,6 +198,9 @@
objectStorageName: "waw-hdd-redundant-2-object",
},
},
+
+ // Prodvider
+ prodvider: prodvider.Environment {},
};
diff --git a/cluster/kube/lib/cockroachdb.libsonnet b/cluster/kube/lib/cockroachdb.libsonnet
index ac4c965..212104d 100644
--- a/cluster/kube/lib/cockroachdb.libsonnet
+++ b/cluster/kube/lib/cockroachdb.libsonnet
@@ -36,6 +36,7 @@
local kube = import "../../../kube/kube.libsonnet";
local cm = import "cert-manager.libsonnet";
+local policies = import "../../../kube/policies.libsonnet";
{
Cluster(name): {
@@ -70,6 +71,8 @@
[if cluster.cfg.ownNamespace then "ns"]: kube.Namespace(cluster.namespaceName),
},
+ insecurePolicy: policies.AllowNamespaceInsecure(cluster.namespaceName),
+
name(suffix):: if cluster.cfg.ownNamespace then suffix else name + "-" + suffix,
pki: {
diff --git a/cluster/kube/lib/metallb.libsonnet b/cluster/kube/lib/metallb.libsonnet
index a56fc90..7f3d746 100644
--- a/cluster/kube/lib/metallb.libsonnet
+++ b/cluster/kube/lib/metallb.libsonnet
@@ -1,6 +1,7 @@
# Deploy MetalLB
local kube = import "../../../kube/kube.libsonnet";
+local policies = import "../../../kube/policies.libsonnet";
local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadata.name) {
roleRef: {
@@ -32,6 +33,8 @@
ns: if cfg.namespaceCreate then kube.Namespace(cfg.namespace),
+ insecurePolicy: policies.AllowNamespaceInsecure(cfg.namespace),
+
saController: kube.ServiceAccount("controller") {
metadata+: {
namespace: cfg.namespace,
diff --git a/cluster/kube/lib/nginx.libsonnet b/cluster/kube/lib/nginx.libsonnet
index a871b96..ab7bbc2 100644
--- a/cluster/kube/lib/nginx.libsonnet
+++ b/cluster/kube/lib/nginx.libsonnet
@@ -1,6 +1,7 @@
# Deploy a per-cluster Nginx Ingress Controller
local kube = import "../../../kube/kube.libsonnet";
+local policies = import "../../../kube/policies.libsonnet";
{
Environment: {
@@ -21,6 +22,8 @@
namespace: kube.Namespace(cfg.namespace),
+ allowInsecure: policies.AllowNamespaceInsecure(cfg.namespace),
+
maps: {
make(name):: kube.ConfigMap(name) {
metadata+: env.metadata,
diff --git a/cluster/kube/lib/prodvider.libsonnet b/cluster/kube/lib/prodvider.libsonnet
new file mode 100644
index 0000000..5b75c79
--- /dev/null
+++ b/cluster/kube/lib/prodvider.libsonnet
@@ -0,0 +1,85 @@
+# Deploy prodvider (prodaccess server) in cluster.
+
+local kube = import "../../../kube/kube.libsonnet";
+
+{
+ Environment: {
+ local env = self,
+ local cfg = env.cfg,
+
+ cfg:: {
+ namespace: "prodvider",
+ image: "registry.k0.hswaw.net/cluster/prodvider:1567199084-2e1c08fa7a41faac2ef3f79a1bb82f8841a68016",
+
+ pki: {
+ intermediate: {
+ cert: importstr "../../certs/ca-kube-prodvider.cert",
+ key: importstr "../../secrets/plain/ca-kube-prodvider.key",
+ },
+ kube: {
+ cert: importstr "../../certs/ca-kube.crt",
+ },
+ }
+ },
+
+ namespace: kube.Namespace(cfg.namespace),
+
+ metadata(component):: {
+ namespace: cfg.namespace,
+ labels: {
+ "app.kubernetes.io/name": "prodvider",
+ "app.kubernetes.io/managed-by": "kubecfg",
+ "app.kubernetes.io/component": component,
+ },
+ },
+
+ secret: kube.Secret("ca") {
+ metadata+: env.metadata("prodvider"),
+ data_: {
+ "intermediate-ca.crt": cfg.pki.intermediate.cert,
+ "intermediate-ca.key": cfg.pki.intermediate.key,
+ "ca.crt": cfg.pki.kube.cert,
+ },
+ },
+
+ deployment: kube.Deployment("prodvider") {
+ metadata+: env.metadata("prodvider"),
+ spec+: {
+ replicas: 3,
+ template+: {
+ spec+: {
+ volumes_: {
+ ca: kube.SecretVolume(env.secret),
+ },
+ containers_: {
+ prodvider: kube.Container("prodvider") {
+ image: cfg.image,
+ args: [
+ "/cluster/prodvider/prodvider",
+ "-listen_address", "0.0.0.0:8080",
+ "-ca_key_path", "/opt/ca/intermediate-ca.key",
+ "-ca_certificate_path", "/opt/ca/intermediate-ca.crt",
+ "-kube_ca_certificate_path", "/opt/ca/ca.crt",
+ ],
+ volumeMounts_: {
+ ca: { mountPath: "/opt/ca" },
+ }
+ },
+ },
+ },
+ },
+ },
+ },
+
+ svc: kube.Service("prodvider") {
+ metadata+: env.metadata("prodvider"),
+ target_pod:: env.deployment.spec.template,
+ spec+: {
+ type: "LoadBalancer",
+ ports: [
+ { name: "public", port: 443, targetPort: 8080, protocol: "TCP" },
+ ],
+ },
+ },
+ },
+}
diff --git a/cluster/kube/lib/registry.libsonnet b/cluster/kube/lib/registry.libsonnet
index 1ce022d..a791acf 100644
--- a/cluster/kube/lib/registry.libsonnet
+++ b/cluster/kube/lib/registry.libsonnet
@@ -152,11 +152,12 @@
},
local data = self,
pushers:: [
- { who: ["q3k", "inf"], what: "vms/*" },
- { who: ["q3k", "inf"], what: "app/*" },
- { who: ["q3k", "inf"], what: "go/svc/*" },
+ { who: ["q3k", "informatic"], what: "vms/*" },
+ { who: ["q3k", "informatic"], what: "app/*" },
+ { who: ["q3k", "informatic"], what: "go/svc/*" },
{ who: ["q3k"], what: "bgpwtf/*" },
{ who: ["q3k"], what: "devtools/*" },
+ { who: ["q3k", "informatic"], what: "cluster/*" },
],
acl: [
{