Merge "bgpwtf/cccampix/peeringdb: allow multiple routers per peer"
diff --git a/app/registry/prod.jsonnet b/app/registry/prod.jsonnet
deleted file mode 100644
index 8e3b562..0000000
--- a/app/registry/prod.jsonnet
+++ /dev/null
@@ -1,318 +0,0 @@
-# registry.k0.hswaw.net, a private docker registry
-# This needs an oauth2 secret provisioned, create with:
-# kubectl -n registry create secret generic auth --from-literal=oauth2_secret=...
-# kubectl get secrets rook-ceph-object-user-waw-hdd-redundant-1-object-registry -n ceph-waw1 -o yaml --export | kubectl replace -f - -n registry
-
-local kube = import "../../kube/kube.libsonnet";
-local cm = import "../../cluster/kube/lib/cert-manager.libsonnet";
-
-{
- local app = self,
- local cfg = app.cfg,
- cfg:: {
- namespace: "registry",
- domain: "k0.hswaw.net",
- storageClassName: "waw-hdd-redundant-1",
- },
-
- metadata(component):: {
- namespace: cfg.namespace,
- labels: {
- "app.kubernetes.io/name": "registry",
- "app.kubernetes.io/managed-by": "kubecfg",
- "app.kubernetes.io/component": component,
- },
- },
-
- namespace: kube.Namespace(cfg.namespace),
-
- registryIssuer: cm.Issuer("registry-issuer") {
- metadata+: app.metadata("registry-issuer"),
- spec: {
- selfSigned: {},
- },
- },
- authCertificate: cm.Certificate("auth") {
- metadata+: app.metadata("auth"),
- spec: {
- secretName: "auth-internal",
- duration: "43800h0m0s", // 5 years
- issuerRef: {
- name: app.registryIssuer.metadata.name,
- },
- commonName: "auth.registry",
- },
- },
- registryCertificate: cm.Certificate("registry") {
- metadata+: app.metadata("registry"),
- spec: {
- secretName: "registry-internal",
- duration: "43800h0m0s", // 5 years
- issuerRef: {
- name: app.registryIssuer.metadata.name,
- },
- commonName: "registry.registry",
- },
- },
-
- registryConfig: kube.ConfigMap("registry-config") {
- metadata+: app.metadata("registry-config"),
- data: {
- "config.yml": std.manifestYamlDoc({
- version: "0.1",
- log: {
- fields: {
- service: "registry",
- },
- },
- storage: {
- cache: {
- blobdescriptor: "inmemory",
- },
- s3: {
- regionendpoint: "https://object.ceph-waw1.hswaw.net",
- bucket: "registry",
- region: "waw-hdd-redunant-1-object:default-placement",
- },
- },
- http: {
- addr: ":5000",
- headers: {
- "X-Content-Type-Options": ["nosniff"],
- },
- tls: {
- certificate: "/certs/tls.crt",
- key: "/certs/tls.key",
- },
- debug: {
- addr: "localhost:5001",
- },
- },
- health: {
- storagedriver: {
- enabled: true,
- interval: "10s",
- threshold: 3,
- },
- },
- auth: {
- token: {
- realm: "https://registry.%s/auth" % [cfg.domain],
- service: "my.docker.registry",
- issuer: "registry.%s auth server" % [cfg.domain],
- rootcertbundle: "/authcerts/tls.crt",
- },
- },
- }),
- },
- },
-
- authVolumeClaim: kube.PersistentVolumeClaim("auth-token-storage") {
- metadata+: app.metadata("auth-token-storage"),
- spec+: {
- storageClassName: cfg.storageClassName,
- accessModes: [ "ReadWriteOnce" ],
- resources: {
- requests: {
- storage: "1Gi",
- },
- },
- },
- },
-
- authConfig: kube.ConfigMap("auth-config") {
- metadata+: app.metadata("auth-config"),
- data: {
- "auth_config.yml": std.manifestYamlDoc({
- server: {
- addr: ":5001",
- certificate: "/certs/tls.crt",
- key: "/certs/tls.key",
- },
- token: {
- issuer: "registry.%s auth server" % [cfg.domain],
- expiration: 900,
- },
- oauth2: {
- client_id: "registry",
- client_secret_file: "/secrets/oauth2_secret",
- authorize_url: "https://sso.hackerspace.pl/oauth/authorize",
- access_token_url: "https://sso.hackerspace.pl/oauth/token",
- profile_url: "https://sso.hackerspace.pl/api/1/profile",
- redirect_url: "https://registry.k0.hswaw.net/oauth2",
- username_key: "username",
- token_db: "/data/oauth2_tokens.ldb",
- registry_url: "https://registry.k0.hswaw.net",
- },
- users: {
- [""]: {}, // '' user are anonymous users.
- },
- local data = self,
- pushers:: [
- { who: ["q3k", "inf"], what: "vms/*" },
- { who: ["q3k", "inf"], what: "app/*" },
- { who: ["q3k", "inf"], what: "go/svc/*" },
- ],
- acl: [
- {
- match: {account: "/.+/", name: "${account}/*"},
- actions: ["*"],
- comment: "Logged in users have full access to images that are in their 'namespace'",
- },
- {
- match: {account: "/.+/", type: "registry", name: "catalog"},
- actions: ["*"],
- comment: "Logged in users can query the catalog.",
- },
- {
- match: {account: ""},
- actions: ["pull"],
- comment: "Anyone can pull all images.",
- },
- ] + [
- {
- match: {
- account: "/(%s)/" % std.join("|", p.who),
- name: p.what,
- },
- actions: ["*"],
- comment: "%s can push to %s" % [std.join(", ", p.who), p.what],
- }
- for p in data.pushers
- ],
- }),
- }
- },
-
- authDeployment: kube.Deployment("auth") {
- metadata+: app.metadata("auth"),
- spec+: {
- replicas: 1,
- template+: {
- spec+: {
- volumes_: {
- data: kube.PersistentVolumeClaimVolume(app.authVolumeClaim),
- config: kube.ConfigMapVolume(app.authConfig),
- certs: {
- secret: { secretName: app.authCertificate.spec.secretName },
- },
- secrets: {
- secret: { secretName: "auth" },
- },
- },
- containers_: {
- auth: kube.Container("auth") {
- image: "informatic/docker_auth:2019040307",
- volumeMounts_: {
- config: { mountPath: "/config" },
- certs: { mountPath: "/certs" },
- secrets: { mountPath: "/secrets" },
- data: { mountPath: "/data" },
- },
- },
- },
- },
- },
- },
- },
- authService: kube.Service("auth") {
- metadata+: app.metadata("auth"),
- target_pod:: app.authDeployment.spec.template,
- spec+: {
- type: "ClusterIP",
- ports: [
- { name: "auth", port: 5001, targetPort: 5001, protocol: "TCP" },
- ],
- }
- },
- registryDeployment: kube.Deployment("docker-registry") {
- metadata+: app.metadata("docker-registry"),
- spec+: {
- replicas: 1,
- template+: {
- spec+: {
- volumes_: {
- config: kube.ConfigMapVolume(app.registryConfig),
- certs: {
- secret: { secretName: app.registryCertificate.spec.secretName },
- },
- authcerts: {
- secret: { secretName: app.authCertificate.spec.secretName },
- },
- },
- containers_: {
- registry: kube.Container("docker-registry") {
- image: "registry:2",
- args: ["/config/config.yml"],
- volumeMounts_: {
- config: { mountPath: "/config" },
- certs: { mountPath: "/certs" },
- authcerts: { mountPath: "/authcerts" },
- },
- env_: {
- REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: {
- name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
- key: "AccessKey"
- }},
- REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: {
- name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
- key: "SecretKey",
- }},
- },
- },
- },
- },
- },
- },
- },
- registryService: kube.Service("docker-registry") {
- metadata+: app.metadata("docker-registry"),
- target_pod:: app.registryDeployment.spec.template,
- spec+: {
- type: "ClusterIP",
- ports: [
- { name: "registry", port: 5000, targetPort: 5000, protocol: "TCP" },
- ],
- }
- },
- registryIngress: kube.Ingress("registry") {
- metadata+: app.metadata("registry") {
- annotations+: {
- "kubernetes.io/tls-acme": "true",
- "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
- "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS",
- "nginx.ingress.kubernetes.io/proxy-body-size": "0",
- },
- },
- spec+: {
- tls: [
- {
- hosts: ["registry.%s" % [cfg.domain]],
- secretName: "registry-tls",
- },
- ],
- rules: [
- {
- host: "registry.%s" % [cfg.domain],
- http: {
- paths: [
- { path: "/auth", backend: app.authService.name_port },
- { path: "/", backend: app.authService.name_port },
- { path: "/v2/", backend: app.registryService.name_port },
- ]
- },
- }
- ],
- },
- },
-
- registryStorageUser: kube._Object("ceph.rook.io/v1", "CephObjectStoreUser", "registry") {
- metadata+: {
- namespace: "ceph-waw1",
- },
- spec: {
- store: "waw-hdd-redundant-1-object",
- displayName: "docker-registry user",
- },
- },
-}
diff --git a/cluster/kube/cluster.jsonnet b/cluster/kube/cluster.jsonnet
index fc0db19..d0b77bd 100644
--- a/cluster/kube/cluster.jsonnet
+++ b/cluster/kube/cluster.jsonnet
@@ -9,10 +9,19 @@
local metallb = import "lib/metallb.libsonnet";
local metrics = import "lib/metrics.libsonnet";
local nginx = import "lib/nginx.libsonnet";
+local registry = import "lib/registry.libsonnet";
local rook = import "lib/rook.libsonnet";
local Cluster(fqdn) = {
local cluster = self,
+ local cfg = cluster.cfg,
+
+ cfg:: {
+ // Storage class used for internal services (like registry). This must
+ // be set to a valid storage class. This can either be a cloud provider class
+ // (when running on GKE &co) or a storage class created using rook.
+ storageClassNameRedundant: error "storageClassNameRedundant must be set",
+ },
// These are required to let the API Server contact kubelets.
crAPIServerToKubelet: kube.ClusterRole("system:kube-apiserver-to-kubelet") {
@@ -88,13 +97,25 @@
},
},
},
+
+ // Docker registry
+ registry: registry.Environment {
+ cfg+: {
+ domain: "registry.%s" % [fqdn],
+ storageClassName: cfg.storageClassNameRedundant,
+ },
+ },
};
{
k0: {
local k0 = self,
- cluster: Cluster("k0.hswaw.net"),
+ cluster: Cluster("k0.hswaw.net") {
+ cfg+: {
+ storageClassNameRedundant: k0.ceph.blockRedundant.name,
+ },
+ },
cockroach: {
waw1: cockroachdb.Cluster("crdb-waw1") {
cfg+: {
diff --git a/cluster/kube/lib/registry.libsonnet b/cluster/kube/lib/registry.libsonnet
new file mode 100644
index 0000000..8b57dd7
--- /dev/null
+++ b/cluster/kube/lib/registry.libsonnet
@@ -0,0 +1,321 @@
+# Deploy a Docker Registry in a cluster.
+
+# This needs an oauth2 secret provisioned, create with:
+# kubectl -n registry create secret generic auth --from-literal=oauth2_secret=...
+# kubectl get secrets rook-ceph-object-user-<ceph-pool>-object-registry -n <ceph-namespace> -o yaml --export | kubectl replace -f - -n registry
+
+local kube = import "../../../kube/kube.libsonnet";
+local cm = import "cert-manager.libsonnet";
+
+{
+ Environment: {
+ local env = self,
+ local cfg = env.cfg,
+ cfg:: {
+ namespace: "registry",
+ domain: error "domain must be set",
+ storageClassName: error "storageClassName must be set",
+ },
+
+ metadata(component):: {
+ namespace: cfg.namespace,
+ labels: {
+ "app.kubernetes.io/name": "registry",
+ "app.kubernetes.io/managed-by": "kubecfg",
+ "app.kubernetes.io/component": component,
+ },
+ },
+
+ namespace: kube.Namespace(cfg.namespace),
+
+ registryIssuer: cm.Issuer("registry-issuer") {
+ metadata+: env.metadata("registry-issuer"),
+ spec: {
+ selfSigned: {},
+ },
+ },
+ authCertificate: cm.Certificate("auth") {
+ metadata+: env.metadata("auth"),
+ spec: {
+ secretName: "auth-internal",
+ duration: "43800h0m0s", // 5 years
+ issuerRef: {
+ name: env.registryIssuer.metadata.name,
+ },
+ commonName: "auth.registry",
+ },
+ },
+ registryCertificate: cm.Certificate("registry") {
+ metadata+: env.metadata("registry"),
+ spec: {
+ secretName: "registry-internal",
+ duration: "43800h0m0s", // 5 years
+ issuerRef: {
+ name: env.registryIssuer.metadata.name,
+ },
+ commonName: "registry.registry",
+ },
+ },
+
+ registryConfig: kube.ConfigMap("registry-config") {
+ metadata+: env.metadata("registry-config"),
+ data: {
+ "config.yml": std.manifestYamlDoc({
+ version: "0.1",
+ log: {
+ fields: {
+ service: "registry",
+ },
+ },
+ storage: {
+ cache: {
+ blobdescriptor: "inmemory",
+ },
+ s3: {
+ regionendpoint: "https://object.ceph-waw1.hswaw.net",
+ bucket: "registry",
+ region: "waw-hdd-redunant-1-object:default-placement",
+ },
+ },
+ http: {
+ addr: ":5000",
+ headers: {
+ "X-Content-Type-Options": ["nosniff"],
+ },
+ tls: {
+ certificate: "/certs/tls.crt",
+ key: "/certs/tls.key",
+ },
+ debug: {
+ addr: "localhost:5001",
+ },
+ },
+ health: {
+ storagedriver: {
+ enabled: true,
+ interval: "10s",
+ threshold: 3,
+ },
+ },
+ auth: {
+ token: {
+ realm: "https://%s/auth" % [cfg.domain],
+ service: "my.docker.registry",
+ issuer: "%s auth server" % [cfg.domain],
+ rootcertbundle: "/authcerts/tls.crt",
+ },
+ },
+ }),
+ },
+ },
+
+ authVolumeClaim: kube.PersistentVolumeClaim("auth-token-storage") {
+ metadata+: env.metadata("auth-token-storage"),
+ spec+: {
+ storageClassName: cfg.storageClassName,
+ accessModes: [ "ReadWriteOnce" ],
+ resources: {
+ requests: {
+ storage: "1Gi",
+ },
+ },
+ },
+ },
+
+ authConfig: kube.ConfigMap("auth-config") {
+ metadata+: env.metadata("auth-config"),
+ data: {
+ "auth_config.yml": std.manifestYamlDoc({
+ server: {
+ addr: ":5001",
+ certificate: "/certs/tls.crt",
+ key: "/certs/tls.key",
+ },
+ token: {
+ issuer: "%s auth server" % [cfg.domain],
+ expiration: 900,
+ },
+ oauth2: {
+ client_id: "registry",
+ client_secret_file: "/secrets/oauth2_secret",
+ authorize_url: "https://sso.hackerspace.pl/oauth/authorize",
+ access_token_url: "https://sso.hackerspace.pl/oauth/token",
+ profile_url: "https://sso.hackerspace.pl/api/1/profile",
+ redirect_url: "https://registry.k0.hswaw.net/oauth2",
+ username_key: "username",
+ token_db: "/data/oauth2_tokens.ldb",
+ registry_url: "https://registry.k0.hswaw.net",
+ },
+ users: {
+ [""]: {}, // '' user are anonymous users.
+ },
+ local data = self,
+ pushers:: [
+ { who: ["q3k", "inf"], what: "vms/*" },
+ { who: ["q3k", "inf"], what: "app/*" },
+ { who: ["q3k", "inf"], what: "go/svc/*" },
+ ],
+ acl: [
+ {
+ match: {account: "/.+/", name: "${account}/*"},
+ actions: ["*"],
+ comment: "Logged in users have full access to images that are in their 'namespace'",
+ },
+ {
+ match: {account: "/.+/", type: "registry", name: "catalog"},
+ actions: ["*"],
+ comment: "Logged in users can query the catalog.",
+ },
+ {
+ match: {account: ""},
+ actions: ["pull"],
+ comment: "Anyone can pull all images.",
+ },
+ ] + [
+ {
+ match: {
+ account: "/(%s)/" % std.join("|", p.who),
+ name: p.what,
+ },
+ actions: ["*"],
+ comment: "%s can push to %s" % [std.join(", ", p.who), p.what],
+ }
+ for p in data.pushers
+ ],
+ }),
+ }
+ },
+
+ authDeployment: kube.Deployment("auth") {
+ metadata+: env.metadata("auth"),
+ spec+: {
+ replicas: 1,
+ template+: {
+ spec+: {
+ volumes_: {
+ data: kube.PersistentVolumeClaimVolume(env.authVolumeClaim),
+ config: kube.ConfigMapVolume(env.authConfig),
+ certs: {
+ secret: { secretName: env.authCertificate.spec.secretName },
+ },
+ secrets: {
+ secret: { secretName: "auth" },
+ },
+ },
+ containers_: {
+ auth: kube.Container("auth") {
+ image: "informatic/docker_auth:2019040307",
+ volumeMounts_: {
+ config: { mountPath: "/config" },
+ certs: { mountPath: "/certs" },
+ secrets: { mountPath: "/secrets" },
+ data: { mountPath: "/data" },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ authService: kube.Service("auth") {
+ metadata+: env.metadata("auth"),
+ target_pod:: env.authDeployment.spec.template,
+ spec+: {
+ type: "ClusterIP",
+ ports: [
+ { name: "auth", port: 5001, targetPort: 5001, protocol: "TCP" },
+ ],
+ }
+ },
+ registryDeployment: kube.Deployment("docker-registry") {
+ metadata+: env.metadata("docker-registry"),
+ spec+: {
+ replicas: 1,
+ template+: {
+ spec+: {
+ volumes_: {
+ config: kube.ConfigMapVolume(env.registryConfig),
+ certs: {
+ secret: { secretName: env.registryCertificate.spec.secretName },
+ },
+ authcerts: {
+ secret: { secretName: env.authCertificate.spec.secretName },
+ },
+ },
+ containers_: {
+ registry: kube.Container("docker-registry") {
+ image: "registry:2",
+ args: ["/config/config.yml"],
+ volumeMounts_: {
+ config: { mountPath: "/config" },
+ certs: { mountPath: "/certs" },
+ authcerts: { mountPath: "/authcerts" },
+ },
+ env_: {
+ REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: {
+ name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
+ key: "AccessKey"
+ }},
+ REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: {
+ name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
+ key: "SecretKey",
+ }},
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ registryService: kube.Service("docker-registry") {
+ metadata+: env.metadata("docker-registry"),
+ target_pod:: env.registryDeployment.spec.template,
+ spec+: {
+ type: "ClusterIP",
+ ports: [
+ { name: "registry", port: 5000, targetPort: 5000, protocol: "TCP" },
+ ],
+ }
+ },
+ registryIngress: kube.Ingress("registry") {
+ metadata+: env.metadata("registry") {
+ annotations+: {
+ "kubernetes.io/tls-acme": "true",
+ "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
+ "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS",
+ "nginx.ingress.kubernetes.io/proxy-body-size": "0",
+ },
+ },
+ spec+: {
+ tls: [
+ {
+ hosts: [cfg.domain],
+ secretName: "registry-tls",
+ },
+ ],
+ rules: [
+ {
+ host: cfg.domain,
+ http: {
+ paths: [
+ { path: "/auth", backend: env.authService.name_port },
+ { path: "/", backend: env.authService.name_port },
+ { path: "/v2/", backend: env.registryService.name_port },
+ ]
+ },
+ }
+ ],
+ },
+ },
+
+ registryStorageUser: kube._Object("ceph.rook.io/v1", "CephObjectStoreUser", "registry") {
+ metadata+: {
+ namespace: "ceph-waw1",
+ },
+ spec: {
+ store: "waw-hdd-redundant-1-object",
+ displayName: "docker-registry user",
+ },
+ },
+ }
+}
diff --git a/cluster/kube/lib/rook.libsonnet b/cluster/kube/lib/rook.libsonnet
index 5223654..512c6a0 100644
--- a/cluster/kube/lib/rook.libsonnet
+++ b/cluster/kube/lib/rook.libsonnet
@@ -491,6 +491,8 @@
ECBlockPool(cluster, name):: {
local pool = self,
+ name:: name,
+
spec:: error "spec must be specified",
pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {