Merge changes I46da0fc4,Ib187de6d
* changes:
cluster/admitomatic: finish up service
cluster/admitomatic: finish up ingress admission logic
diff --git a/cluster/admitomatic/BUILD.bazel b/cluster/admitomatic/BUILD.bazel
index 5cb23ab..55c7466 100644
--- a/cluster/admitomatic/BUILD.bazel
+++ b/cluster/admitomatic/BUILD.bazel
@@ -5,13 +5,18 @@
srcs = [
"ingress.go",
"main.go",
+ "service.go",
],
importpath = "code.hackerspace.pl/hscloud/cluster/admitomatic",
visibility = ["//visibility:private"],
deps = [
+ "//cluster/admitomatic/config:go_default_library",
"//go/mirko:go_default_library",
"@com_github_golang_glog//:go_default_library",
"@io_k8s_api//admission/v1beta1:go_default_library",
+ "@io_k8s_api//networking/v1beta1:go_default_library",
+ "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+ "@org_golang_google_protobuf//encoding/prototext:go_default_library",
],
)
@@ -25,4 +30,10 @@
name = "go_default_test",
srcs = ["ingress_test.go"],
embed = [":go_default_library"],
+ deps = [
+ "@io_k8s_api//admission/v1beta1:go_default_library",
+ "@io_k8s_api//networking/v1beta1:go_default_library",
+ "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+ "@io_k8s_apimachinery//pkg/runtime:go_default_library",
+ ],
)
diff --git a/cluster/admitomatic/config/BUILD.bazel b/cluster/admitomatic/config/BUILD.bazel
new file mode 100644
index 0000000..0344526
--- /dev/null
+++ b/cluster/admitomatic/config/BUILD.bazel
@@ -0,0 +1,23 @@
+load("@rules_proto//proto:defs.bzl", "proto_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
+
+proto_library(
+ name = "config_proto",
+ srcs = ["config.proto"],
+ visibility = ["//visibility:public"],
+)
+
+go_proto_library(
+ name = "config_go_proto",
+ importpath = "code.hackerspace.pl/hscloud/cluster/admitomatic/config",
+ proto = ":config_proto",
+ visibility = ["//visibility:public"],
+)
+
+go_library(
+ name = "go_default_library",
+ embed = [":config_go_proto"],
+ importpath = "code.hackerspace.pl/hscloud/cluster/admitomatic/config",
+ visibility = ["//visibility:public"],
+)
diff --git a/cluster/admitomatic/config/config.proto b/cluster/admitomatic/config/config.proto
new file mode 100644
index 0000000..460c571
--- /dev/null
+++ b/cluster/admitomatic/config/config.proto
@@ -0,0 +1,37 @@
+syntax = "proto3";
+package config;
+option go_package = "code.hackerspace.pl/hscloud/cluster/admitomatic/config";
+
+// Admitomatic configuration, passed as a text proto, for
+// example:
+//
+// $ cat sample.pb.text
+// allow_domain { namespace: "example" dns: "*.example.com" }
+// allow_domain {
+// namespace: "personal-q3k" dns: "foo.q3k.org"
+// }
+// allow_domain {
+// namespace: "personal-q3k" dns: "bar.q3k.org"
+// }
+//
+message Config {
+ // List of domains that are allowed to be configured as
+ // ingresses in a given namespace. If a domain does not
+ // appear in this list, it will be allowed to run in any
+ // namespace.
+ repeated AllowDomain allow_domain = 1;
+}
+
+message AllowDomain {
+ // namespace is a kubernetes namespace. An empty string is
+ // treated as the 'default' namespace.
+ string namespace = 1;
+ // dns is a domain name like 'example.com' or a wildcard
+ // like '*.foo.example.com'.
+ // Wildcards match domains at any level beneath the root,
+ // so the example above would match 'bar.foo.example.com'
+ // and 'baz.bar.foo.example.com'. However, they do not
+ // catch the root itself, ie. the above would not catch
+ // 'foo.example.com'.
+ string dns = 2;
+}
diff --git a/cluster/admitomatic/config/sample.pb.text b/cluster/admitomatic/config/sample.pb.text
new file mode 100644
index 0000000..21e98bc
--- /dev/null
+++ b/cluster/admitomatic/config/sample.pb.text
@@ -0,0 +1,7 @@
+allow_domain { namespace: "example" dns: "*.example.com" }
+allow_domain {
+ namespace: "personal-q3k" dns: "foo.q3k.org"
+}
+allow_domain {
+ namespace: "personal-q3k" dns: "bar.q3k.org"
+}
diff --git a/cluster/admitomatic/ingress.go b/cluster/admitomatic/ingress.go
index 42cab98..a1d57a5 100644
--- a/cluster/admitomatic/ingress.go
+++ b/cluster/admitomatic/ingress.go
@@ -1,10 +1,14 @@
package main
import (
+ "encoding/json"
"fmt"
"strings"
+ "github.com/golang/glog"
admission "k8s.io/api/admission/v1beta1"
+ networking "k8s.io/api/networking/v1beta1"
+ meta "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// ingressFilter is a filter which allows or denies the creation of an ingress
@@ -125,6 +129,99 @@
if req.Kind.Group != "networking.k8s.io" || req.Kind.Kind != "Ingress" {
return nil, fmt.Errorf("not an ingress")
}
- // TODO(q3k); implement
- return nil, fmt.Errorf("unimplemented")
+
+ result := func(s string, args ...interface{}) (*admission.AdmissionResponse, error) {
+ res := &admission.AdmissionResponse{
+ UID: req.UID,
+ }
+ if s == "" {
+ res.Allowed = true
+ } else {
+ res.Allowed = false
+ res.Result = &meta.Status{
+ Code: 403,
+ Message: fmt.Sprintf("admitomatic: %s", fmt.Sprintf(s, args...)),
+ }
+ }
+ return res, nil
+ }
+
+ // Permit any actions on critical system namespaes. See:
+ // https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/
+ // “Avoiding operating on the kube-system namespace”
+ if req.Namespace == "kube-system" {
+ return result("")
+ }
+
+ switch req.Operation {
+ case "CREATE":
+ case "UPDATE":
+ default:
+ // We only care about creations/updates, everything else is referred to plain RBAC.
+ return result("")
+ }
+
+ ingress := networking.Ingress{}
+ err := json.Unmarshal(req.Object.Raw, &ingress)
+ if err != nil {
+ glog.Errorf("Unmarshaling Ingress failed: %v", err)
+ return result("invalid object")
+ }
+
+ // Check TLS config for hosts.
+ for j, t := range ingress.Spec.TLS {
+ for k, h := range t.Hosts {
+ if strings.Contains(h, "*") {
+ // TODO(q3k): support wildcards
+ return result("wildcard host %q (%d in TLS entry %d) is not permitted", h, k, j)
+ }
+ if !i.domainAllowed(req.Namespace, h) {
+ return result("host %q (%d) in TLS entry %d is not allowed in namespace %q", h, k, j, req.Namespace)
+ }
+ }
+ }
+
+ // Check rules for hosts.
+ for j, r := range ingress.Spec.Rules {
+ h := r.Host
+ // Per IngressRule spec:
+ // If the host is unspecified, the Ingress routes all traffic based
+ // on the specified IngressRuleValue. Host can be "precise" which is
+ // a domain name without the terminating dot of a network host (e.g.
+ // "foo.bar.com") or "wildcard", which is a domain name prefixed with
+ // a single wildcard label (e.g. "*.foo.com").
+ //
+ // We reject everything other than precise hosts.
+ if h == "" {
+ return result("empty host %q (in rule %d) is not permitted", h, j)
+ }
+ if strings.Contains(h, "*") {
+ // TODO(q3k): support wildcards
+ return result("wildcard host %q (in rule %d) is not permitted", h, j)
+ }
+ if !i.domainAllowed(req.Namespace, h) {
+ return result("host %q (in rule %d) is not allowed in namespace %q", h, j, req.Namespace)
+ }
+ }
+
+ // Only allow a trusted subset of n-i-c annotations.
+ // TODO(q3k): allow opt-out for some namespaces
+ allowed := map[string]bool{
+ "proxy-body-size": true,
+ "ssl-redirect": true,
+ "backend-protocol": true,
+ }
+ prefix := "nginx.ingress.kubernetes.io/"
+ for k, _ := range ingress.Annotations {
+ if !strings.HasPrefix(k, prefix) {
+ continue
+ }
+ k = strings.TrimPrefix(k, prefix)
+ if !allowed[k] {
+ return result("forbidden annotation %q", k)
+ }
+ }
+
+ // All clear, accept this Ingress.
+ return result("")
}
diff --git a/cluster/admitomatic/ingress_test.go b/cluster/admitomatic/ingress_test.go
index 91cf2b9..15a6049 100644
--- a/cluster/admitomatic/ingress_test.go
+++ b/cluster/admitomatic/ingress_test.go
@@ -1,6 +1,15 @@
package main
-import "testing"
+import (
+ "encoding/json"
+ "strings"
+ "testing"
+
+ admission "k8s.io/api/admission/v1beta1"
+ networking "k8s.io/api/networking/v1beta1"
+ meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+ runtime "k8s.io/apimachinery/pkg/runtime"
+)
func TestPatterns(t *testing.T) {
f := ingressFilter{}
@@ -76,3 +85,121 @@
}
}
}
+
+func TestIngressPermitted(t *testing.T) {
+ f := ingressFilter{}
+ // Errors discarded, tested in TestPatterns.
+ f.allow("matrix", "matrix.hackerspace.pl")
+ f.allow("ceph-waw3", "*.hackerspace.pl")
+ f.allow("personal-q3k", "*.k0.q3k.org")
+ f.allow("personal-vuko", "shells.vuko.pl")
+ f.allow("minecraft", "*.k0.q3k.org")
+
+ mkReq := func(ns string, annotations map[string]string, is *networking.IngressSpec) *admission.AdmissionRequest {
+ i := &networking.Ingress{
+ Spec: *is,
+ }
+ i.Annotations = annotations
+ raw, err := json.Marshal(i)
+ if err != nil {
+ t.Fatalf("marshaling test ingress: %v", err)
+ }
+ return &admission.AdmissionRequest{
+ UID: "test",
+ Kind: meta.GroupVersionKind{
+ Group: "networking.k8s.io",
+ Version: "v1beta1",
+ Kind: "Ingress",
+ },
+ Namespace: ns,
+ Operation: "CREATE",
+ Object: runtime.RawExtension{
+ Raw: raw,
+ },
+ }
+ }
+
+ for i, el := range []struct {
+ req *admission.AdmissionRequest
+ err string
+ }{
+ // 0: unrelated domain, should be allowed
+ {mkReq("default", nil, &networking.IngressSpec{
+ Rules: []networking.IngressRule{
+ {Host: "example.com"},
+ },
+ TLS: []networking.IngressTLS{
+ {
+ Hosts: []string{"example.com"},
+ },
+ },
+ }), ""},
+ // 1: permitted restricted domain, should be allowed
+ {mkReq("matrix", nil, &networking.IngressSpec{
+ Rules: []networking.IngressRule{
+ {Host: "matrix.hackerspace.pl"},
+ },
+ TLS: []networking.IngressTLS{
+ {
+ Hosts: []string{"matrix.hackerspace.pl"},
+ },
+ },
+ }), ""},
+ // 2: forbidden restricted domain, should be rejected
+ {mkReq("personal-hacker", nil, &networking.IngressSpec{
+ Rules: []networking.IngressRule{
+ {Host: "matrix.hackerspace.pl"},
+ },
+ TLS: []networking.IngressTLS{
+ {
+ Hosts: []string{"matrix.hackerspace.pl"},
+ },
+ },
+ }), "not allowed in namespace"},
+ // 3: weird ingress but okay
+ {mkReq("personal-hacker", nil, &networking.IngressSpec{}), ""},
+ // 4: janky annotations, should be rejected
+ {mkReq("matrix", map[string]string{
+ "nginx.ingress.kubernetes.io/configuration-snippet": "omghax",
+ }, &networking.IngressSpec{
+ Rules: []networking.IngressRule{
+ {Host: "matrix.hackerspace.pl"},
+ },
+ TLS: []networking.IngressTLS{
+ {
+ Hosts: []string{"matrix.hackerspace.pl"},
+ },
+ },
+ }), "forbidden annotation"},
+ // 5: accepted annotations, should be allowed
+ {mkReq("matrix", map[string]string{
+ "nginx.ingress.kubernetes.io/proxy-body-size": "2137",
+ "foo.q3k.org/bar": "baz",
+ }, &networking.IngressSpec{
+ Rules: []networking.IngressRule{
+ {Host: "matrix.hackerspace.pl"},
+ },
+ TLS: []networking.IngressTLS{
+ {
+ Hosts: []string{"matrix.hackerspace.pl"},
+ },
+ },
+ }), ""},
+ } {
+ res, err := f.admit(el.req)
+ if err != nil {
+ t.Errorf("test %d: admit: %v", i, err)
+ }
+ if el.err == "" {
+ if !res.Allowed {
+ t.Errorf("test %d: wanted allow, got %q", i, res.Result.Message)
+ }
+ } else {
+ if res.Allowed {
+ t.Errorf("test %d: wanted %q, got allowed", i, el.err)
+ } else if !strings.Contains(res.Result.Message, el.err) {
+ t.Errorf("test %d: wanted %q, got %q", i, el.err, res.Result.Message)
+ }
+ }
+ }
+}
diff --git a/cluster/admitomatic/main.go b/cluster/admitomatic/main.go
index 3178818..b05ff2f 100644
--- a/cluster/admitomatic/main.go
+++ b/cluster/admitomatic/main.go
@@ -3,6 +3,7 @@
import (
"context"
"flag"
+ "io/ioutil"
"net/http"
"time"
@@ -11,13 +12,29 @@
)
var (
- flagListen = "127.0.0.1:8080"
+ flagListen = "127.0.0.1:8080"
+ flagConfig = ""
+ flagTLSKey = ""
+ flagTLSCert = ""
)
func main() {
- flag.StringVar(&flagListen, "pub_listen", flagListen, "Address to listen on for HTTP traffic")
+ flag.StringVar(&flagListen, "admitomatic_listen", flagListen, "Address to listen on for HTTP traffic")
+ flag.StringVar(&flagTLSKey, "admitomatic_tls_key", flagTLSKey, "TLS key to serve HTTP with")
+ flag.StringVar(&flagTLSCert, "admitomatic_tls_cert", flagTLSCert, "TLS certificate to serve HTTP with")
+ flag.StringVar(&flagConfig, "admitomatic_config", flagConfig, "Config path (prototext format)")
flag.Parse()
+ if flagConfig == "" {
+ glog.Exitf("-admitomatic_config must be set")
+ }
+ if flagTLSKey == "" {
+ glog.Exitf("-admitomatic_tls_key must be set")
+ }
+ if flagTLSCert == "" {
+ glog.Exitf("-admitomatic_tls_cert must be set")
+ }
+
m := mirko.New()
if err := m.Listen(); err != nil {
glog.Exitf("Listen(): %v", err)
@@ -27,13 +44,24 @@
glog.Exitf("Serve(): %v", err)
}
+ configData, err := ioutil.ReadFile(flagConfig)
+ if err != nil {
+ glog.Exitf("Could not read config: %v", err)
+ }
+
+ s, err := newService(configData)
+ if err != nil {
+ glog.Exitf("Could not start service: %v", err)
+ }
+
mux := http.NewServeMux()
+ mux.HandleFunc("/", s.handler)
// TODO(q3k): implement admission controller
srv := &http.Server{Addr: flagListen, Handler: mux}
glog.Infof("Listening on %q...", flagListen)
go func() {
- if err := srv.ListenAndServe(); err != nil {
+ if err := srv.ListenAndServeTLS(flagTLSCert, flagTLSKey); err != nil {
glog.Error(err)
}
}()
diff --git a/cluster/admitomatic/service.go b/cluster/admitomatic/service.go
new file mode 100644
index 0000000..8fa2698
--- /dev/null
+++ b/cluster/admitomatic/service.go
@@ -0,0 +1,105 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+
+ "github.com/golang/glog"
+ "google.golang.org/protobuf/encoding/prototext"
+ admission "k8s.io/api/admission/v1beta1"
+ meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ pb "code.hackerspace.pl/hscloud/cluster/admitomatic/config"
+)
+
+type service struct {
+ ingress ingressFilter
+}
+
+// newService creates an admitomatic service from a given prototext config.
+func newService(configuration []byte) (*service, error) {
+ var cfg pb.Config
+ if err := prototext.Unmarshal(configuration, &cfg); err != nil {
+ return nil, fmt.Errorf("parsing config: %v", err)
+ }
+
+ s := service{}
+
+ for i, ad := range cfg.AllowDomain {
+ if ad.Namespace == "" {
+ ad.Namespace = "default"
+ }
+ if ad.Dns == "" {
+ return nil, fmt.Errorf("config entry %d: dns must be set", i)
+ }
+ if err := s.ingress.allow(ad.Namespace, ad.Dns); err != nil {
+ return nil, fmt.Errorf("config entry %d: %v", i, err)
+ }
+ glog.Infof("Ingress: allowing %s in %s", ad.Dns, ad.Namespace)
+ }
+ return &s, nil
+}
+
+// handler is the main HTTP handler of the admitomatic service. It servers the
+// AdmissionReview API, and is called by the Kubernetes API server to
+// permit/deny creation/updating of resources.
+func (s *service) handler(w http.ResponseWriter, r *http.Request) {
+ var body []byte
+ if r.Body != nil {
+ if data, err := ioutil.ReadAll(r.Body); err == nil {
+ body = data
+ }
+ }
+
+ if r.Method != "POST" {
+ glog.Errorf("%s %s: invalid method", r.Method, r.URL)
+ return
+ }
+
+ contentType := r.Header.Get("Content-Type")
+ if contentType != "application/json" {
+ glog.Errorf("%s %s: invalid content-type", r.Method, r.URL)
+ return
+ }
+
+ var review admission.AdmissionReview
+ if err := json.Unmarshal(body, &review); err != nil {
+ glog.Errorf("%s %s: cannot decode: %v", r.Method, r.URL, err)
+ return
+ }
+
+ if review.Kind != "AdmissionReview" {
+ glog.Errorf("%s %s: invalid Kind (%q)", r.Method, r.URL, review.Kind)
+ return
+ }
+
+ var err error
+ req := review.Request
+ resp := &admission.AdmissionResponse{
+ UID: req.UID,
+ Allowed: true,
+ }
+ switch {
+ case req.Kind.Group == "networking.k8s.io" && req.Kind.Kind == "Ingress":
+ resp, err = s.ingress.admit(req)
+ if err != nil {
+ glog.Errorf("%s %s %s: %v", req.Operation, req.Name, req.Namespace, err)
+ // Fail safe.
+ // TODO(q3k): monitor this?
+ resp = &admission.AdmissionResponse{
+ UID: req.UID,
+ Allowed: false,
+ Result: &meta.Status{
+ Code: 500,
+ Message: "admitomatic: internal server error",
+ },
+ }
+ }
+ }
+
+ glog.Infof("%s %s %s in %s: %v (%v)", req.Operation, req.Kind.Kind, req.Name, req.Namespace, resp.Allowed, resp.Result)
+ review.Response = resp
+ json.NewEncoder(w).Encode(review)
+}