Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 1 | package mirko |
| 2 | |
| 3 | import ( |
Serge Bazanski | aa81aa2 | 2018-10-14 08:36:05 -0700 | [diff] [blame] | 4 | "context" |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 5 | "flag" |
| 6 | "fmt" |
| 7 | "net" |
| 8 | "net/http" |
Sergiusz Bazanski | 400ac7a | 2020-01-23 14:17:30 +0100 | [diff] [blame] | 9 | "net/http/pprof" |
Serge Bazanski | 4f7ee51 | 2018-10-14 18:20:59 +0100 | [diff] [blame] | 10 | "os" |
| 11 | "os/signal" |
Sergiusz Bazanski | 8fe651b | 2019-07-21 23:50:05 +0200 | [diff] [blame] | 12 | "sort" |
| 13 | "strings" |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 14 | "time" |
| 15 | |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 16 | "github.com/golang/glog" |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 17 | "golang.org/x/net/trace" |
| 18 | "google.golang.org/grpc" |
| 19 | "google.golang.org/grpc/reflection" |
Sergiusz Bazanski | 8fe651b | 2019-07-21 23:50:05 +0200 | [diff] [blame] | 20 | "k8s.io/client-go/kubernetes" |
| 21 | |
| 22 | "code.hackerspace.pl/hscloud/go/pki" |
| 23 | "code.hackerspace.pl/hscloud/go/statusz" |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 24 | ) |
| 25 | |
| 26 | var ( |
| 27 | flagListenAddress string |
| 28 | flagDebugAddress string |
Serge Bazanski | 446c9e1 | 2018-10-14 17:06:09 +0100 | [diff] [blame] | 29 | flagDebugAllowAll bool |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 30 | ) |
| 31 | |
| 32 | func init() { |
Serge Bazanski | 69de9cb | 2018-10-14 08:49:04 -0700 | [diff] [blame] | 33 | flag.StringVar(&flagListenAddress, "listen_address", "127.0.0.1:4200", "gRPC listen address") |
| 34 | flag.StringVar(&flagDebugAddress, "debug_address", "127.0.0.1:4201", "HTTP debug/status listen address") |
Serge Bazanski | f77e7d3 | 2018-10-14 17:11:08 +0100 | [diff] [blame] | 35 | flag.BoolVar(&flagDebugAllowAll, "debug_allow_all", false, "HTTP debug/status available to everyone") |
Serge Bazanski | aa81aa2 | 2018-10-14 08:36:05 -0700 | [diff] [blame] | 36 | flag.Set("logtostderr", "true") |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 37 | } |
| 38 | |
| 39 | type Mirko struct { |
| 40 | grpcListen net.Listener |
| 41 | grpcServer *grpc.Server |
| 42 | httpListen net.Listener |
| 43 | httpServer *http.Server |
| 44 | httpMux *http.ServeMux |
Serge Bazanski | 4f7ee51 | 2018-10-14 18:20:59 +0100 | [diff] [blame] | 45 | |
Sergiusz Bazanski | 8fe651b | 2019-07-21 23:50:05 +0200 | [diff] [blame] | 46 | kubernetesCS *kubernetes.Clientset |
| 47 | |
Sergiusz Bazanski | 9dc4b68 | 2019-04-05 23:51:49 +0200 | [diff] [blame] | 48 | ctx context.Context |
| 49 | cancel context.CancelFunc |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 50 | } |
| 51 | |
| 52 | func New() *Mirko { |
Serge Bazanski | 4f7ee51 | 2018-10-14 18:20:59 +0100 | [diff] [blame] | 53 | ctx, cancel := context.WithCancel(context.Background()) |
| 54 | return &Mirko{ |
Sergiusz Bazanski | 9dc4b68 | 2019-04-05 23:51:49 +0200 | [diff] [blame] | 55 | ctx: ctx, |
| 56 | cancel: cancel, |
Serge Bazanski | 4f7ee51 | 2018-10-14 18:20:59 +0100 | [diff] [blame] | 57 | } |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 58 | } |
| 59 | |
Serge Bazanski | 446c9e1 | 2018-10-14 17:06:09 +0100 | [diff] [blame] | 60 | func authRequest(req *http.Request) (any, sensitive bool) { |
| 61 | host, _, err := net.SplitHostPort(req.RemoteAddr) |
| 62 | if err != nil { |
| 63 | host = req.RemoteAddr |
| 64 | } |
| 65 | |
| 66 | if flagDebugAllowAll { |
| 67 | return true, true |
| 68 | } |
| 69 | |
| 70 | switch host { |
| 71 | case "localhost", "127.0.0.1", "::1": |
| 72 | return true, true |
| 73 | default: |
| 74 | return false, false |
| 75 | } |
| 76 | } |
| 77 | |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 78 | func (m *Mirko) Listen() error { |
| 79 | grpc.EnableTracing = true |
Serge Bazanski | 446c9e1 | 2018-10-14 17:06:09 +0100 | [diff] [blame] | 80 | trace.AuthRequest = authRequest |
| 81 | |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 82 | grpcLis, err := net.Listen("tcp", flagListenAddress) |
| 83 | if err != nil { |
| 84 | return fmt.Errorf("net.Listen: %v", err) |
| 85 | } |
| 86 | m.grpcListen = grpcLis |
Serge Bazanski | dd3d40f | 2018-10-25 12:14:18 +0100 | [diff] [blame] | 87 | m.grpcServer = grpc.NewServer(pki.WithServerHSPKI()...) |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 88 | reflection.Register(m.grpcServer) |
| 89 | |
| 90 | httpLis, err := net.Listen("tcp", flagDebugAddress) |
| 91 | if err != nil { |
| 92 | return fmt.Errorf("net.Listen: %v", err) |
| 93 | } |
| 94 | |
| 95 | m.httpMux = http.NewServeMux() |
| 96 | // Canonical URLs |
Serge Bazanski | 446c9e1 | 2018-10-14 17:06:09 +0100 | [diff] [blame] | 97 | m.httpMux.HandleFunc("/debug/status", func(w http.ResponseWriter, r *http.Request) { |
Serge Bazanski | bbb5b5f | 2018-10-14 17:13:16 +0100 | [diff] [blame] | 98 | any, _ := authRequest(r) |
Serge Bazanski | 446c9e1 | 2018-10-14 17:06:09 +0100 | [diff] [blame] | 99 | if !any { |
| 100 | http.Error(w, "not allowed", http.StatusUnauthorized) |
| 101 | return |
| 102 | } |
| 103 | statusz.StatusHandler(w, r) |
| 104 | }) |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 105 | m.httpMux.HandleFunc("/debug/requests", trace.Traces) |
| 106 | |
Sergiusz Bazanski | 400ac7a | 2020-01-23 14:17:30 +0100 | [diff] [blame] | 107 | // pprof endpoints |
| 108 | m.httpMux.HandleFunc("/debug/pprof/", pprof.Index) |
| 109 | m.httpMux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) |
| 110 | m.httpMux.HandleFunc("/debug/pprof/profile", pprof.Profile) |
| 111 | m.httpMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) |
| 112 | m.httpMux.HandleFunc("/debug/pprof/trace", pprof.Trace) |
| 113 | |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 114 | // -z legacy URLs |
| 115 | m.httpMux.HandleFunc("/statusz", func(w http.ResponseWriter, r *http.Request) { |
| 116 | http.Redirect(w, r, "/debug/status", http.StatusSeeOther) |
| 117 | }) |
| 118 | m.httpMux.HandleFunc("/rpcz", func(w http.ResponseWriter, r *http.Request) { |
| 119 | http.Redirect(w, r, "/debug/requests", http.StatusSeeOther) |
| 120 | }) |
| 121 | m.httpMux.HandleFunc("/requestz", func(w http.ResponseWriter, r *http.Request) { |
| 122 | http.Redirect(w, r, "/debug/requests", http.StatusSeeOther) |
| 123 | }) |
Sergiusz Bazanski | 400ac7a | 2020-01-23 14:17:30 +0100 | [diff] [blame] | 124 | m.httpMux.HandleFunc("/profilez", func(w http.ResponseWriter, r *http.Request) { |
| 125 | http.Redirect(w, r, "/debug/pprof", http.StatusSeeOther) |
| 126 | }) |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 127 | |
| 128 | // root redirect |
| 129 | m.httpMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { |
| 130 | http.Redirect(w, r, "/debug/status", http.StatusSeeOther) |
| 131 | }) |
| 132 | |
Sergiusz Bazanski | 8fe651b | 2019-07-21 23:50:05 +0200 | [diff] [blame] | 133 | m.kubernetesConnect() |
| 134 | |
| 135 | debugParts := strings.Split(flagDebugAddress, ":") |
| 136 | debugPort := debugParts[len(debugParts)-1] |
| 137 | statusz.PublicAddress = fmt.Sprintf("http://%s:%s/", m.Address().String(), debugPort) |
| 138 | |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 139 | m.httpListen = httpLis |
| 140 | m.httpServer = &http.Server{ |
| 141 | Addr: flagDebugAddress, |
| 142 | Handler: m.httpMux, |
| 143 | } |
| 144 | |
| 145 | return nil |
| 146 | } |
| 147 | |
Serge Bazanski | 4f7ee51 | 2018-10-14 18:20:59 +0100 | [diff] [blame] | 148 | // Trace logs debug information to either a context trace (if present) |
| 149 | // or stderr (if not) |
Serge Bazanski | c99e0b9 | 2018-10-14 18:03:48 +0100 | [diff] [blame] | 150 | func Trace(ctx context.Context, f string, args ...interface{}) { |
Serge Bazanski | aa81aa2 | 2018-10-14 08:36:05 -0700 | [diff] [blame] | 151 | tr, ok := trace.FromContext(ctx) |
| 152 | if !ok { |
| 153 | fmtd := fmt.Sprintf(f, args...) |
| 154 | glog.Warningf("No trace in %v: %s", ctx, fmtd) |
| 155 | return |
| 156 | } |
| 157 | tr.LazyPrintf(f, args...) |
| 158 | } |
| 159 | |
Serge Bazanski | 4f7ee51 | 2018-10-14 18:20:59 +0100 | [diff] [blame] | 160 | // GRPC returns the microservice's grpc.Server object |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 161 | func (m *Mirko) GRPC() *grpc.Server { |
| 162 | if m.grpcServer == nil { |
| 163 | panic("GRPC() called before Listen()") |
| 164 | } |
| 165 | return m.grpcServer |
| 166 | } |
| 167 | |
Serge Bazanski | 4f7ee51 | 2018-10-14 18:20:59 +0100 | [diff] [blame] | 168 | // HTTPMux returns the microservice's debug HTTP mux |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 169 | func (m *Mirko) HTTPMux() *http.ServeMux { |
| 170 | if m.httpMux == nil { |
| 171 | panic("HTTPMux() called before Listen()") |
| 172 | } |
| 173 | return m.httpMux |
| 174 | } |
| 175 | |
Serge Bazanski | 4f7ee51 | 2018-10-14 18:20:59 +0100 | [diff] [blame] | 176 | // Context returns a background microservice context that will be canceled |
| 177 | // when the service is shut down |
| 178 | func (m *Mirko) Context() context.Context { |
| 179 | return m.ctx |
| 180 | } |
| 181 | |
| 182 | // Done() returns a channel that will emit a value when the service is |
| 183 | // shut down. This should be used in the main() function instead of a select{} |
| 184 | // call, to allow the background context to be canceled fully. |
Sergiusz Bazanski | 9dc4b68 | 2019-04-05 23:51:49 +0200 | [diff] [blame] | 185 | func (m *Mirko) Done() <-chan struct{} { |
| 186 | return m.Context().Done() |
Serge Bazanski | 4f7ee51 | 2018-10-14 18:20:59 +0100 | [diff] [blame] | 187 | } |
| 188 | |
| 189 | // Serve starts serving HTTP and gRPC requests |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 190 | func (m *Mirko) Serve() error { |
| 191 | errs := make(chan error, 1) |
| 192 | go func() { |
| 193 | if err := m.grpcServer.Serve(m.grpcListen); err != nil { |
| 194 | errs <- err |
| 195 | } |
| 196 | }() |
| 197 | go func() { |
| 198 | if err := m.httpServer.Serve(m.httpListen); err != nil { |
| 199 | errs <- err |
| 200 | } |
| 201 | }() |
| 202 | |
Serge Bazanski | 4f7ee51 | 2018-10-14 18:20:59 +0100 | [diff] [blame] | 203 | signalCh := make(chan os.Signal, 1) |
| 204 | signal.Notify(signalCh, os.Interrupt) |
| 205 | go func() { |
| 206 | select { |
| 207 | case <-signalCh: |
| 208 | m.cancel() |
Serge Bazanski | 4f7ee51 | 2018-10-14 18:20:59 +0100 | [diff] [blame] | 209 | } |
| 210 | }() |
| 211 | |
Serge Bazanski | 3fd70d8 | 2018-10-14 08:12:46 -0700 | [diff] [blame] | 212 | ticker := time.NewTicker(1 * time.Second) |
| 213 | select { |
| 214 | case <-ticker.C: |
| 215 | glog.Infof("gRPC listening on %s", flagListenAddress) |
| 216 | glog.Infof("HTTP listening on %s", flagDebugAddress) |
| 217 | return nil |
| 218 | case err := <-errs: |
| 219 | return err |
| 220 | } |
| 221 | } |
Sergiusz Bazanski | 8fe651b | 2019-07-21 23:50:05 +0200 | [diff] [blame] | 222 | |
| 223 | // Address returns a linkable address where this service is running, sans port. |
| 224 | // If running within kubernetes, this will return the pod IP. |
| 225 | // Otherwise, this will guess the main, 'external' IP address of the machine it's running on. |
| 226 | // On failures, returns loopback address. |
| 227 | func (m *Mirko) Address() net.IP { |
| 228 | // If we're not running in Kubernetes and binding to 127.0.0.1, return loopback. |
| 229 | if m.kubernetesCS == nil && strings.HasPrefix(flagListenAddress, "127.0.0.1:") { |
| 230 | return net.ParseIP("127.0.0.1") |
| 231 | } |
| 232 | |
| 233 | ifaces, err := net.Interfaces() |
| 234 | if err != nil { |
| 235 | glog.Errorf("net.Interface(): %v", err) |
| 236 | return net.ParseIP("127.0.0.1") |
| 237 | } |
| 238 | |
| 239 | addrmap := make(map[string]net.IP) |
| 240 | |
| 241 | for _, iface := range ifaces { |
| 242 | addrs, err := iface.Addrs() |
| 243 | if err != nil { |
| 244 | glog.Errorf("iface(%q).Addrs(): %v", iface.Name, err) |
| 245 | continue |
| 246 | } |
| 247 | |
| 248 | for _, addr := range addrs { |
| 249 | var ip net.IP |
| 250 | switch v := addr.(type) { |
| 251 | case *net.IPNet: |
| 252 | ip = v.IP |
| 253 | case *net.IPAddr: |
| 254 | ip = v.IP |
| 255 | default: |
| 256 | continue |
| 257 | } |
| 258 | |
| 259 | if strings.HasPrefix(ip.String(), "fe80:") { |
| 260 | continue |
| 261 | } |
| 262 | addrmap[iface.Name] = ip |
| 263 | } |
| 264 | } |
| 265 | |
| 266 | if m.kubernetesCS != nil { |
| 267 | addr, ok := addrmap["eth0"] |
| 268 | if !ok { |
| 269 | glog.Errorf("Running on Kubernetes but no eth0! Available interfaces: %v", addrmap) |
| 270 | return net.ParseIP("127.0.0.1") |
| 271 | } |
| 272 | |
| 273 | return addr |
| 274 | } |
| 275 | |
| 276 | if len(addrmap) == 0 { |
| 277 | glog.Errorf("No interfaces found!") |
| 278 | return net.ParseIP("127.0.0.1") |
| 279 | } |
| 280 | |
| 281 | // Heuristics ahoy! |
| 282 | prioritized := []*ifaceWithPriority{} |
| 283 | for iface, addr := range addrmap { |
| 284 | prio := &ifaceWithPriority{ |
| 285 | iface: iface, |
| 286 | addr: addr, |
| 287 | } |
| 288 | switch { |
| 289 | case strings.HasPrefix(iface, "lo"): |
| 290 | prio.priority = -10 |
| 291 | case strings.HasPrefix(iface, "tap"): |
| 292 | prio.priority = -5 |
| 293 | case strings.HasPrefix(iface, "tun"): |
| 294 | prio.priority = -5 |
| 295 | case strings.HasPrefix(iface, "veth"): |
| 296 | prio.priority = 5 |
| 297 | case strings.HasPrefix(iface, "wl"): |
| 298 | prio.priority = 5 |
| 299 | case strings.HasPrefix(iface, "enp"): |
| 300 | prio.priority = 10 |
| 301 | case strings.HasPrefix(iface, "eth"): |
| 302 | prio.priority = 10 |
| 303 | } |
| 304 | |
| 305 | prioritized = append(prioritized, prio) |
| 306 | } |
| 307 | |
| 308 | sort.Slice(prioritized, func(i, j int) bool { return prioritized[i].priority > prioritized[j].priority }) |
| 309 | return prioritized[0].addr |
| 310 | } |
| 311 | |
| 312 | type ifaceWithPriority struct { |
| 313 | iface string |
| 314 | addr net.IP |
| 315 | priority int |
| 316 | } |