teleimg: init
This is a shitty small proxy to unfuck telegram's bot image URLs, ie. do
not add content-disposition and send a proper MIME in content-type.
It also does some local caching and hides the Telegram API token.
Change-Id: I0afb29ca3f1807a13fa157fdcf486ee4c857f08d
diff --git a/WORKSPACE b/WORKSPACE
index 3d6c2c2..391980c 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -1884,3 +1884,23 @@
importpath = "go.uber.org/zap",
tag = "v1.10.0",
)
+
+go_repository(
+ name = "com_github_dgraph_io_ristretto",
+ commit = "83508260cb49a2c3261c2774c991870fd18b5a1b",
+ importpath = "github.com/dgraph-io/ristretto",
+)
+
+go_repository(
+ name = "com_github_cespare_xxhash",
+ commit = "d7df74196a9e781ede915320c11c378c1b2f3a1f",
+ importpath = "github.com/cespare/xxhash",
+)
+
+go_repository(
+ name = "com_github_ulule_limiter_v3",
+ commit = "6911899e37a5788df86f770b3f85c1c3eb0313d5",
+ importpath = "github.com/ulule/limiter/v3",
+ remote = "https://github.com/ulule/limiter",
+ vcs = "git",
+)
diff --git a/hswaw/kube/hswaw.jsonnet b/hswaw/kube/hswaw.jsonnet
index 905d964..49c9aa3 100644
--- a/hswaw/kube/hswaw.jsonnet
+++ b/hswaw/kube/hswaw.jsonnet
@@ -3,6 +3,7 @@
local smsgw = import "smsgw.libsonnet";
local ldapweb = import "ldapweb.libsonnet";
+local teleimg = import "teleimg.libsonnet";
{
hswaw(name):: mirko.Environment(name) {
@@ -12,11 +13,13 @@
cfg+: {
smsgw: smsgw.cfg,
ldapweb: ldapweb.cfg,
+ teleimg: teleimg.cfg,
},
components: {
smsgw: smsgw.component(cfg.smsgw, env),
ldapweb: ldapweb.component(cfg.ldapweb, env),
+ teleimg: teleimg.component(cfg.teleimg, env),
},
},
@@ -31,6 +34,12 @@
ldapweb+: {
webFQDN: "profile.hackerspace.pl",
},
+ teleimg+: {
+ webFQDN: "teleimg.hswaw.net",
+ secret+: {
+ telegram_token: std.base64(std.split(importstr "secrets/plain/prod-telegram-token", "\n")[0]),
+ },
+ },
},
},
}
diff --git a/hswaw/kube/teleimg.libsonnet b/hswaw/kube/teleimg.libsonnet
new file mode 100644
index 0000000..58026fb
--- /dev/null
+++ b/hswaw/kube/teleimg.libsonnet
@@ -0,0 +1,41 @@
+local mirko = import "../../kube/mirko.libsonnet";
+local kube = import "../../kube/kube.libsonnet";
+
+{
+ cfg:: {
+ secret: {
+ telegram_token: error "telegram_token must be set",
+ },
+ image: "registry.k0.hswaw.net/q3k/teleimg:1578240550-1525c84e4cef4f382e2dca2210f31830533dc7c4",
+ webFQDN: error "webFQDN must be set!",
+ },
+
+ component(cfg, env):: mirko.Component(env, "teleimg") {
+ local teleimg = self,
+ cfg+: {
+ image: cfg.image,
+ container: teleimg.GoContainer("main", "/teleimg/teleimg") {
+ env_: {
+ TELEGRAM_TOKEN: kube.SecretKeyRef(teleimg.secret, "telegram_token"),
+ },
+ command+: [
+ "-public_listen", "0.0.0.0:5000",
+ "-telegram_token", "$(TELEGRAM_TOKEN)",
+ ],
+ },
+ ports+: {
+ publicHTTP: {
+ public: {
+ port: 5000,
+ dns: cfg.webFQDN,
+ },
+ },
+ },
+ },
+
+ secret: kube.Secret("teleimg") {
+ metadata+: teleimg.metadata,
+ data: cfg.secret,
+ },
+ },
+}
diff --git a/personal/q3k/teleimg/BUILD b/personal/q3k/teleimg/BUILD
new file mode 100644
index 0000000..8101719
--- /dev/null
+++ b/personal/q3k/teleimg/BUILD
@@ -0,0 +1,48 @@
+load("@io_bazel_rules_docker//container:container.bzl", "container_image", "container_layer", "container_push")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["main.go"],
+ importpath = "code.hackerspace.pl/hscloud/personal/q3k/teleimg",
+ visibility = ["//visibility:private"],
+ deps = [
+ "//go/mirko:go_default_library",
+ "@com_github_dgraph_io_ristretto//:go_default_library",
+ "@com_github_go_telegram_bot_api_telegram_bot_api//:go_default_library",
+ "@com_github_golang_glog//:go_default_library",
+ "@com_github_ulule_limiter_v3//:go_default_library",
+ "@com_github_ulule_limiter_v3//drivers/store/memory:go_default_library",
+ ],
+)
+
+go_binary(
+ name = "teleimg",
+ embed = [":go_default_library"],
+ visibility = ["//visibility:public"],
+)
+
+container_layer(
+ name = "layer_bin",
+ files = [
+ ":teleimg",
+ ],
+ directory = "/teleimg/",
+)
+
+container_image(
+ name = "runtime",
+ base = "@prodimage-bionic//image",
+ layers = [
+ ":layer_bin",
+ ],
+)
+
+container_push(
+ name = "push",
+ image = ":runtime",
+ format = "Docker",
+ registry = "registry.k0.hswaw.net",
+ repository = "q3k/teleimg",
+ tag = "{BUILD_TIMESTAMP}-{STABLE_GIT_COMMIT}",
+)
diff --git a/personal/q3k/teleimg/README.md b/personal/q3k/teleimg/README.md
new file mode 100644
index 0000000..b97b5f8
--- /dev/null
+++ b/personal/q3k/teleimg/README.md
@@ -0,0 +1,10 @@
+Teleimg
+=======
+
+A small proxy to retrieve and get telegram file by FileID.
+
+For any fileid, you can request:
+
+ https://<teleimg>/fileid/<fileid>.jpg
+
+It will be served with the most sensible MIME headers according to the requested extension.
diff --git a/personal/q3k/teleimg/main.go b/personal/q3k/teleimg/main.go
new file mode 100644
index 0000000..85cce31
--- /dev/null
+++ b/personal/q3k/teleimg/main.go
@@ -0,0 +1,190 @@
+package main
+
+import (
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "regexp"
+
+ "code.hackerspace.pl/hscloud/go/mirko"
+ "github.com/dgraph-io/ristretto"
+ tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api"
+ "github.com/golang/glog"
+ "github.com/ulule/limiter/v3"
+ "github.com/ulule/limiter/v3/drivers/store/memory"
+)
+
+func init() {
+ flag.Set("logtostderr", "true")
+}
+
+var (
+ flagPublicListen string
+ flagTelegramToken string
+ reTelegram = regexp.MustCompile(`/fileid/([a-zA-Z0-9']+).([a-z0-9]+)`)
+)
+
+type server struct {
+ cache *ristretto.Cache
+ limiter *limiter.Limiter
+ tel *tgbotapi.BotAPI
+}
+
+func main() {
+ flag.StringVar(&flagPublicListen, "public_listen", "127.0.0.1:5000", "Listen address for public HTTP handler")
+ flag.StringVar(&flagTelegramToken, "telegram_token", "", "Telegram Bot API Token")
+ flag.Parse()
+
+ if flagTelegramToken == "" {
+ glog.Exitf("telegram_token must be set")
+ }
+
+ cache, err := ristretto.NewCache(&ristretto.Config{
+ NumCounters: 1e7, // number of keys to track frequency of (10M).
+ MaxCost: 1 << 30, // maximum cost of cache (1GB).
+ BufferItems: 64, // number of keys per Get buffer.
+ })
+ if err != nil {
+ glog.Exit(err)
+ }
+
+ tel, err := tgbotapi.NewBotAPI(flagTelegramToken)
+ if err != nil {
+ glog.Exitf("Error when creating telegram bot: %v", err)
+ }
+
+ rate, err := limiter.NewRateFromFormatted("10-M")
+ if err != nil {
+ glog.Exit(err)
+ }
+
+ store := memory.NewStore()
+ instance := limiter.New(store, rate, limiter.WithTrustForwardHeader(true))
+
+ s := &server{
+ cache: cache,
+ limiter: instance,
+ tel: tel,
+ }
+
+ m := mirko.New()
+ if err := m.Listen(); err != nil {
+ glog.Exitf("Listen(): %v", err)
+ }
+
+ if err := m.Serve(); err != nil {
+ glog.Exitf("Serve(): %v", err)
+ }
+
+ publicMux := http.NewServeMux()
+ publicMux.HandleFunc("/", s.publicHandler)
+ publicSrv := http.Server{
+ Addr: flagPublicListen,
+ Handler: publicMux,
+ }
+ go func() {
+ if err := publicSrv.ListenAndServe(); err != nil {
+ glog.Exitf("public ListenAndServe: %v", err)
+ }
+ }()
+
+ <-m.Done()
+}
+
+func setMime(w http.ResponseWriter, ext string) {
+ switch ext {
+ case "jpg":
+ w.Header().Set("Content-Type", "image/jpeg")
+ case "mp4":
+ w.Header().Set("Content-Type", "video/mp4")
+ }
+}
+
+func (s *server) publicHandler(w http.ResponseWriter, r *http.Request) {
+ ctx := r.Context()
+
+ if !reTelegram.MatchString(r.URL.Path) {
+ http.NotFound(w, r)
+ return
+ }
+ parts := reTelegram.FindStringSubmatch(r.URL.Path)
+ fileid := parts[1]
+ fileext := parts[2]
+ glog.Infof("FileID: %s", fileid)
+
+ c, ok := s.cache.Get(fileid)
+ if ok {
+ glog.Infof("Get %q - cache hit", fileid)
+ // cache hit
+ setMime(w, fileext)
+ w.Write(c.([]byte))
+ return
+ }
+
+ glog.Infof("Get %q - cache miss", fileid)
+
+ limit, err := s.limiter.Get(ctx, s.limiter.GetIPKey(r))
+ if err != nil {
+ w.WriteHeader(500)
+ fmt.Fprintf(w, ":(")
+ glog.Errorf("limiter.Get(%q): %v", s.limiter.GetIPKey(r), err)
+ return
+ }
+
+ if limit.Reached {
+ w.WriteHeader(420)
+ fmt.Fprintf(w, "enhance your calm")
+ glog.Warningf("Limit reached by %q", s.limiter.GetIPKey(r))
+ return
+ }
+
+ f, err := s.tel.GetFile(tgbotapi.FileConfig{fileid})
+ if err != nil {
+ w.WriteHeader(502)
+ fmt.Fprintf(w, "telegram mumbles.")
+ glog.Errorf("tel.GetFile(%q): %v", fileid, err)
+ return
+ }
+
+ target := f.Link(flagTelegramToken)
+
+ req, err := http.NewRequest("GET", target, nil)
+ if err != nil {
+ w.WriteHeader(500)
+ fmt.Fprintf(w, ":(")
+ glog.Errorf("NewRequest(GET, %q, nil): %v", target, err)
+ return
+ }
+
+ req = req.WithContext(ctx)
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ w.WriteHeader(500)
+ fmt.Fprintf(w, ":(")
+ glog.Errorf("GET(%q): %v", target, err)
+ return
+ }
+ defer res.Body.Close()
+
+ if res.StatusCode != 200 {
+ // do not cache errors
+ w.WriteHeader(res.StatusCode)
+ io.Copy(w, res.Body)
+ return
+ }
+
+ b, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ w.WriteHeader(500)
+ fmt.Fprintf(w, ":(")
+ glog.Errorf("Read(%q): %v", target, err)
+ return
+ }
+
+ s.cache.Set(fileid, b, int64(len(b)))
+
+ setMime(w, fileext)
+ w.Write(b)
+}