go/mirko: add SQL migrations machinery
This uses github.com/golang-migrate/migrate and adds a Source that
allows using go_embed data files.
We also provide a test/example.
Change-Id: Icd2b6c7f7d0f728073b3fdf39b432b33ce61a3cd
diff --git a/go/mirko/BUILD.bazel b/go/mirko/BUILD.bazel
index 89b40ed..405987e 100644
--- a/go/mirko/BUILD.bazel
+++ b/go/mirko/BUILD.bazel
@@ -5,13 +5,16 @@
srcs = [
"kubernetes.go",
"mirko.go",
+ "sql_migrations.go",
],
importpath = "code.hackerspace.pl/hscloud/go/mirko",
visibility = ["//visibility:public"],
deps = [
"//go/pki:go_default_library",
"//go/statusz:go_default_library",
+ "@com_github_gchaincl_sqlhooks//:go_default_library",
"@com_github_golang_glog//:go_default_library",
+ "@com_github_golang_migrate_migrate_v4//source:go_default_library",
"@io_k8s_client_go//kubernetes:go_default_library",
"@io_k8s_client_go//rest:go_default_library",
"@org_golang_google_grpc//:go_default_library",
diff --git a/go/mirko/sql_migrations.go b/go/mirko/sql_migrations.go
new file mode 100644
index 0000000..61c26f2
--- /dev/null
+++ b/go/mirko/sql_migrations.go
@@ -0,0 +1,172 @@
+package mirko
+
+// Migration support via github.com/golang-migrations/migrate for go_embed data in Bazel.
+// For example usage, see go/mirko/tests/sql.
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "strconv"
+ "strings"
+
+ "github.com/golang-migrate/migrate/v4/source"
+)
+
+func NewMigrationsFromBazel(data map[string][]byte) (source.Driver, error) {
+ migrations := make(map[uint]*migration)
+
+ for k, v := range data {
+ parts := strings.Split(k, ".")
+ errInvalid := fmt.Errorf("invalid migration filename: %q", k)
+
+ if len(parts) != 3 {
+ return nil, errInvalid
+ }
+ if parts[2] != "sql" {
+ return nil, errInvalid
+ }
+ if parts[1] != "up" && parts[1] != "down" {
+ return nil, errInvalid
+ }
+ direction := parts[1]
+
+ nameParts := strings.SplitN(parts[0], "_", 2)
+ if len(nameParts) != 2 {
+ return nil, errInvalid
+ }
+
+ name := nameParts[1]
+
+ version32, err := strconv.ParseUint(nameParts[0], 10, 32)
+ if err != nil {
+ return nil, errInvalid
+ }
+ version := uint(version32)
+
+ m, ok := migrations[version]
+ if !ok {
+ migrations[version] = &migration{
+ version: version,
+ name: name,
+ }
+ m = migrations[version]
+ } else {
+ if m.name != name {
+ if err != nil {
+ return nil, fmt.Errorf("migration version %d exists under diffrent names (%q vs %q)", version, name, m.name)
+ }
+ }
+ }
+
+ if direction == "up" {
+ m.up = v
+ } else {
+ m.down = v
+ }
+ }
+
+ var first uint
+ for version, migration := range migrations {
+ if migration.up == nil {
+ return nil, fmt.Errorf("migration version %d has no up file", version)
+ }
+ if migration.down == nil {
+ return nil, fmt.Errorf("migration version %d has no down file", version)
+ }
+ if first == 0 {
+ first = version
+ }
+ if version < first {
+ first = version
+ }
+ }
+
+ if first == 0 {
+ return nil, fmt.Errorf("no migrations, or lowest migration version is 0")
+ }
+
+ return &migrationSource{
+ migrations: migrations,
+ first: first,
+ }, nil
+}
+
+type migrationSource struct {
+ migrations map[uint]*migration
+ first uint
+}
+
+type migration struct {
+ version uint
+ name string
+ up []byte
+ down []byte
+}
+
+func (s *migrationSource) Open(url string) (source.Driver, error) {
+ if url != "" {
+ return nil, fmt.Errorf("bazel migration source is not configure via an URL")
+ }
+ return s, nil
+}
+
+func (s *migrationSource) Close() error {
+ return nil
+}
+
+func (s *migrationSource) First() (uint, error) {
+ return s.first, nil
+}
+
+func (s *migrationSource) Prev(version uint) (uint, error) {
+ var prev uint
+ for ver, _ := range s.migrations {
+ if ver > prev && ver < version {
+ prev = ver
+ }
+ }
+ if prev == 0 {
+ return 0, os.ErrNotExist
+ }
+ return prev, nil
+}
+
+func (s *migrationSource) Next(version uint) (uint, error) {
+ var next uint
+ for ver, _ := range s.migrations {
+ if ver <= version {
+ continue
+ }
+ if next == 0 {
+ next = ver
+ }
+ if ver < next {
+ next = ver
+ }
+ }
+ if next <= version {
+ return 0, os.ErrNotExist
+ }
+ return next, nil
+}
+
+func (s *migrationSource) ReadUp(version uint) (io.ReadCloser, string, error) {
+ m, ok := s.migrations[version]
+ if !ok {
+ return nil, "", os.ErrNotExist
+ }
+
+ return ioutil.NopCloser(bytes.NewReader(m.up)), m.name, nil
+}
+
+func (s *migrationSource) ReadDown(version uint) (io.ReadCloser, string, error) {
+ m, ok := s.migrations[version]
+ if !ok {
+ return nil, "", os.ErrNotExist
+ }
+
+ return ioutil.NopCloser(bytes.NewReader(m.down)), m.name, nil
+}
diff --git a/go/mirko/tests/sql/BUILD.bazel b/go/mirko/tests/sql/BUILD.bazel
new file mode 100644
index 0000000..5831dcd
--- /dev/null
+++ b/go/mirko/tests/sql/BUILD.bazel
@@ -0,0 +1,10 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+
+go_test(
+ name = "go_default_test",
+ srcs = ["sql_test.go"],
+ deps = [
+ "//go/mirko/tests/sql/migrations:go_default_library",
+ "@com_github_golang_migrate_migrate_v4//database/sqlite3:go_default_library",
+ ],
+)
diff --git a/go/mirko/tests/sql/migrations/1564669958_one.down.sql b/go/mirko/tests/sql/migrations/1564669958_one.down.sql
new file mode 100644
index 0000000..cc1f647
--- /dev/null
+++ b/go/mirko/tests/sql/migrations/1564669958_one.down.sql
@@ -0,0 +1 @@
+DROP TABLE users;
diff --git a/go/mirko/tests/sql/migrations/1564669958_one.up.sql b/go/mirko/tests/sql/migrations/1564669958_one.up.sql
new file mode 100644
index 0000000..ddefda2
--- /dev/null
+++ b/go/mirko/tests/sql/migrations/1564669958_one.up.sql
@@ -0,0 +1,4 @@
+CREATE TABLE users (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ username STRING
+);
diff --git a/go/mirko/tests/sql/migrations/1564669976_two.down.sql b/go/mirko/tests/sql/migrations/1564669976_two.down.sql
new file mode 100644
index 0000000..bfb3c33
--- /dev/null
+++ b/go/mirko/tests/sql/migrations/1564669976_two.down.sql
@@ -0,0 +1 @@
+DELETE FROM users WHERE username = 'q3k';
diff --git a/go/mirko/tests/sql/migrations/1564669976_two.up.sql b/go/mirko/tests/sql/migrations/1564669976_two.up.sql
new file mode 100644
index 0000000..b24d2fa
--- /dev/null
+++ b/go/mirko/tests/sql/migrations/1564669976_two.up.sql
@@ -0,0 +1 @@
+INSERT INTO users (username) VALUES ("q3k");
diff --git a/go/mirko/tests/sql/migrations/1564669988_three.down.sql b/go/mirko/tests/sql/migrations/1564669988_three.down.sql
new file mode 100644
index 0000000..345612e
--- /dev/null
+++ b/go/mirko/tests/sql/migrations/1564669988_three.down.sql
@@ -0,0 +1,7 @@
+ALTER TABLE users RENAME TO users_old;
+CREATE TABLE users (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ username STRING
+);
+INSERT INTO users (username) SELECT (username) FROM users_old;
+DROP TABLE users_old;
diff --git a/go/mirko/tests/sql/migrations/1564669988_three.up.sql b/go/mirko/tests/sql/migrations/1564669988_three.up.sql
new file mode 100644
index 0000000..8cef579
--- /dev/null
+++ b/go/mirko/tests/sql/migrations/1564669988_three.up.sql
@@ -0,0 +1 @@
+ALTER TABLE users ADD cool BOOLEAN NULL;
diff --git a/go/mirko/tests/sql/migrations/BUILD.bazel b/go/mirko/tests/sql/migrations/BUILD.bazel
new file mode 100644
index 0000000..585ea48
--- /dev/null
+++ b/go/mirko/tests/sql/migrations/BUILD.bazel
@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//extras:embed_data.bzl", "go_embed_data")
+
+go_embed_data(
+ name = "migrations_data",
+ srcs = glob(["*.sql"]),
+ package = "migrations",
+ flatten = True,
+)
+
+go_library(
+ name = "go_default_library",
+ srcs = [
+ "migrations.go",
+ ":migrations_data", # keep
+ ],
+ importpath = "code.hackerspace.pl/hscloud/go/mirko/tests/sql/migrations",
+ visibility = ["//go/mirko/tests/sql:__subpackages__"],
+ deps = [
+ "//go/mirko:go_default_library",
+ "@com_github_golang_migrate_migrate_v4//:go_default_library",
+ ],
+)
diff --git a/go/mirko/tests/sql/migrations/migrations.go b/go/mirko/tests/sql/migrations/migrations.go
new file mode 100644
index 0000000..1782c2e
--- /dev/null
+++ b/go/mirko/tests/sql/migrations/migrations.go
@@ -0,0 +1,17 @@
+package migrations
+
+import (
+ "fmt"
+
+ "code.hackerspace.pl/hscloud/go/mirko"
+
+ "github.com/golang-migrate/migrate/v4"
+)
+
+func New(dburl string) (*migrate.Migrate, error) {
+ source, err := mirko.NewMigrationsFromBazel(Data)
+ if err != nil {
+ return nil, fmt.Errorf("could not create migrations: %v", err)
+ }
+ return migrate.NewWithSourceInstance("bazel", source, dburl)
+}
diff --git a/go/mirko/tests/sql/sql_test.go b/go/mirko/tests/sql/sql_test.go
new file mode 100644
index 0000000..c76c3c1
--- /dev/null
+++ b/go/mirko/tests/sql/sql_test.go
@@ -0,0 +1,38 @@
+package sql
+
+import (
+ "testing"
+
+ _ "github.com/golang-migrate/migrate/v4/database/sqlite3"
+
+ "code.hackerspace.pl/hscloud/go/mirko/tests/sql/migrations"
+)
+
+// TestOkay goes up and down fully through migrations.
+func TestOkay(t *testing.T) {
+ m, err := migrations.New("sqlite3://:memory:")
+ if err != nil {
+ t.Fatalf("migrations.New: %v", err)
+ }
+
+ err = m.Up()
+ if err != nil {
+ t.Fatalf("m.Up() failed: %v", err)
+ }
+
+ vers, dirty, err := m.Version()
+ if err != nil {
+ t.Fatalf("m.Version() failed: %v", err)
+ }
+ if dirty {
+ t.Errorf("database migration shouldn't be dirty")
+ }
+ if want, got := uint(1564669988), vers; want != got {
+ t.Errorf("got database version %d, want %d", want, got)
+ }
+
+ err = m.Down()
+ if err != nil {
+ t.Fatalf("m.Down() failed: %v", err)
+ }
+}