devtools/depotview: init
This is a small service for accessing git repos read-only over gRPC.
It's going to be used to allow hackdoc to render arbitrary versions of
hscloud.
Change-Id: Ib3c5eb5a8bc679e8062142e6fa30505d9550e2fa
diff --git a/WORKSPACE b/WORKSPACE
index d4d45b1..bf25c0c 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -183,23 +183,12 @@
http_archive(
name = "io_bazel_rules_go",
urls = [
- "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.21.3/rules_go-v0.21.3.tar.gz",
- "https://github.com/bazelbuild/rules_go/releases/download/v0.21.3/rules_go-v0.21.3.tar.gz",
+ "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.22.2/rules_go-v0.22.2.tar.gz",
+ "https://github.com/bazelbuild/rules_go/releases/download/v0.22.2/rules_go-v0.22.2.tar.gz",
],
- sha256 = "af04c969321e8f428f63ceb73463d6ea817992698974abeff0161e069cd08bd6",
+ sha256 = "142dd33e38b563605f0d20e89d9ef9eda0fc3cb539a14be1bdb1350de2eda659",
)
-# Invoke go_rules_dependencies depending on host platform.
-load("//tools:go_sdk.bzl", "gen_imports")
-
-gen_imports(name = "go_sdk_imports")
-
-load("@go_sdk_imports//:imports.bzl", "load_go_sdk")
-
-load_go_sdk()
-
-# Go Gazelle rules
-
http_archive(
name = "bazel_gazelle",
urls = [
@@ -208,9 +197,20 @@
],
sha256 = "d8c45ee70ec39a57e7a05e5027c32b1576cc7f16d9dd37135b0eddde45cf1b10",
)
-
load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository")
+go_repository(
+ name = "org_golang_x_net",
+ commit = "d3edc9973b7eb1fb302b0ff2c62357091cea9a30",
+ importpath = "golang.org/x/net",
+)
+
+# Invoke go_rules_dependencies depending on host platform.
+load("//tools:go_sdk.bzl", "gen_imports")
+gen_imports(name = "go_sdk_imports")
+load("@go_sdk_imports//:imports.bzl", "load_go_sdk")
+load_go_sdk()
+
gazelle_dependencies()
# For devtools/gerrit/gerrit-oauth-provider
@@ -481,12 +481,6 @@
)
go_repository(
- name = "org_golang_x_net",
- commit = "13f9640d40b9",
- importpath = "golang.org/x/net",
-)
-
-go_repository(
name = "com_github_stackexchange_wmi",
commit = "cbe66965904dbe8a6cd589e2298e5d8b986bd7dd",
importpath = "github.com/stackexchange/wmi",
@@ -1963,3 +1957,58 @@
commit = "7bfe4c7ecddb3666a94b053b422cdd8f5aaa3615",
importpath = "github.com/shurcooL/sanitized_anchor_name",
)
+
+go_repository(
+ name = "com_github_go_git_go_billy_v5",
+ commit = "d7a8afccaed297c30f8dff5724dbe422b491dd0d",
+ importpath = "github.com/go-git/go-billy/v5",
+ remote = "https://github.com/go-git/go-billy",
+ vcs = "git",
+)
+
+go_repository(
+ name = "com_github_go_git_go_git_v5",
+ commit = "3127ad9a44a2ee935502816065dfe39f494f583d",
+ importpath = "github.com/go-git/go-git/v5",
+ remote = "https://github.com/go-git/go-git",
+ vcs = "git",
+ build_extra_args = [
+ "-known_import=github.com/go-git/go-billy/v5",
+ ],
+)
+
+go_repository(
+ name = "com_github_go_git_gcfg",
+ commit = "22f18f9a74d34e3b1a7d59cfa33043bc50ebe376",
+ importpath = "github.com/go-git/gcfg",
+)
+
+go_repository(
+ name = "in_gopkg_warnings_v0",
+ commit = "ec4a0fea49c7b46c2aeb0b51aac55779c607e52b",
+ importpath = "gopkg.in/warnings.v0",
+)
+
+go_repository(
+ name = "com_github_emirpasic_gods",
+ commit = "80e934ed68b9084f386ae25f74f839aaecfb54d8",
+ importpath = "github.com/emirpasic/gods",
+)
+
+go_repository(
+ name = "com_github_jbenet_go_context",
+ commit = "d14ea06fba99483203c19d92cfcd13ebe73135f4",
+ importpath = "github.com/jbenet/go-context",
+)
+
+go_repository(
+ name = "com_github_kevinburke_ssh_config",
+ commit = "01f96b0aa0cdcaa93f9495f89bbc6cb5a992ce6e",
+ importpath = "github.com/kevinburke/ssh_config",
+)
+
+go_repository(
+ name = "com_github_xanzy_ssh_agent",
+ commit = "6a3e2ff9e7c564f36873c2e36413f634534f1c44",
+ importpath = "github.com/xanzy/ssh-agent",
+)
diff --git a/devtools/depotview/BUILD.bazel b/devtools/depotview/BUILD.bazel
new file mode 100644
index 0000000..223f64c
--- /dev/null
+++ b/devtools/depotview/BUILD.bazel
@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["main.go"],
+ importpath = "code.hackerspace.pl/hscloud/devtools/depotview",
+ visibility = ["//visibility:private"],
+ deps = [
+ "//devtools/depotview/proto:go_default_library",
+ "//devtools/depotview/service:go_default_library",
+ "//go/mirko:go_default_library",
+ "@com_github_golang_glog//:go_default_library",
+ ],
+)
+
+go_binary(
+ name = "depotview",
+ embed = [":go_default_library"],
+ visibility = ["//visibility:public"],
+)
diff --git a/devtools/depotview/README.md b/devtools/depotview/README.md
new file mode 100644
index 0000000..9b806a6
--- /dev/null
+++ b/devtools/depotview/README.md
@@ -0,0 +1,19 @@
+depotview
+=========
+
+Git-as-a-service over gRPC. Useful to get read-only access to hscloud.
+
+Development
+-----------
+
+ $ bazel run //devtools/depotview -- -hspki_disable
+ $ grpcurl -plaintext -d '{"ref": "master"}' 127.0.0.1:4200 depotview.DepotView.Resolve
+ {
+ "hash": "154baf1cf6ed99ae5b2849f512ea4d58dbbf199e",
+ "lastChecked": 1586377071253733703
+ }
+ $ grpcurl -plaintext -d '{"hash": "154baf1cf6ed99ae5b2849f512ea4d58dbbf199e", "path": "//README"}' 127.0.0.1:4200 depotview.DepotView.Read
+ {
+ "data": "SFNDbG...."
+ }
+
diff --git a/devtools/depotview/main.go b/devtools/depotview/main.go
new file mode 100644
index 0000000..7b4aed4
--- /dev/null
+++ b/devtools/depotview/main.go
@@ -0,0 +1,34 @@
+package main
+
+import (
+ "flag"
+
+ "code.hackerspace.pl/hscloud/go/mirko"
+ "github.com/golang/glog"
+
+ pb "code.hackerspace.pl/hscloud/devtools/depotview/proto"
+ "code.hackerspace.pl/hscloud/devtools/depotview/service"
+)
+
+var (
+ flagRemote = "https://gerrit.hackerspace.pl/hscloud"
+)
+
+func main() {
+ flag.StringVar(&flagRemote, "git_remote", flagRemote, "Address of Git repository to serve")
+ flag.Parse()
+
+ m := mirko.New()
+ if err := m.Listen(); err != nil {
+ glog.Exitf("Listen(): %v", err)
+ }
+
+ s := service.New(flagRemote)
+ pb.RegisterDepotViewServer(m.GRPC(), s)
+
+ if err := m.Serve(); err != nil {
+ glog.Exitf("Serve(): %v", err)
+ }
+
+ <-m.Done()
+}
diff --git a/devtools/depotview/proto/BUILD.bazel b/devtools/depotview/proto/BUILD.bazel
new file mode 100644
index 0000000..47df920
--- /dev/null
+++ b/devtools/depotview/proto/BUILD.bazel
@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
+
+proto_library(
+ name = "proto_proto",
+ srcs = ["depotview.proto"],
+ visibility = ["//visibility:public"],
+)
+
+go_proto_library(
+ name = "proto_go_proto",
+ compilers = ["@io_bazel_rules_go//proto:go_grpc"],
+ importpath = "code.hackerspace.pl/hscloud/devtools/depotview/proto",
+ proto = ":proto_proto",
+ visibility = ["//visibility:public"],
+)
+
+go_library(
+ name = "go_default_library",
+ embed = [":proto_go_proto"],
+ importpath = "code.hackerspace.pl/hscloud/devtools/depotview/proto",
+ visibility = ["//visibility:public"],
+)
diff --git a/devtools/depotview/proto/depotview.proto b/devtools/depotview/proto/depotview.proto
new file mode 100644
index 0000000..2584961
--- /dev/null
+++ b/devtools/depotview/proto/depotview.proto
@@ -0,0 +1,46 @@
+syntax = "proto3";
+package depotview;
+option go_package = "code.hackerspace.pl/hscloud/devtools/depotview/proto";
+
+message ResolveRequest {
+ string ref = 1;
+}
+
+message ResolveResponse {
+ string hash = 1;
+ int64 last_checked = 2;
+}
+
+message StatRequest {
+ string hash = 1;
+ string path = 2;
+}
+
+message StatResponse {
+ enum Type {
+ TYPE_INVALID = 0;
+ TYPE_NOT_PRESENT = 1;
+ TYPE_FILE = 2;
+ TYPE_DIRECTORY = 3;
+ };
+ Type type = 1;
+}
+
+message ReadRequest {
+ string hash = 1;
+ string path = 2;
+}
+
+message ReadResponse {
+ // Chunk of data. Empty once everything has been sent over.
+ bytes data = 1;
+}
+
+service DepotView {
+ // Resolve a git branch/tag/ref... into a commit hash.
+ rpc Resolve(ResolveRequest) returns (ResolveResponse);
+
+ // Minimal file access API. It kinda stinks.
+ rpc Stat(StatRequest) returns (StatResponse);
+ rpc Read(ReadRequest) returns (stream ReadResponse);
+}
diff --git a/devtools/depotview/service/BUILD.bazel b/devtools/depotview/service/BUILD.bazel
new file mode 100644
index 0000000..6f7337c
--- /dev/null
+++ b/devtools/depotview/service/BUILD.bazel
@@ -0,0 +1,27 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["service.go"],
+ importpath = "code.hackerspace.pl/hscloud/devtools/depotview/service",
+ visibility = ["//visibility:public"],
+ deps = [
+ "//devtools/depotview/proto:go_default_library",
+ "@com_github_go_git_go_git_v5//:go_default_library",
+ "@com_github_go_git_go_git_v5//plumbing:go_default_library",
+ "@com_github_go_git_go_git_v5//plumbing/filemode:go_default_library",
+ "@com_github_go_git_go_git_v5//plumbing/object:go_default_library",
+ "@com_github_go_git_go_git_v5//storage:go_default_library",
+ "@com_github_go_git_go_git_v5//storage/memory:go_default_library",
+ "@com_github_golang_glog//:go_default_library",
+ "@org_golang_google_grpc//codes:go_default_library",
+ "@org_golang_google_grpc//status:go_default_library",
+ ],
+)
+
+go_test(
+ name = "go_default_test",
+ srcs = ["service_test.go"],
+ embed = [":go_default_library"],
+ deps = ["//devtools/depotview/proto:go_default_library"],
+)
diff --git a/devtools/depotview/service/service.go b/devtools/depotview/service/service.go
new file mode 100644
index 0000000..910bf36
--- /dev/null
+++ b/devtools/depotview/service/service.go
@@ -0,0 +1,221 @@
+package service
+
+import (
+ "context"
+ "io"
+ "regexp"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/golang/glog"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ git "github.com/go-git/go-git/v5"
+ "github.com/go-git/go-git/v5/plumbing"
+ "github.com/go-git/go-git/v5/plumbing/filemode"
+ "github.com/go-git/go-git/v5/plumbing/object"
+ "github.com/go-git/go-git/v5/storage"
+ "github.com/go-git/go-git/v5/storage/memory"
+
+ pb "code.hackerspace.pl/hscloud/devtools/depotview/proto"
+)
+
+var (
+ reHash = regexp.MustCompile(`[a-f0-9]{40,64}`)
+)
+
+type Service struct {
+ remote string
+ storer storage.Storer
+
+ mu sync.Mutex
+ repo *git.Repository
+ lastPull time.Time
+}
+
+func New(remote string) *Service {
+ return &Service{
+ remote: remote,
+ storer: memory.NewStorage(),
+ }
+}
+
+func (s *Service) ensureRepo() error {
+ // Clone repository if necessary.
+ if s.repo == nil {
+ repo, err := git.Clone(s.storer, nil, &git.CloneOptions{
+ URL: s.remote,
+ })
+ if err != nil {
+ glog.Errorf("Clone(%q): %v", s.remote, err)
+ return status.Error(codes.Unavailable, "could not clone repository")
+ }
+ s.repo = repo
+ s.lastPull = time.Now()
+ }
+
+ // Fetch if necessary.
+ if time.Since(s.lastPull) > time.Minute {
+ err := s.repo.Fetch(&git.FetchOptions{})
+ if err != nil && err != git.NoErrAlreadyUpToDate {
+ glog.Errorf("Fetch(): %v", err)
+ } else {
+ s.lastPull = time.Now()
+ }
+ }
+
+ return nil
+}
+
+func (s *Service) Resolve(ctx context.Context, req *pb.ResolveRequest) (*pb.ResolveResponse, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ if req.Ref == "" {
+ return nil, status.Error(codes.InvalidArgument, "ref must be set")
+ }
+
+ if err := s.ensureRepo(); err != nil {
+ return nil, err
+ }
+
+ h, err := s.repo.ResolveRevision(plumbing.Revision(req.Ref))
+ switch {
+ case err == plumbing.ErrReferenceNotFound:
+ return &pb.ResolveResponse{Hash: "", LastChecked: s.lastPull.UnixNano()}, nil
+ case err != nil:
+ return nil, status.Errorf(codes.Unavailable, "git resolve error: %v", err)
+ default:
+ return &pb.ResolveResponse{Hash: h.String(), LastChecked: s.lastPull.UnixNano()}, nil
+ }
+}
+
+func (s *Service) getFile(hash, path string, notFoundOkay bool) (*object.File, error) {
+ if !reHash.MatchString(hash) {
+ return nil, status.Error(codes.InvalidArgument, "hash must be valid full git hash string")
+ }
+ if path == "" {
+ return nil, status.Error(codes.InvalidArgument, "path must be set")
+ }
+
+ path = pathNormalize(path)
+ if path == "" {
+ return nil, status.Error(codes.InvalidArgument, "path must be a valid unix or depot-style path")
+ }
+
+ if err := s.ensureRepo(); err != nil {
+ return nil, err
+ }
+
+ c, err := s.repo.CommitObject(plumbing.NewHash(hash))
+ switch {
+ case err == plumbing.ErrObjectNotFound:
+ return nil, status.Error(codes.NotFound, "hash not found")
+ case err != nil:
+ return nil, status.Errorf(codes.Unavailable, "git error: %v", err)
+ }
+
+ file, err := c.File(path)
+ switch {
+ case err == object.ErrFileNotFound && !notFoundOkay:
+ return nil, status.Error(codes.NotFound, "file not found")
+ case err == object.ErrFileNotFound && notFoundOkay:
+ return nil, nil
+ case err != nil:
+ return nil, status.Errorf(codes.Unavailable, "git error: %v", err)
+ }
+
+ return file, nil
+}
+
+func (s *Service) Stat(ctx context.Context, req *pb.StatRequest) (*pb.StatResponse, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ file, err := s.getFile(req.Hash, req.Path, true)
+ if err != nil {
+ return nil, err
+ }
+
+ if file == nil {
+ return &pb.StatResponse{Type: pb.StatResponse_TYPE_NOT_PRESENT}, nil
+ }
+
+ switch {
+ case file.Mode == filemode.Dir:
+ return &pb.StatResponse{Type: pb.StatResponse_TYPE_DIRECTORY}, nil
+ case file.Mode.IsFile():
+ return &pb.StatResponse{Type: pb.StatResponse_TYPE_FILE}, nil
+ default:
+ return nil, status.Errorf(codes.Unimplemented, "unknown file type %o", file.Mode)
+ }
+}
+
+func (s *Service) Read(req *pb.ReadRequest, srv pb.DepotView_ReadServer) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ file, err := s.getFile(req.Hash, req.Path, false)
+ if err != nil {
+ return err
+ }
+
+ reader, err := file.Reader()
+ if err != nil {
+ return status.Errorf(codes.Unavailable, "file read error: %v", err)
+ }
+ defer reader.Close()
+
+ ctx := srv.Context()
+ for {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+
+ // 1 MB read
+ chunk := make([]byte, 16*1024)
+ n, err := reader.Read(chunk)
+ switch {
+ case err == io.EOF:
+ n = 0
+ case err != nil:
+ return status.Errorf(codes.Unavailable, "file read error: %v", err)
+ }
+
+ err = srv.Send(&pb.ReadResponse{Data: chunk[:n]})
+ if err != nil {
+ return err
+ }
+
+ if n == 0 {
+ break
+ }
+ }
+
+ return nil
+}
+
+func pathNormalize(path string) string {
+ leadingSlashes := 0
+ for _, c := range path {
+ if c != '/' {
+ break
+ }
+ leadingSlashes += 1
+ }
+
+ // Only foo/bar, /foo/bar, and //foo/bar paths allowed.
+ if leadingSlashes > 2 {
+ return ""
+ }
+ path = path[leadingSlashes:]
+
+ // No trailing slashes allowed.
+ if strings.HasSuffix(path, "/") {
+ return ""
+ }
+
+ return path
+}
diff --git a/devtools/depotview/service/service_test.go b/devtools/depotview/service/service_test.go
new file mode 100644
index 0000000..8ea0764
--- /dev/null
+++ b/devtools/depotview/service/service_test.go
@@ -0,0 +1,32 @@
+package service
+
+import (
+ "context"
+ "testing"
+
+ pb "code.hackerspace.pl/hscloud/devtools/depotview/proto"
+)
+
+func TestIntegration(t *testing.T) {
+ // TODO(q3k); bring up fake git
+ s := New("https://gerrit.hackerspace.pl/hscloud")
+ ctx := context.Background()
+
+ res, err := s.Resolve(ctx, &pb.ResolveRequest{Ref: "master"})
+ if err != nil {
+ t.Fatalf("Resolve(master): %v", err)
+ }
+
+ if len(res.Hash) != 40 {
+ t.Fatalf("Resolve returned odd hash: %q", res.Hash)
+ }
+
+ res2, err := s.Stat(ctx, &pb.StatRequest{Hash: res.Hash, Path: "//WORKSPACE"})
+ if err != nil {
+ t.Fatalf("Stat(//WORKSPACE): %v", err)
+ }
+
+ if want, got := pb.StatResponse_TYPE_FILE, res2.Type; want != got {
+ t.Fatalf("Stat(//WORKSPACE): got %v, want %v", got, want)
+ }
+}