devtools/depotview: init
This is a small service for accessing git repos read-only over gRPC.
It's going to be used to allow hackdoc to render arbitrary versions of
hscloud.
Change-Id: Ib3c5eb5a8bc679e8062142e6fa30505d9550e2fa
diff --git a/devtools/depotview/BUILD.bazel b/devtools/depotview/BUILD.bazel
new file mode 100644
index 0000000..223f64c
--- /dev/null
+++ b/devtools/depotview/BUILD.bazel
@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["main.go"],
+ importpath = "code.hackerspace.pl/hscloud/devtools/depotview",
+ visibility = ["//visibility:private"],
+ deps = [
+ "//devtools/depotview/proto:go_default_library",
+ "//devtools/depotview/service:go_default_library",
+ "//go/mirko:go_default_library",
+ "@com_github_golang_glog//:go_default_library",
+ ],
+)
+
+go_binary(
+ name = "depotview",
+ embed = [":go_default_library"],
+ visibility = ["//visibility:public"],
+)
diff --git a/devtools/depotview/README.md b/devtools/depotview/README.md
new file mode 100644
index 0000000..9b806a6
--- /dev/null
+++ b/devtools/depotview/README.md
@@ -0,0 +1,19 @@
+depotview
+=========
+
+Git-as-a-service over gRPC. Useful to get read-only access to hscloud.
+
+Development
+-----------
+
+ $ bazel run //devtools/depotview -- -hspki_disable
+ $ grpcurl -plaintext -d '{"ref": "master"}' 127.0.0.1:4200 depotview.DepotView.Resolve
+ {
+ "hash": "154baf1cf6ed99ae5b2849f512ea4d58dbbf199e",
+ "lastChecked": 1586377071253733703
+ }
+ $ grpcurl -plaintext -d '{"hash": "154baf1cf6ed99ae5b2849f512ea4d58dbbf199e", "path": "//README"}' 127.0.0.1:4200 depotview.DepotView.Read
+ {
+ "data": "SFNDbG...."
+ }
+
diff --git a/devtools/depotview/main.go b/devtools/depotview/main.go
new file mode 100644
index 0000000..7b4aed4
--- /dev/null
+++ b/devtools/depotview/main.go
@@ -0,0 +1,34 @@
+package main
+
+import (
+ "flag"
+
+ "code.hackerspace.pl/hscloud/go/mirko"
+ "github.com/golang/glog"
+
+ pb "code.hackerspace.pl/hscloud/devtools/depotview/proto"
+ "code.hackerspace.pl/hscloud/devtools/depotview/service"
+)
+
+var (
+ flagRemote = "https://gerrit.hackerspace.pl/hscloud"
+)
+
+func main() {
+ flag.StringVar(&flagRemote, "git_remote", flagRemote, "Address of Git repository to serve")
+ flag.Parse()
+
+ m := mirko.New()
+ if err := m.Listen(); err != nil {
+ glog.Exitf("Listen(): %v", err)
+ }
+
+ s := service.New(flagRemote)
+ pb.RegisterDepotViewServer(m.GRPC(), s)
+
+ if err := m.Serve(); err != nil {
+ glog.Exitf("Serve(): %v", err)
+ }
+
+ <-m.Done()
+}
diff --git a/devtools/depotview/proto/BUILD.bazel b/devtools/depotview/proto/BUILD.bazel
new file mode 100644
index 0000000..47df920
--- /dev/null
+++ b/devtools/depotview/proto/BUILD.bazel
@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
+
+proto_library(
+ name = "proto_proto",
+ srcs = ["depotview.proto"],
+ visibility = ["//visibility:public"],
+)
+
+go_proto_library(
+ name = "proto_go_proto",
+ compilers = ["@io_bazel_rules_go//proto:go_grpc"],
+ importpath = "code.hackerspace.pl/hscloud/devtools/depotview/proto",
+ proto = ":proto_proto",
+ visibility = ["//visibility:public"],
+)
+
+go_library(
+ name = "go_default_library",
+ embed = [":proto_go_proto"],
+ importpath = "code.hackerspace.pl/hscloud/devtools/depotview/proto",
+ visibility = ["//visibility:public"],
+)
diff --git a/devtools/depotview/proto/depotview.proto b/devtools/depotview/proto/depotview.proto
new file mode 100644
index 0000000..2584961
--- /dev/null
+++ b/devtools/depotview/proto/depotview.proto
@@ -0,0 +1,46 @@
+syntax = "proto3";
+package depotview;
+option go_package = "code.hackerspace.pl/hscloud/devtools/depotview/proto";
+
+message ResolveRequest {
+ string ref = 1;
+}
+
+message ResolveResponse {
+ string hash = 1;
+ int64 last_checked = 2;
+}
+
+message StatRequest {
+ string hash = 1;
+ string path = 2;
+}
+
+message StatResponse {
+ enum Type {
+ TYPE_INVALID = 0;
+ TYPE_NOT_PRESENT = 1;
+ TYPE_FILE = 2;
+ TYPE_DIRECTORY = 3;
+ };
+ Type type = 1;
+}
+
+message ReadRequest {
+ string hash = 1;
+ string path = 2;
+}
+
+message ReadResponse {
+ // Chunk of data. Empty once everything has been sent over.
+ bytes data = 1;
+}
+
+service DepotView {
+ // Resolve a git branch/tag/ref... into a commit hash.
+ rpc Resolve(ResolveRequest) returns (ResolveResponse);
+
+ // Minimal file access API. It kinda stinks.
+ rpc Stat(StatRequest) returns (StatResponse);
+ rpc Read(ReadRequest) returns (stream ReadResponse);
+}
diff --git a/devtools/depotview/service/BUILD.bazel b/devtools/depotview/service/BUILD.bazel
new file mode 100644
index 0000000..6f7337c
--- /dev/null
+++ b/devtools/depotview/service/BUILD.bazel
@@ -0,0 +1,27 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["service.go"],
+ importpath = "code.hackerspace.pl/hscloud/devtools/depotview/service",
+ visibility = ["//visibility:public"],
+ deps = [
+ "//devtools/depotview/proto:go_default_library",
+ "@com_github_go_git_go_git_v5//:go_default_library",
+ "@com_github_go_git_go_git_v5//plumbing:go_default_library",
+ "@com_github_go_git_go_git_v5//plumbing/filemode:go_default_library",
+ "@com_github_go_git_go_git_v5//plumbing/object:go_default_library",
+ "@com_github_go_git_go_git_v5//storage:go_default_library",
+ "@com_github_go_git_go_git_v5//storage/memory:go_default_library",
+ "@com_github_golang_glog//:go_default_library",
+ "@org_golang_google_grpc//codes:go_default_library",
+ "@org_golang_google_grpc//status:go_default_library",
+ ],
+)
+
+go_test(
+ name = "go_default_test",
+ srcs = ["service_test.go"],
+ embed = [":go_default_library"],
+ deps = ["//devtools/depotview/proto:go_default_library"],
+)
diff --git a/devtools/depotview/service/service.go b/devtools/depotview/service/service.go
new file mode 100644
index 0000000..910bf36
--- /dev/null
+++ b/devtools/depotview/service/service.go
@@ -0,0 +1,221 @@
+package service
+
+import (
+ "context"
+ "io"
+ "regexp"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/golang/glog"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ git "github.com/go-git/go-git/v5"
+ "github.com/go-git/go-git/v5/plumbing"
+ "github.com/go-git/go-git/v5/plumbing/filemode"
+ "github.com/go-git/go-git/v5/plumbing/object"
+ "github.com/go-git/go-git/v5/storage"
+ "github.com/go-git/go-git/v5/storage/memory"
+
+ pb "code.hackerspace.pl/hscloud/devtools/depotview/proto"
+)
+
+var (
+ reHash = regexp.MustCompile(`[a-f0-9]{40,64}`)
+)
+
+type Service struct {
+ remote string
+ storer storage.Storer
+
+ mu sync.Mutex
+ repo *git.Repository
+ lastPull time.Time
+}
+
+func New(remote string) *Service {
+ return &Service{
+ remote: remote,
+ storer: memory.NewStorage(),
+ }
+}
+
+func (s *Service) ensureRepo() error {
+ // Clone repository if necessary.
+ if s.repo == nil {
+ repo, err := git.Clone(s.storer, nil, &git.CloneOptions{
+ URL: s.remote,
+ })
+ if err != nil {
+ glog.Errorf("Clone(%q): %v", s.remote, err)
+ return status.Error(codes.Unavailable, "could not clone repository")
+ }
+ s.repo = repo
+ s.lastPull = time.Now()
+ }
+
+ // Fetch if necessary.
+ if time.Since(s.lastPull) > time.Minute {
+ err := s.repo.Fetch(&git.FetchOptions{})
+ if err != nil && err != git.NoErrAlreadyUpToDate {
+ glog.Errorf("Fetch(): %v", err)
+ } else {
+ s.lastPull = time.Now()
+ }
+ }
+
+ return nil
+}
+
+func (s *Service) Resolve(ctx context.Context, req *pb.ResolveRequest) (*pb.ResolveResponse, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ if req.Ref == "" {
+ return nil, status.Error(codes.InvalidArgument, "ref must be set")
+ }
+
+ if err := s.ensureRepo(); err != nil {
+ return nil, err
+ }
+
+ h, err := s.repo.ResolveRevision(plumbing.Revision(req.Ref))
+ switch {
+ case err == plumbing.ErrReferenceNotFound:
+ return &pb.ResolveResponse{Hash: "", LastChecked: s.lastPull.UnixNano()}, nil
+ case err != nil:
+ return nil, status.Errorf(codes.Unavailable, "git resolve error: %v", err)
+ default:
+ return &pb.ResolveResponse{Hash: h.String(), LastChecked: s.lastPull.UnixNano()}, nil
+ }
+}
+
+func (s *Service) getFile(hash, path string, notFoundOkay bool) (*object.File, error) {
+ if !reHash.MatchString(hash) {
+ return nil, status.Error(codes.InvalidArgument, "hash must be valid full git hash string")
+ }
+ if path == "" {
+ return nil, status.Error(codes.InvalidArgument, "path must be set")
+ }
+
+ path = pathNormalize(path)
+ if path == "" {
+ return nil, status.Error(codes.InvalidArgument, "path must be a valid unix or depot-style path")
+ }
+
+ if err := s.ensureRepo(); err != nil {
+ return nil, err
+ }
+
+ c, err := s.repo.CommitObject(plumbing.NewHash(hash))
+ switch {
+ case err == plumbing.ErrObjectNotFound:
+ return nil, status.Error(codes.NotFound, "hash not found")
+ case err != nil:
+ return nil, status.Errorf(codes.Unavailable, "git error: %v", err)
+ }
+
+ file, err := c.File(path)
+ switch {
+ case err == object.ErrFileNotFound && !notFoundOkay:
+ return nil, status.Error(codes.NotFound, "file not found")
+ case err == object.ErrFileNotFound && notFoundOkay:
+ return nil, nil
+ case err != nil:
+ return nil, status.Errorf(codes.Unavailable, "git error: %v", err)
+ }
+
+ return file, nil
+}
+
+func (s *Service) Stat(ctx context.Context, req *pb.StatRequest) (*pb.StatResponse, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ file, err := s.getFile(req.Hash, req.Path, true)
+ if err != nil {
+ return nil, err
+ }
+
+ if file == nil {
+ return &pb.StatResponse{Type: pb.StatResponse_TYPE_NOT_PRESENT}, nil
+ }
+
+ switch {
+ case file.Mode == filemode.Dir:
+ return &pb.StatResponse{Type: pb.StatResponse_TYPE_DIRECTORY}, nil
+ case file.Mode.IsFile():
+ return &pb.StatResponse{Type: pb.StatResponse_TYPE_FILE}, nil
+ default:
+ return nil, status.Errorf(codes.Unimplemented, "unknown file type %o", file.Mode)
+ }
+}
+
+func (s *Service) Read(req *pb.ReadRequest, srv pb.DepotView_ReadServer) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ file, err := s.getFile(req.Hash, req.Path, false)
+ if err != nil {
+ return err
+ }
+
+ reader, err := file.Reader()
+ if err != nil {
+ return status.Errorf(codes.Unavailable, "file read error: %v", err)
+ }
+ defer reader.Close()
+
+ ctx := srv.Context()
+ for {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+
+ // 1 MB read
+ chunk := make([]byte, 16*1024)
+ n, err := reader.Read(chunk)
+ switch {
+ case err == io.EOF:
+ n = 0
+ case err != nil:
+ return status.Errorf(codes.Unavailable, "file read error: %v", err)
+ }
+
+ err = srv.Send(&pb.ReadResponse{Data: chunk[:n]})
+ if err != nil {
+ return err
+ }
+
+ if n == 0 {
+ break
+ }
+ }
+
+ return nil
+}
+
+func pathNormalize(path string) string {
+ leadingSlashes := 0
+ for _, c := range path {
+ if c != '/' {
+ break
+ }
+ leadingSlashes += 1
+ }
+
+ // Only foo/bar, /foo/bar, and //foo/bar paths allowed.
+ if leadingSlashes > 2 {
+ return ""
+ }
+ path = path[leadingSlashes:]
+
+ // No trailing slashes allowed.
+ if strings.HasSuffix(path, "/") {
+ return ""
+ }
+
+ return path
+}
diff --git a/devtools/depotview/service/service_test.go b/devtools/depotview/service/service_test.go
new file mode 100644
index 0000000..8ea0764
--- /dev/null
+++ b/devtools/depotview/service/service_test.go
@@ -0,0 +1,32 @@
+package service
+
+import (
+ "context"
+ "testing"
+
+ pb "code.hackerspace.pl/hscloud/devtools/depotview/proto"
+)
+
+func TestIntegration(t *testing.T) {
+ // TODO(q3k); bring up fake git
+ s := New("https://gerrit.hackerspace.pl/hscloud")
+ ctx := context.Background()
+
+ res, err := s.Resolve(ctx, &pb.ResolveRequest{Ref: "master"})
+ if err != nil {
+ t.Fatalf("Resolve(master): %v", err)
+ }
+
+ if len(res.Hash) != 40 {
+ t.Fatalf("Resolve returned odd hash: %q", res.Hash)
+ }
+
+ res2, err := s.Stat(ctx, &pb.StatRequest{Hash: res.Hash, Path: "//WORKSPACE"})
+ if err != nil {
+ t.Fatalf("Stat(//WORKSPACE): %v", err)
+ }
+
+ if want, got := pb.StatResponse_TYPE_FILE, res2.Type; want != got {
+ t.Fatalf("Stat(//WORKSPACE): got %v, want %v", got, want)
+ }
+}