devtools/depotview/service/service.go

package service

import (
	"context"
	"fmt"
	"io"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	git "github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/config"
	"github.com/go-git/go-git/v5/plumbing"
	"github.com/go-git/go-git/v5/plumbing/filemode"
	"github.com/go-git/go-git/v5/plumbing/object"
	"github.com/go-git/go-git/v5/storage"
	"github.com/go-git/go-git/v5/storage/memory"

	pb "code.hackerspace.pl/hscloud/devtools/depotview/proto"
)

var (
	reHash = regexp.MustCompile(`[a-f0-9]{40,64}`)
)

type Service struct {
	remote string
	storer storage.Storer

	mu        sync.Mutex
	repo      *git.Repository
	lastFetch time.Time
}

func New(remote string) *Service {
	return &Service{
		remote: remote,
		storer: memory.NewStorage(),
	}
}

func (s *Service) ensureRepo(ctx context.Context) error {
	// Clone repository if necessary.
	// Use background context - we don't want this to get canceled.
	if s.repo == nil {
		glog.Infof("Cloning %q...", s.remote)
		repo, err := git.CloneContext(context.Background(), s.storer, nil, &git.CloneOptions{
			URL: s.remote,
		})
		if err != nil {
			glog.Errorf("Clone(%q): %v", s.remote, err)
			return status.Error(codes.Unavailable, "could not clone repository")
		}
		s.repo = repo
		glog.Infof("Clone done.")
	}

	// We could've gotten canceled by now.
	if err := ctx.Err(); err != nil {
		return err
	}

	// Fetch if necessary.
	if time.Since(s.lastFetch) > 10*time.Second {
		err := s.repo.FetchContext(ctx, &git.FetchOptions{
			RefSpecs: []config.RefSpec{
				config.RefSpec("+refs/heads/*:refs/heads/*"),
				config.RefSpec("+refs/changes/*:refs/changes/*"),
			},
			Force: true,
		})
		if err != nil && err != git.NoErrAlreadyUpToDate {
			glog.Errorf("Fetch(): %v", err)
		} else {
			s.lastFetch = time.Now()
		}

	}

	return nil
}

func (s *Service) Resolve(ctx context.Context, req *pb.ResolveRequest) (*pb.ResolveResponse, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	if req.Ref == "" {
		return nil, status.Error(codes.InvalidArgument, "ref must be set")
	}

	if err := s.ensureRepo(ctx); err != nil {
		return nil, err
	}

	h, err := s.repo.ResolveRevision(plumbing.Revision(req.Ref))
	switch {
	case err == plumbing.ErrReferenceNotFound:
		return &pb.ResolveResponse{Hash: "", LastChecked: s.lastFetch.UnixNano()}, nil
	case err != nil:
		return nil, status.Errorf(codes.Unavailable, "git resolve error: %v", err)
	default:
		return &pb.ResolveResponse{Hash: h.String(), LastChecked: s.lastFetch.UnixNano()}, nil
	}
}

func (s *Service) ResolveGerritChange(ctx context.Context, req *pb.ResolveGerritChangeRequest) (*pb.ResolveGerritChangeResponse, error) {
	if err := s.ensureRepo(ctx); err != nil {
		return nil, err
	}

	// I'm totally guessing this, from these examples:
	//    refs/changes/03/3/meta
	//    refs/changes/77/77/meta
	//    refs/changes/47/247/meta
	// etc...
	shard := fmt.Sprintf("%02d", req.Change%100)
	metaRef := fmt.Sprintf("refs/changes/%s/%d/meta", shard, req.Change)

	h, err := s.repo.ResolveRevision(plumbing.Revision(metaRef))
	switch {
	case err == plumbing.ErrReferenceNotFound:
		return &pb.ResolveGerritChangeResponse{Hash: "", LastChecked: s.lastFetch.UnixNano()}, nil
	case err != nil:
		return nil, status.Errorf(codes.Unavailable, "git metadata resolve error: %v", err)
	}

	c, err := s.repo.CommitObject(*h)
	if err != nil {
		return nil, status.Errorf(codes.Unavailable, "git error: %v", err)
	}

	var messages []string
	for {
		messages = append([]string{c.Message}, messages...)

		if len(c.ParentHashes) != 1 {
			break
		}

		c, err = s.repo.CommitObject(c.ParentHashes[0])
		if err != nil {
			return nil, status.Errorf(codes.Unavailable, "git error: %v", err)
		}
	}

	meta := parseGerritMetadata(messages)
	if meta == nil {
		return nil, status.Errorf(codes.Internal, "could not parse gerrit metadata for ref %q", metaRef)
	}
	return &pb.ResolveGerritChangeResponse{Hash: meta.commit, LastChecked: s.lastFetch.UnixNano()}, nil
}

func (s *Service) getFile(ctx context.Context, hash, path string, notFoundOkay bool) (*object.File, error) {
	if !reHash.MatchString(hash) {
		return nil, status.Error(codes.InvalidArgument, "hash must be valid full git hash string")
	}
	if path == "" {
		return nil, status.Error(codes.InvalidArgument, "path must be set")
	}

	path = pathNormalize(path)
	if path == "" {
		return nil, status.Error(codes.InvalidArgument, "path must be a valid unix or depot-style path")
	}

	if err := s.ensureRepo(ctx); err != nil {
		return nil, err
	}

	c, err := s.repo.CommitObject(plumbing.NewHash(hash))
	switch {
	case err == plumbing.ErrObjectNotFound:
		return nil, status.Error(codes.NotFound, "hash not found")
	case err != nil:
		return nil, status.Errorf(codes.Unavailable, "git error: %v", err)
	}

	file, err := c.File(path)
	switch {
	case err == object.ErrFileNotFound && !notFoundOkay:
		return nil, status.Error(codes.NotFound, "file not found")
	case err == object.ErrFileNotFound && notFoundOkay:
		return nil, nil
	case err != nil:
		return nil, status.Errorf(codes.Unavailable, "git error: %v", err)
	}

	return file, nil
}

func (s *Service) Stat(ctx context.Context, req *pb.StatRequest) (*pb.StatResponse, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	file, err := s.getFile(ctx, req.Hash, req.Path, true)
	if err != nil {
		return nil, err
	}

	if file == nil {
		return &pb.StatResponse{Type: pb.StatResponse_TYPE_NOT_PRESENT}, nil
	}

	switch {
	case file.Mode == filemode.Dir:
		return &pb.StatResponse{Type: pb.StatResponse_TYPE_DIRECTORY}, nil
	case file.Mode.IsFile():
		return &pb.StatResponse{Type: pb.StatResponse_TYPE_FILE}, nil
	default:
		return nil, status.Errorf(codes.Unimplemented, "unknown file type %o", file.Mode)
	}
}

func (s *Service) Read(req *pb.ReadRequest, srv pb.DepotView_ReadServer) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	ctx := srv.Context()

	file, err := s.getFile(ctx, req.Hash, req.Path, false)
	if err != nil {
		return err
	}

	reader, err := file.Reader()
	if err != nil {
		return status.Errorf(codes.Unavailable, "file read error: %v", err)
	}
	defer reader.Close()

	for {
		if ctx.Err() != nil {
			return ctx.Err()
		}

		// 1 MB read
		chunk := make([]byte, 16*1024)
		n, err := reader.Read(chunk)
		switch {
		case err == io.EOF:
			n = 0
		case err != nil:
			return status.Errorf(codes.Unavailable, "file read error: %v", err)
		}

		err = srv.Send(&pb.ReadResponse{Data: chunk[:n]})
		if err != nil {
			return err
		}

		if n == 0 {
			break
		}
	}

	return nil
}

func pathNormalize(path string) string {
	leadingSlashes := 0
	for _, c := range path {
		if c != '/' {
			break
		}
		leadingSlashes += 1
	}

	// Only foo/bar, /foo/bar, and //foo/bar paths allowed.
	if leadingSlashes > 2 {
		return ""
	}
	path = path[leadingSlashes:]

	// No trailing slashes allowed.
	if strings.HasSuffix(path, "/") {
		return ""
	}

	return path
}