revisiting the json schema stuff

jetify-com · Bee-Mar · May 25, 2024 · May 25, 2024 · May 25, 2024 · May 26, 2024
commit ea649568173da65c6900476a321e26075934be59
diff --git a/internal/plugin/git.go b/internal/plugin/git.go
@@ -76,36 +76,57 @@ func (p *gitPlugin) Hash() string {
 }
 
 func (p *gitPlugin) FileContent(subpath string) ([]byte, error) {
-	contentURL, err := p.url(subpath)
+	pluginLocation, err := p.url(subpath)
 
 	if err != nil {
 		return nil, err
 	}
 
-	readFile := func() ([]byte, time.Duration, error) {
-		archive := filepath.Join("/", "tmp", p.ref.Dir+".tar.gz")
-		args := strings.Fields(contentURL)
-		cmd := exec.Command(args[0], args[1:]...) // Maybe make async?
+	retrieveArchive := func() ([]byte, time.Duration, error) {
+		archiveDir, _ := os.MkdirTemp("", p.ref.Owner)
+		archive := filepath.Join(archiveDir, p.ref.Owner+".tar.gz")
+		args := strings.Fields(pluginLocation + archive) // this is really just the base git archive command + file
 
+		// TODO get this working properly
+		//defer func() {
+		//	slog.Debug("Removing archive " + archive)
+		//	os.RemoveAll(archive)
+		//	slog.Debug("Removing archive directory " + archiveDir)
+		//	os.RemoveAll(archiveDir)
+		//}()
+
+		cmd := exec.Command(args[0], args[1:]...)
 		_, err := cmd.Output()
 
 		if err != nil {
 			slog.Error("Error executing git archive: ", err)
-			return nil, 24 * time.Hour, err
+			return nil, 0, err
 		}
 
 		reader, err := os.Open(archive)
-		io.ReadAll(reader)
-		err = fileutil.Untar(reader, "/tmp") // TODO: add UUID?
+		err = fileutil.Untar(reader, archiveDir)
+
+		if err != nil {
+			slog.Error("Encountered error while trying to extract "+archive+": ", err)
+			return nil, 0, err
+		}
+
+		pluginJson := filepath.Join(archiveDir, p.ref.Dir, "plugin.json")
+		file, err := os.Open(pluginJson)
+		defer file.Close()
 
-		file, err := os.Open(contentURL)
 		info, err := file.Stat()
 
-		if err != nil || info.Size() == 0 {
+		if err != nil {
+			slog.Error("Error extracting file " + file.Name() + ". Cannot process plugin.")
+			return nil, 0, err
+		}
+
+		if info.Size() == 0 {
+			slog.Error("Extracted file " + file.Name() + " is empty. Cannot process plugin.")
 			return nil, 0, err
 		}
 
-		defer file.Close()
 		body, err := io.ReadAll(file)
 
 		if err != nil {
@@ -115,8 +136,8 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) {
 		return body, 24 * time.Hour, nil
 	}
 
-	retrieve := func() ([]byte, time.Duration, error) {
-		req, err := p.request(contentURL)
+	retrieveHttp := func() ([]byte, time.Duration, error) {
+		req, err := p.request(pluginLocation)
 
 		if err != nil {
 			return nil, 0, err
@@ -155,13 +176,13 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) {
 
 	switch p.ref.Type {
 	case flake.TypeSSH:
-		return sshCache.GetOrSet(contentURL, readFile)
+		return sshCache.GetOrSet(pluginLocation, retrieveArchive)
 	case flake.TypeGitHub:
-		return githubCache.GetOrSet(contentURL, retrieve)
+		return githubCache.GetOrSet(pluginLocation, retrieveHttp)
 	case flake.TypeGitLab:
-		return gitlabCache.GetOrSet(contentURL, retrieve)
+		return gitlabCache.GetOrSet(pluginLocation, retrieveHttp)
 	case flake.TypeBitBucket:
-		return bitbucketCache.GetOrSet(contentURL, retrieve)
+		return bitbucketCache.GetOrSet(pluginLocation, retrieveHttp)
 	default:
 		return nil, err
 	}
@@ -170,76 +191,40 @@ func (p *gitPlugin) FileContent(subpath string) ([]byte, error) {
 func (p *gitPlugin) url(subpath string) (string, error) {
 	switch p.ref.Type {
 	case flake.TypeSSH:
-		return p.sshGitUrl()
-	case flake.TypeHttps:
-		u, err := p.repoUrl(subpath)
-		slog.Debug(u)
-		return u, err
-		//return p.repoUrl(subpath)
+		return p.sshBaseGitCommand()
+	case flake.TypeBitBucket:
+		fallthrough
+	case flake.TypeGitHub:
+		fallthrough
+	case flake.TypeGitLab:
+		return p.repoUrl(subpath)
 	default:
 		return "", errors.New("Unsupported plugin type: " + p.ref.Type)
 	}
 }
 
-func (p *gitPlugin) sshGitUrl() (string, error) {
+func (p *gitPlugin) sshBaseGitCommand() (string, error) {
 	defaultBranch := "main"
 
-	if p.ref.Host == flake.TypeGitHub {
+	if p.ref.Host == flake.TypeGitHub+".com" {
 		// using master for GitHub repos for the same reasoning established in `githubUrl`
 		defaultBranch = "master"
 	}
 
-	fileFormat := "tar.gz"
-	baseCommand := fmt.Sprintf("git archive --format=%s --remote=ssh://git@", fileFormat)
+	p.ref.Ref = defaultBranch
 
-	archive := filepath.Join("/", "tmp", p.ref.Dir+"."+fileFormat)
+	prefix := "git archive --format=tar.gz --remote=ssh://git@"
+	path, _ := url.JoinPath(p.ref.Owner, p.ref.Repo)
 	branch := cmp.Or(p.ref.Rev, p.ref.Ref, defaultBranch)
 
 	host := p.ref.Host
-
 	if p.ref.Port != "" {
 		host += ":" + p.ref.Port
 	}
 
-	// TODO: try to use the Devbox file hashing mechanism to make sure it's stored properly
-	command := fmt.Sprintf("%s%s/%s/%s %s %s -o %s", baseCommand, host, p.ref.Owner, p.ref.Repo, branch, p.ref.Dir, archive)
-
-	slog.Debug("Generated git archive command: " + command)
-
+	command := fmt.Sprintf("%s%s/%s %s %s -o", prefix, host, path, branch, p.ref.Dir)
+	slog.Debug("Generated base git archive command: " + command)
 	return command, nil
-
-	//sshCache.GetOrSet(command, func() ([]byte, time.Duration, error) {
-
-	//})
-
-	// 24 hours is currently when files are considered "expired" in other FileContent function
-	//currentTime := time.Now()
-	//threshold := 24 * time.Hour
-	//expiration := currentTime.Add(-threshold)
-
-	//args := strings.Fields(command)
-	//archiveInfo, err := os.Stat(archive)
-
-	//if os.IsNotExist(err) || archiveInfo.ModTime().Before(expiration) {
-	//	cmd := exec.Command(args[0], args[1:]...) // Maybe make async?
-
-	//	_, err := cmd.Output()
-
-	//	if err != nil {
-	//		slog.Error("Error executing git archive: ", err)
-	//		return "", err
-	//	}
-
-	//	reader, err := os.Open(archive)
-	//	io.ReadAll(reader)
-	//	err = fileutil.Untar(reader, "/tmp") // TODO: add UUID?
-
-	//	if err == nil {
-	//		return "", err
-	//	}
-	//}
-
-	//return filepath.Join("/", "tmp", p.ref.Dir, "plugin.json"), nil
 }
 
 func (p *gitPlugin) githubUrl(subpath string) (string, error) {

diff --git a/internal/plugin/includable.go b/internal/plugin/includable.go
@@ -18,18 +18,16 @@ type Includable interface {
 
 // TODO UPDATEME
 func parseIncludable(ref flake.Ref, workingDir string) (Includable, error) {
-	//ref, err := flake.ParseRef(path)
-
-	//if err != nil {
-	//	return nil, err
-	//}
-
 	switch ref.Type {
 	case flake.TypePath:
 		return newLocalPlugin(ref, workingDir)
 	case flake.TypeSSH:
 		fallthrough
-	case flake.TypeHttps:
+	case flake.TypeBitBucket:
+		fallthrough
+	case flake.TypeGitHub:
+		fallthrough
+	case flake.TypeGitLab:
 		return newGitPlugin(ref)
 	default:
 		return nil, fmt.Errorf("unsupported ref type %q", ref.Type)

diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go
@@ -2,6 +2,8 @@
 package flake
 
 import (
+	//"log/slog"
+	"log/slog"
 	"net/url"
 	"path"
 	"slices"
@@ -16,10 +18,10 @@ const (
 	TypePath      = "path"
 	TypeFile      = "file"
 	TypeSSH       = "ssh"
-	TypeGitHub    = "github"    // TODO UPDATEME delete
-	TypeGitLab    = "gitlab"    // TODO UPDATEME delete
-	TypeBitBucket = "bitbucket" // TODO UPDATEME delete
-	TypeHttps     = "https"     // TODO UPDATEME, this should take place of Github, GitLab, and Bitbucket types
+	TypeGitHub    = "github"
+	TypeGitLab    = "gitlab"
+	TypeBitBucket = "bitbucket"
+	TypeHttps     = "https"
 	TypeTarball   = "tarball"
 )
 
@@ -73,11 +75,6 @@ type Ref struct {
 
 	// Port of the server git server, to support privately hosted git servers or tunnels
 	Port string `json:port,omitempty`
-
-	// TODO UPDATEME
-	// Subgroup pertains to GitLab. GitHub and Bitbucket don't support multi-level
-	// hierarchy, and this allows the subgroup to exist without breaking the parsing logic already in place
-	Subgroup string `json:subgroup,omitempty`
 }
 
 // ParseRef parses a raw flake reference. Nix supports a variety of flake ref
@@ -239,16 +236,20 @@ func parseURLRef(ref string) (parsed Ref, fragment string, err error) {
 func parseGitRef(refURL *url.URL, parsed *Ref) error {
 	// github:<owner>/<repo>(/<rev-or-ref>)?(\?<params>)?
 
-	// Only split up to 3 times (owner, repo, ref/rev) so that we handle
-	// refs that have slashes in them. For example,
-	// "github:jetify-com/devbox/gcurtis/flakeref" parses as "gcurtis/flakeref".
-	split, err := splitPathOrOpaque(refURL, 3)
+	// gitlab supports up to 20 levels of nesting: https://docs.gitlab.com/ee/user/group/subgroups/
+	// using an object structure allows the subgroup(s) to be easily identified by just getting the splits
+
+	slog.Debug("REEEEEEEEEEEPO: ", parsed.Repo)
+	split, err := splitRepoString(parsed.Repo, 20)
 	if err != nil {
+		slog.Debug("FAILED TO DO THE SPLITS")
 		return err
 	}
 
-	parsed.Owner = split[0]
-	parsed.Repo = split[1]
+	parsed.Owner = strings.Join(split[0:len(split)-2], "/")
+	parsed.Repo = split[len(split)-1]
+
+	slog.Debug("THINGGGGGGS", parsed.Owner, parsed.Repo)
 
 	if len(split) > 2 {
 		if revOrRef := split[2]; isGitHash(revOrRef) {
@@ -258,11 +259,6 @@ func parseGitRef(refURL *url.URL, parsed *Ref) error {
 		}
 	}
 
-	parsed.Host = refURL.Query().Get("host")
-	parsed.Dir = refURL.Query().Get("dir")
-	parsed.Subgroup = refURL.Query().Get("subgroup")
-	parsed.Port = refURL.Query().Get("port")
-
 	if qRef := refURL.Query().Get("ref"); qRef != "" {
 		if parsed.Rev != "" {
 			return redact.Errorf("%s flake reference has a ref and a rev", parsed.Type)
@@ -423,6 +419,8 @@ func isArchive(path string) bool {
 // ensuring that path elements with an encoded '/' (%2F) are not split.
 // For example, "/dir/file%2Fname" becomes the elements "dir" and "file/name".
 // The count limits the number of substrings per [strings.SplitN]
+
+// TODO git rid of this
 func splitPathOrOpaque(u *url.URL, n int) ([]string, error) {
 	upath := u.EscapedPath()
 	if upath == "" {
@@ -450,6 +448,31 @@ func splitPathOrOpaque(u *url.URL, n int) ([]string, error) {
 	return split, nil
 }
 
+// TODO maybe use this?
+func splitRepoString(repo string, n int) ([]string, error) {
+	repo = strings.TrimSpace(repo)
+
+	if repo == "" {
+		return nil, nil
+	}
+
+	// We don't want an empty element if the path is rooted.
+	if repo[0] == '/' {
+		repo = repo[1:]
+	}
+	repo = path.Clean(repo)
+
+	var err error
+	split := strings.SplitN(repo, "/", n)
+	for i := range split {
+		split[i], err = url.PathUnescape(split[i])
+		if err != nil {
+			return nil, err
+		}
+	}
+	return split, nil
+}
+
 // buildEscapedPath escapes and joins path elements for a URL flake ref. The
 // resulting path is cleaned according to url.JoinPath.
 func buildEscapedPath(elem ...string) string {