Implement fetching dependencies in QUICHE depstool
PiperOrigin-RevId: 541021571
diff --git a/depstool/deps/fetch.go b/depstool/deps/fetch.go
new file mode 100644
index 0000000..39fc0ee
--- /dev/null
+++ b/depstool/deps/fetch.go
@@ -0,0 +1,70 @@
+package deps
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "path"
+)
+
+func fetchIntoFile(url string, file *os.File) error {
+ resp, err := http.Get(url)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ _, err = io.Copy(file, resp.Body)
+ return err
+}
+
+func fileSHA256(file *os.File) (string, error) {
+ file.Seek(0, 0)
+ hasher := sha256.New()
+ if _, err := io.Copy(hasher, file); err != nil {
+ return "", nil
+ }
+ return hex.EncodeToString(hasher.Sum(nil)), nil
+}
+
+// FetchURL fetches the specified URL into the specified file path, and returns
+// the SHA-256 hash of the file fetched.
+func FetchURL(url string, path string) (string, error) {
+ file, err := os.Create(path)
+ if err != nil {
+ return "", err
+ }
+ defer file.Close()
+
+ if err = fetchIntoFile(url, file); err != nil {
+ os.Remove(path)
+ return "", err
+ }
+
+ checksum, err := fileSHA256(file)
+ if err != nil {
+ os.Remove(path)
+ return "", err
+ }
+
+ return checksum, nil
+}
+
+// FetchEntry retrieves an existing WORKSPACE file entry into a specified directory,
+// verifies its checksum, and then returns the full path to the resulting file.
+func FetchEntry(entry *Entry, dir string) (string, error) {
+ filename := path.Join(dir, entry.SHA256+".tar.gz")
+ checksum, err := FetchURL(entry.URL, filename)
+ if err != nil {
+ return "", err
+ }
+
+ if checksum != entry.SHA256 {
+ os.Remove(filename)
+ return "", fmt.Errorf("SHA-256 mismatch: expected %s, got %s", entry.SHA256, checksum)
+ }
+
+ return filename, nil
+}
diff --git a/depstool/deps/fetch_test.go b/depstool/deps/fetch_test.go
new file mode 100644
index 0000000..07c161e
--- /dev/null
+++ b/depstool/deps/fetch_test.go
@@ -0,0 +1,55 @@
+package deps
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "os"
+ "testing"
+)
+
+func serveTestString(w http.ResponseWriter, _ *http.Request) {
+ io.WriteString(w, "test")
+}
+
+func TestFetch(t *testing.T) {
+ http.HandleFunc("/test", serveTestString)
+
+ listener, err := net.Listen("tcp", ":0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ port := listener.Addr().(*net.TCPAddr).Port
+ url := fmt.Sprintf("http://localhost:%d/test", port)
+ go http.Serve(listener, nil)
+
+ tmpdir, err := os.MkdirTemp("", "*")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(tmpdir)
+
+ entry := Entry{
+ Name: "com_example",
+ SHA256: "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", // SHA256("test")
+ Prefix: "",
+ URL: url,
+ LastUpdated: "2022-05-18",
+ }
+
+ filename, err := FetchEntry(&entry, tmpdir)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ contents, err := os.ReadFile(filename)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !bytes.Equal(contents, []byte("test")) {
+ t.Errorf("Expected to get 'test', instead got '%s'", contents)
+ }
+}
diff --git a/depstool/depstool.go b/depstool/depstool.go
index 8ada2d9..7733841 100644
--- a/depstool/depstool.go
+++ b/depstool/depstool.go
@@ -59,6 +59,22 @@
os.Exit(0)
}
+func fetch(path string, contents []byte, outdir string) {
+ rules, err := deps.ParseHTTPArchiveRules(contents)
+ if err != nil {
+ log.Fatalf("Failed to parse %s: %v", path, err)
+ }
+
+ for _, rule := range rules {
+ log.Printf("Fetching %s into %s...", rule.Name, outdir)
+ outfile, err := deps.FetchEntry(rule, outdir)
+ if err != nil {
+ log.Fatalf("Failed to fetch %s: %s", rule.Name, err)
+ }
+ log.Printf("Successfully fetched %s into %s", rule.Name, outfile)
+ }
+}
+
func usage() {
fmt.Fprintf(flag.CommandLine.Output(), `
usage: depstool [WORKSPACE file] [subcommand]
@@ -66,6 +82,7 @@
Available subcommands:
list Lists all of the rules in the file
validate Validates that the WORKSPACE file is parsable
+ fetch Fetches all dependencies into the specified directory
If no subcommand is specified, "list" is assumed.
`)
@@ -93,6 +110,8 @@
list(path, contents)
case "validate":
validate(path, contents)
+ case "fetch":
+ fetch(path, contents, flag.Arg(2))
default:
log.Fatalf("Unknown command: %s", subcommand)
}
