rewrite; attempt to infer extension if missing

odl.go

@@ -2,9 +2,11 @@ package main
 
 import (
 	"bufio"
+	"bytes"
 	"fmt"
 	"io"
 	"log"
+	"mime"
 	"net/http"
 	"net/url"
 	"os"
@@ -13,76 +15,83 @@ import (
 )
 
 func main() {
-	Download(Read(".", os.Stdin))
+	ReadAndDownload(".", os.Stdin)
 }
 
-type request struct {
+func ReadAndDownload(root string, r io.Reader) {
-	w   io.Writer
+	var n int
-	url string
+	var name string
-}
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		txt := scanner.Text()
 
-func Download(c <-chan request) {
+		if name == "" || txt == "" {
-	dl := func(w io.Writer, url string) error {
+			name = txt
-		response, err := http.Get(url)
+			n = 0
+			continue
+		}
+		err := download(n, root, name, txt)
 		if err != nil {
-			return fmt.Errorf("downloading file %s: %v", url, err)
+			log.Println(err)
 		}
+		n++
+	}
+}
+
+func download(n int, root, name, uri string) error {
+	u, err := url.Parse(uri)
+	if err != nil {
+		return fmt.Errorf("parsing uri %s: %v", uri, err)
+	}
+	ext := filepath.Ext(u.Path)
+
+	response, err := http.Get(uri)
+	switch {
+	case err != nil:
+		return fmt.Errorf("downloading file %s: %v", uri, err)
+	case response.StatusCode != http.StatusOK:
+		return fmt.Errorf("downloading file %s: status code %d", uri, response.StatusCode)
+	default:
 		defer response.Body.Close()
+	}
 
-		if response.StatusCode != http.StatusOK {
+	var r io.Reader = response.Body
-			return fmt.Errorf("unexpected status code for %s: %d", url, response.StatusCode)
+	if ext == "" {
+		var err error
+		ext, r, err = guessExt(response.Body)
+		if err != nil {
+			return err
 		}
+	}
 
-		_, err = io.Copy(w, response.Body)
+	filename := filepath.Join(root, name+strconv.Itoa(n)+ext)
+	f, err := os.Create(filename)
+	if err != nil {
 		return err
 	}
 
-	for r := range c {
+	_, err = io.Copy(f, r)
-		if err := dl(r.w, r.url); err != nil {
+	return err
-			log.Println(err)
-		}
-	}
 }
 
-func Read(root string, r io.Reader) <-chan request {
+func guessExt(r io.Reader) (string, io.Reader, error) {
-	scanner := bufio.NewScanner(r)
+	head, headCopy := make([]byte, 512), make([]byte, 512)
-
+	n, readErr := io.ReadFull(r, head)
-	var n int
+	head = head[:n]
-	var name string
+	copy(headCopy, head)
-	c := make(chan request)
+	hr := bytes.NewReader(headCopy)
-
-	create := func(name, uri string, n int) (io.Writer, error) {
-		var f io.Writer
 
-		u, err := url.Parse(uri)
+	t := http.DetectContentType(head)
-		if err != nil {
+	ext, err := mime.ExtensionsByType(t)
-			return f, fmt.Errorf("parsing url %s: %v", uri, err)
-		}
 
-		filename := filepath.Join(root, name+strconv.Itoa(n)+filepath.Ext(u.Path))
+	if readErr == io.EOF || readErr == io.ErrUnexpectedEOF {
-		return os.Create(filename)
+		return ext[0], hr, nil
 	}
 
-	go func() {
+	if err != nil {
-
+		return "", nil, err
-		for scanner.Scan() {
+	}
-			txt := scanner.Text()
-
-			if name == "" || txt == "" {
-				name = txt
-				n = 0
-				continue
-			}
 
-			f, err := create(name, txt, n)
+	return ext[0], io.MultiReader(hr, r), nil
-			if err != nil {
-				log.Println(err)
-				continue
-			}
-			c <- request{w: f, url: txt}
-			n++
-		}
-		close(c)
-	}()
-	return c
 }
+

odl_test.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"bytes"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -14,44 +13,28 @@ var fs = http.FileServer(http.Dir("./testdata"))
 
 var srv = httptest.NewServer(fs)
 
-func TestDownload(t *testing.T) {
+func TestReadAndDownload(t *testing.T) {
-	var (
+	lines := []string{
-		b bytes.Buffer
+		"text",
-		c = make(chan request)
+		srv.URL + "/sometext.txt",
-	)
+		srv.URL + "/moretext.txt",
-
+		"",
-	url := srv.URL + "/somefile"
+		"image",
-	t.Logf("file URL: %s", url)
+		srv.URL + "/tiny-fuji",
-
-	go func() { c <- request{&b, url}; close(c) }()
-	Download(c)
-
-	want := "somefile inner data"
-	if got := b.String(); got != want {
-		t.Errorf("file buffer received unexpected value\nGot:\n%s\nWant:\n%s", got, want)
 	}
-}
+	input := strings.Join(lines, "\n")
-
-func TestRead(t *testing.T) {
-	input := `boston
-http://localhost/someboston.jpg
-http://localhost/anotherboston.mp4
 
-orlando
-http://localhost/someorlando.jpg
-`
 	dir := t.TempDir()
 	t.Logf("root directory: %s", dir)
+	t.Logf("\ninput:\n%s\n", input)
 
-	c := Read(dir, strings.NewReader(input))
+	ReadAndDownload(dir, strings.NewReader(input))
-	for range c {
-	} // drain the channel
 
 	err := fstest.TestFS(
 		os.DirFS(dir),
-		"boston0.jpg",
+		"text0.txt",
-		"boston1.mp4",
+		"text1.txt",
-		"orlando0.jpg",
+		"image0.webp",
 	)
 
 	if err != nil {

testdata/moretext.txt

@@ -0,0 +1,2 @@
+another file of text
+