odl/odl.go

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
)

func main() {
	flag.Parse()

	var err error
	f := os.Stdin
	filename := flag.Arg(0)

	if filename != "-" && filename != "" {
		f, err = os.Open(filename)
		if err != nil {
			log.Fatalf("failed to open file %s: %v", filename, err)
		}
	}

	Download(Read(".", f))
}

type request struct {
	w   io.Writer
	url string
}

func Download(c <-chan request) {
	dl := func(w io.Writer, url string) error {
		response, err := http.Get(url)
		if err != nil {
			return fmt.Errorf("downloading file %s: %v", url, err)
		}
		defer response.Body.Close()

		if response.StatusCode != http.StatusOK {
			return fmt.Errorf("unexpected status code for %s: %d", url, response.StatusCode)
		}

		_, err = io.Copy(w, response.Body)
		return err
	}

	for r := range c {
		if err := dl(r.w, r.url); err != nil {
			log.Println(err)
		}
	}
}

func Read(root string, r io.Reader) <-chan request {
	scanner := bufio.NewScanner(r)

	var n int
	var name string
	c := make(chan request)

	create := func(name, uri string, n int) (io.Writer, error) {
		var f io.Writer

		u, err := url.Parse(uri)
		if err != nil {
			return f, fmt.Errorf("parsing url %s: %v", uri, err)
		}

		filename := filepath.Join(root, name+strconv.Itoa(n)+filepath.Ext(u.Path))
		return os.Create(filename)
	}

	go func() {

		for scanner.Scan() {
			txt := scanner.Text()

			if name == "" || txt == "" {
				name = txt
				n = 0
				continue
			}

			f, err := create(name, txt, n)
			if err != nil {
				log.Println(err)
				continue
			}
			c <- request{w: f, url: txt}
			n++
		}
		close(c)
	}()
	return c
}