go|通过HTTP请求分片下载一个大文件

核心是计算好分片的起始位置，然后设置好Header的Range字段

p.request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", p.start, p.end))

分片合并，按序号将分片组装即可，shell的简单实现：

cat (ls |sort -n) >1.zip

借助Goroutine和channel可以实现并发下载分片

完整代码如下：

package main
import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"os"
	"strconv"
	"sync"
)
var (
	size    int64 = 5 * 1024 * 1024 // 分块大小
	workers       = 3               // 并发下载数
)
type part struct {
	partID  int
	start   int64
	end     int64
	request *http.Request
	client  *http.Client
}
func (p *part) Do() *http.Response {
	p.request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", p.start, p.end))
	resp, err := p.client.Do(p.request)
	if err != nil {
		fmt.Println(err)
	}
	log.Printf("[%d]Content-Range:%s, %v\n", p.partID, resp.Header.Get("Content-Range"), resp.Request.Header)
	return resp
}
func complatePart(partNum int, filename string) {
	// 分块合并
	file, err := os.Create(filename)
	if err != nil {
		log.Println(err)
		return
	}
	var offset int64
	defer file.Close()
	for x := 0; x < partNum; x++ {
		partFile := fmt.Sprintf("data/%d", x)
		buf, err := ioutil.ReadFile(partFile)
		if err != nil {
			log.Println(err)
			continue
		}
		file.WriteAt(buf, offset)
		offset += int64(len(buf))
		//os.Remove(partFile)
	}
	log.Println("written to ", filename)
}
func sendPart(partNum int, length int64, req *http.Request, ch chan part) {
	for i := 0; i < partNum; i++ {
		start := int64(i) * size
		var end int64
		if i == partNum-1 {
			end = length - 1
		} else {
			end = start + size - 1
		}
		log.Println(start, end)
		p := part{
			partID:  i,
			start:   int64(start),
			end:     int64(end),
			request: req,
			client:  &http.Client{},
		}
		log.Printf("send part %d to queue \n", i)
		ch <- p
	}
	close(ch)
}
func worker(ch chan part, wg *sync.WaitGroup) {
	for d := range ch {
		// 下载分片
		log.Printf("download part %d\n", d.partID)
		resp := d.Do()
		defer resp.Body.Close()
		// 保存分片
		filename := fmt.Sprintf("data/%d", d.partID)
		fd, err := os.Create(filename)
		if err != nil {
			log.Println(err)
		}
		defer fd.Close()
		_, err = io.Copy(fd, resp.Body)
		if err != nil {
			log.Println(err)
		}
	}
	wg.Done()
}
func Download(url string) {
	client := http.Client{}
	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatal(err)
	}
	response, err := client.Do(request)
	response.Body.Close()
	num := response.Header.Get("Content-Length")
	length, _ := strconv.ParseInt(num, 10, 64)
	log.Println("Conetnt-Length", length)
	partNum := int(math.Ceil(float64(length) / float64(size)))
	log.Println("partNum: ", partNum)
	var wg sync.WaitGroup
	ch := make(chan part)
	go func(req *http.Request) {
		sendPart(partNum, length, req, ch)
	}(request)
	for i := 0; i < workers; i++ {
		wg.Add(1)
		go worker(ch, &wg)
	}
	wg.Wait()
	complatePart(partNum, "./data/1.zip")
}
func main() {
	url := "https://bj-example-oss.oss-cn-beijing.aliyuncs.com/t1/oss-browser-linux-x64.zip?OSSAccessKeyId=L093&Expires=1602587072&Signature=TmKRA%3D"
	Download(url)
}

下载如下：

分片情况如下：