Go并发读写文件、分片写、分片下载文件（实用版）

?倾心分享，如有帮助，欢迎留下足迹~

目录

简单读取

读取&分片写

读取文件流+分片写-1

读取文件流+分片写-2

读取文件流+并发分片写

更好用的Copy方法

http并发、分片下载

---

简单读取

func ReadFile(filePath string) (chunks []byte, err error) {
	f, err := os.Open(filePath)
	if err != nil {
		return
	}
    
    defer f.Close()
	reader := bufio.NewReader(f)

	for {
		dataByte := make([]byte, 5*1024)
		var n int
		n, err = reader.Read(dataByte)
		if err != nil || 0 == n {
			break
		}

		chunks = append(chunks, dataByte[:n]...)
		fmt.Printf("file: %s, len(chunks):%v", filePath, len(chunks))
	}

	isEOF := strings.Compare(err.Error(), "EOF")
	if isEOF == 0 {
		err = nil
		fmt.Printf("read %s success: \n, len=%v", filePath, len(chunks))
		return
	}

	fmt.Printf("readFile over")
	return
}

可以看到如文件较大，chunks会变得很大，此法只适用特定条件下的一般做法。

读取&分片写

读取文件流+分片写-1

var bufLen = 2 * 1024 * 1024

func DownLoadFileShardByFilePath1(writerFilePath string, body io.Reader) (err error) {

	f, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModeAppend)
	defer f.Close()
	if err != nil {
		fmt.Println("open err:" + err.Error())
		return
	}

	writer := bufio.NewWriter(f)
	bs := make([]byte, bufLen)
	for {
		var read int
		read, err = body.Read(bs)
		if err != nil || 0 == read {
			break
		}

		_, err = writer.Write(bs[:read])
		if err != nil {
			fmt.Println("write err:" + err.Error())
			break
		}
	}

	if err == io.EOF {
		err = nil
	}

	if err != nil {
		return
	}

	if err = writer.Flush(); err != nil {
		fmt.Println("writer flush err: ", err.Error())
		return
	}

	fmt.Printf("downLoad over")
	return
}

读取文件流+分片写-2

var bufLen = 2 * 1024 * 1024

func DownLoadFileShard(writerFilePath string, body io.Reader) (err error) {

	f, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModeAppend)
	if err != nil {
		fmt.Println("open err:" + err.Error())
		return
	}

    defer f.Close()
	bs := make([]byte, bufLen)
	writer := bufio.NewWriter(f)
	for {
		var read int
		switch read, err = body.Read(bs[:]); true {
		case read < 0:
			fmt.Println("read err: ", err.Error())
			return
		case read == 0, err == io.EOF:
			fmt.Printf("downLoad over")
			return writer.Flush()
		case read > 0:
			_, err = writer.Write(bs[:read])
			if err != nil {
				fmt.Println("write err:" + err.Error())
				return
			}
		}
	}

	return
}

读取文件流+并发分片写

type FileShard struct {
	Data []byte
	Err  error
	Code int // 0-正常  -1=失败
}

var bufLen = 2 * 1024 * 1024

func DownLoadFileShardCon(writerFilePath string, body io.Reader) (err error) {

	writerFile, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModeAppend)
	if err != nil {
		fmt.Println("open err:" + err.Error())
		return
	}

    defer writerFile.Close()
	ch, complete := make(chan *FileShard), make(chan struct{})
	go func() {
		writer := bufio.NewWriter(writerFile)

	youKnow:
		for {
			select {
			case data := <-ch:
				if data == nil {
					err = writer.Flush()
					break youKnow
				}

				if data.Code != 0 {
					err = data.Err
					break youKnow
				}

				if _, err = writer.Write(data.Data); err != nil {
					fmt.Println("write err:", err.Error())
				}
			}
		}

		close(complete)
	}()

	go func() {
		bs := make([]byte, bufLen)
		for {
			switch read, readErr := body.Read(bs[:]); true {
			case read < 0:
				ch <- &FileShard{Code: -1, Err: readErr}
				close(ch)
				return
			case read == 0, err == io.EOF:
				close(ch)
				return
			case read > 0:
				ch <- &FileShard{Data: bs[:read], Code: 0}
			}
		}

	}()

	select {
	case <-complete:
		break
	}

	fmt.Printf("downLoad over")
	return
}

并发思路有很多种，看你代码怎么写哦，条条大路通罗马！

更好用的Copy方法

要提醒的是，还有一个很不错的方法在io包里，就是io.Copy()，可防止大文件处理时的内存溢出，也可以替换上述主要流程，比如：

func IOCopyExample(writerFilePath string, body io.Reader) (err error) {
	f, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		return
	}
	
	defer f.Close()
	writer := bufio.NewWriter(f)
	_, err = io.Copy(writer, body)
	_ = writer.Flush()
	return
}

http并发、分片下载

基于http的Range来完成：

func DownloadFileRange(url, writeFile string) error {
	f, err := os.OpenFile(writeFile , os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		return err
	}

	defer f.Close()

	resp, err := http.Head(url)
	if err != nil {
		return err
	}

	size, err := strconv.Atoi(resp.Header.Get("Content-Length"))
	if err != nil {
		return err
	}

	con := getSize(size)  // getSize函数用来计算每次的并发数，可按自己方式自行指定
	var start, end int64
	for i := 0; i < con; i++ {
		
		start = int64(i) * int64(size/con)
		end = start + int64(size/con) - 1

		go func(n int, offset, end int64) {
			req := &http.Request{}
			req, err = http.NewRequest(http.MethodGet, url, nil)
			req.Header.Set("Range", fmt.Sprintf("bytes=%v-%v", offset, end))

			client := &http.Client{}
			resp, err = client.Do(req)
			if err != nil {
				return
			}

			defer resp.Body.Close()

			f.Seek(offset, 0)
			_, err = io.Copy(f, resp.Body)
			if err != nil {
				// log
			}

		}(i, start, end)
	}

	return nil
}