go服务分片下载文件+分片上传文件

目录

先看效果

全代码实现

注意

完整输出

---

如有帮助，欢迎留下足迹哦！

先看效果

待上传的源文件（这里以一个140多M的为例）

上传后服务端下载完成的文件（result.exe）：

全代码实现

定义主要路径、分片大小等

const (
	chunkSize = 10 * 1024 * 1024 // 分片大小为10MB
	baseURL   = "http://localhost:9001/upload/chunk"

	fileToUpload = "D:\\debug_path\\src\\jdk-11.0.20_windows-x64_bin.exe" // 待上传的大文件
	uploadDir    = "D:\\debug_path\\dst\\"                                // 存储临时文件的目录
	downloadDir  = "D:\\debug_path\\dst"                                  // 结果文件的目录
)

启动一个服务并直接执行上传

func main() {
	go func() {
		http.HandleFunc("/upload/chunk", handleChunkUpload)
		http.ListenAndServe("localhost:9001", nil)
	}()

	time.Sleep(time.Second * 2)
	file, err := os.Open(fileToUpload)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	err = chunkedUpload(file, filepath.Base(fileToUpload))
	if err != nil {
		log.Fatal(err)
	}

	select {}
}

分片上传核心逻辑

这里用go程序模拟，可由包括界面在内的常见手段发起：

func chunkedUpload(file *os.File, fileName string) error {
	md5hash := md5.New()
	
	totalSize, err := io.Copy(md5hash, file) // 148260984
	if err != nil {
		return err
	}

	completeMD5 := fmt.Sprintf("%x", md5hash.Sum(nil)) 

	_, err = file.Seek(0, 0)
	if err != nil {
		fmt.Println("seek err: ", err.Error())
		return err
	}

	reader := bufio.NewReader(file)

	var transferSize int       // 已传输的大小
	var currentChunk int64 = 1 // 当前是第几片
	buf := make([]byte, chunkSize)
	for {
		var n int
		n, err = reader.Read(buf)
		// 已读完所有
		if n == 0 && err == io.EOF {
			break
		}

		if err != nil {
			return err
		}

		body := bytes.NewReader(buf[:n])
		req, _ := http.NewRequest("POST", baseURL, body)
		req.Header.Set("Content-Type", "application/octet-stream")
		req.Header.Set("X-File-Name", fileName)
		req.Header.Set("X-Chunk-Number", fmt.Sprintf("%d", currentChunk))                      // 当前第几片
		req.Header.Set("X-Total-Chunks", fmt.Sprintf("%d", (totalSize+chunkSize-1)/chunkSize)) // 总分片数量
		req.Header.Set("X-File-MD5", completeMD5)
		req.Header.Set("X-File-Size", fmt.Sprintf("%d", totalSize))

		client := &http.Client{}
		resp, err := client.Do(req)
		if err != nil || resp.StatusCode != http.StatusOK {
			log.Printf("[%d] Error uploading chunk %d: %v.\n", resp.StatusCode, currentChunk, err)
			return err
		}

		defer resp.Body.Close()

		transferSize += n

		fmt.Printf("---> %d request done. currentChunk=%d, transferSize=%d, total=%d.\n", currentChunk, n, transferSize, totalSize)
		currentChunk++
	}

	return nil
}

服务端分片接收和下载

func handleChunkUpload(w http.ResponseWriter, r *http.Request) {
	fileName := r.Header.Get("X-File-Name")
	chunkNumberStr := r.Header.Get("X-Chunk-Number")
	totalChunksStr := r.Header.Get("X-Total-Chunks")
	expectedMD5 := r.Header.Get("X-File-MD5")
	totalSizeStr := r.Header.Get("X-File-Size")

	fmt.Printf("正在处理上传的文件：fileSize=%s, chunk=%s/%s\n", totalSizeStr, chunkNumberStr, totalChunksStr)

	totalSize, err := strconv.Atoi(totalSizeStr)
	if err != nil {
		http.Error(w, "Invalid file size", http.StatusBadRequest)
		return
	}

	chunkNumber, err := strconv.Atoi(chunkNumberStr)
	if err != nil {
		http.Error(w, "invalid chunk number", http.StatusBadRequest)
		return
	}

	totalChunks, err := strconv.Atoi(totalChunksStr)
	if err != nil {
		http.Error(w, "invalid total chunks", http.StatusBadRequest)
		return
	}

	tempFileName := filepath.Join(uploadDir, fmt.Sprintf("%s.chunk.%d", fileName, chunkNumber))
	fmt.Println("当前第 ", chunkNumber, "片  临时文件名：", tempFileName)

	tmpFile, err := os.Create(tempFileName)
	if err != nil {
		fmt.Printf("failed to create temp file: %s\n", err.Error())
		http.Error(w, fmt.Sprintf("failed to create temp file: %s", err.Error()), http.StatusInternalServerError)
		return
	}
	defer tmpFile.Close()

	md5hash := md5.New()

	_, err = io.Copy(io.MultiWriter(tmpFile, md5hash), r.Body)
	if err != nil {
		fmt.Printf("failed to copy tmpFile: %s\n", err.Error())
		http.Error(w, "failed to write chunk data", http.StatusInternalServerError)
		return
	}

	actualMD5 := fmt.Sprintf("%x", md5hash.Sum(nil))

	fmt.Printf("actualMD5=%s, expectedMD5=%s. ==%v\n", actualMD5, expectedMD5, actualMD5 == expectedMD5)

	w.WriteHeader(http.StatusOK)
	fmt.Fprintf(w, "%s", "success")

	if chunkNumber == totalChunks {
		go merge(totalSize, totalChunks, fileName, expectedMD5)
	}

	w.WriteHeader(http.StatusOK)
}

分片传输完成后，服务端完成合并和校验

func merge(fileSize, totalChunks int, fileName, expectedMD5 string) {
	fmt.Println("所有已传完，准备合并。 totalChunks = ", totalChunks)

	outputPath := path.Join(downloadDir, "result.exe")
	outputFile, err := os.Create(outputPath)
	if err != nil {
		fmt.Println("create failed: ", err)
		return
	}
	defer outputFile.Close()

	var filePaths []string

	//  生成每个文件的路径
	for i := 1; i <= totalChunks; i++ {
		tempName := fmt.Sprintf("%s.chunk.%d", fileName, i)
		filePath := filepath.Join(uploadDir, tempName)
		filePaths = append(filePaths, filePath)
	}

	for i, filePath := range filePaths {
		fmt.Printf("第 %d 个临时文件路径为：%s.\n", i+1, filePath)
		inputFile, openErr := os.Open(filePath)
		if openErr != nil {
			fmt.Println("打开失败：", openErr)
			return
		}
		defer inputFile.Close()

		_, copyErr := io.Copy(outputFile, inputFile)
		if copyErr != nil {
			fmt.Println("copy 失败：", copyErr)
			return
		}

		fmt.Printf("第 %d 个文件已合并.\n", i+1)

		info, statErr := outputFile.Stat()
		if statErr != nil {
			fmt.Println("获取文件信息失败：", statErr)
			return
		}

		fmt.Printf("当前outputFile size=%d, fileSize=%d.\n", info.Size(), fileSize)
	}

	outputFile.Seek(0, 0)
	actualMD5, err := GetMd5ByFile(outputFile)
	if err != nil {
		fmt.Println("获取文件md5失败：", err)
		return
	}

	if actualMD5 != expectedMD5 {
		fmt.Println("校验失败：", actualMD5, expectedMD5)
		return
	}

	info, err := outputFile.Stat()
	if err != nil {
		fmt.Println("获取文件信息失败：", err)
		return
	}

	fmt.Printf("合并完成。文件md5=%s. resultSize=%d, fileSize=%d.\n", actualMD5, info.Size(), fileSize)
}

工具函数

func GetMd5ByFile(file *os.File) (string, error) {

	md5Hash := md5.New()
	if _, err := io.Copy(md5Hash, file); err != nil {
		return "", err
	}

	hashInBytes := md5Hash.Sum(nil)
	file.Seek(0, 0)
	return fmt.Sprintf("%x", hashInBytes), nil
}

注意

1、在完成合并后，临时文件可删除，尤其是在并发时注意删除临时文件的时机，不能删除的过早，可自行添加删除逻辑。

2、基于代码中双方的数据，服务端可添加缓存记

完整输出

正在处理上传的文件：fileSize=148260984, chunk=1/15
当前第  1 片  临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.1
actualMD5=2bcc1c1a609a20c15fe8b0d64874e3b4, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:45 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 1 request done. currentChunk=10485760, transferSize=10485760, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=2/15
当前第  2 片  临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.2
actualMD5=1c2fbb73080cb86db44f2814078d2609, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:45 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 2 request done. currentChunk=10485760, transferSize=20971520, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=3/15
当前第  3 片  临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.3
actualMD5=f13fe6748c5048e8d21635186e8ddd54, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:46 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 3 request done. currentChunk=10485760, transferSize=31457280, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=4/15
当前第  4 片  临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.4
actualMD5=507bac5d8884abe68098360414ae3c47, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:46 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 4 request done. currentChunk=10485760, transferSize=41943040, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=5/15
当前第  5 片  临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.5
actualMD5=92e4cf595991d77330c69a7d91052783, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:46 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 5 request done. currentChunk=10485760, transferSize=52428800, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=6/15
当前第  6 片  临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.6
actualMD5=74b84f9ba86a6b4648ce9f06026f9819, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:47 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 6 request done. currentChunk=10485760, transferSize=62914560, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=7/15
当前第  7 片  临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.7
actualMD5=ca6d4fae23ee61b54a5c18cdc1137702, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:47 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 7 request done. currentChunk=10485760, transferSize=73400320, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=8/15
当前第  8 片  临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.8
actualMD5=e24234da9d55c4e577cdfa9ab932170b, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:47 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 8 request done. currentChunk=10485760, transferSize=83886080, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=9/15
当前第  9 片  临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.9
actualMD5=c7697c3cbd8a145bb921704bdc83692b, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:48 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 9 request done. currentChunk=10485760, transferSize=94371840, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=10/15
当前第  10 片  临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.10
actualMD5=bc647d44a4b094e00c27d67c521f63d1, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:48 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 10 request done. currentChunk=10485760, transferSize=104857600, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=11/15
当前第  11 片  临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.11
actualMD5=57edc6d0e665279325d25d4238085475, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:48 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 11 request done. currentChunk=10485760, transferSize=115343360, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=12/15
当前第  12 片  临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.12
actualMD5=74c7b12cdf056b8812af19bdf96528d6, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:49 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 12 request done. currentChunk=10485760, transferSize=125829120, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=13/15
当前第  13 片  临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.13
actualMD5=9c6e375efa255425828d16235210b956, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:49 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 13 request done. currentChunk=10485760, transferSize=136314880, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=14/15
当前第  14 片  临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.14
actualMD5=d118498b6056355e84ea46b35369d988, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:49 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 14 request done. currentChunk=10485760, transferSize=146800640, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=15/15
当前第  15 片  临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.15
actualMD5=4b0c19522a6b242ddfed4b4d6c5dda54, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
所有已传完，准备合并。 totalChunks =  15
2023/12/09 14:45:50 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 15 request done. currentChunk=1460344, transferSize=148260984, total=148260984.
第 1 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.1.
第 1 个文件已合并.
当前outputFile size=10485760, fileSize=148260984.
第 2 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.2.
第 2 个文件已合并.
当前outputFile size=20971520, fileSize=148260984.
第 8 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.8.
第 8 个文件已合并.
当前outputFile size=83886080, fileSize=148260984.
第 9 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.9.
第 9 个文件已合并.
当前outputFile size=94371840, fileSize=148260984.
第 10 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.10.
第 10 个文件已合并.
当前outputFile size=104857600, fileSize=148260984.
第 11 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.11.
第 11 个文件已合并.
当前outputFile size=115343360, fileSize=148260984.
第 12 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.12.
第 12 个文件已合并.
当前outputFile size=125829120, fileSize=148260984.
第 13 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.13.
第 13 个文件已合并.
当前outputFile size=136314880, fileSize=148260984.
第 14 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.14.
第 14 个文件已合并.
当前outputFile size=146800640, fileSize=148260984.
第 15 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.15.
第 15 个文件已合并.
当前outputFile size=148260984, fileSize=148260984.
合并完成。文件md5=ade9fb7e919b2642191de90325b66cc6. resultSize=148260984, fileSize=148260984.

录每次接收到后的文件位置，便于后续处理；同时上传方也每次都能知道上传的进度。

完整输出