go服务分片下载文件+分片上传文件
发布时间：2024年01月16日
目录
如有帮助，欢迎留下足迹哦！
先看效果
待上传的源文件（这里以一个140多M的为例）
上传后服务端下载完成的文件（result.exe）：
全代码实现
定义主要路径、分片大小等
const (
chunkSize = 10 * 1024 * 1024 // 分片大小为10MB
baseURL = "http://localhost:9001/upload/chunk"
fileToUpload = "D:\\debug_path\\src\\jdk-11.0.20_windows-x64_bin.exe" // 待上传的大文件
uploadDir = "D:\\debug_path\\dst\\" // 存储临时文件的目录
downloadDir = "D:\\debug_path\\dst" // 结果文件的目录
)启动一个服务并直接执行上传
func main() {
go func() {
http.HandleFunc("/upload/chunk", handleChunkUpload)
http.ListenAndServe("localhost:9001", nil)
}()
time.Sleep(time.Second * 2)
file, err := os.Open(fileToUpload)
if err != nil {
log.Fatal(err)
}
defer file.Close()
err = chunkedUpload(file, filepath.Base(fileToUpload))
if err != nil {
log.Fatal(err)
}
select {}
}分片上传核心逻辑
这里用go程序模拟，可由包括界面在内的常见手段发起：
func chunkedUpload(file *os.File, fileName string) error {
md5hash := md5.New()
totalSize, err := io.Copy(md5hash, file) // 148260984
if err != nil {
return err
}
completeMD5 := fmt.Sprintf("%x", md5hash.Sum(nil))
_, err = file.Seek(0, 0)
if err != nil {
fmt.Println("seek err: ", err.Error())
return err
}
reader := bufio.NewReader(file)
var transferSize int // 已传输的大小
var currentChunk int64 = 1 // 当前是第几片
buf := make([]byte, chunkSize)
for {
var n int
n, err = reader.Read(buf)
// 已读完所有
if n == 0 && err == io.EOF {
break
}
if err != nil {
return err
}
body := bytes.NewReader(buf[:n])
req, _ := http.NewRequest("POST", baseURL, body)
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("X-File-Name", fileName)
req.Header.Set("X-Chunk-Number", fmt.Sprintf("%d", currentChunk)) // 当前第几片
req.Header.Set("X-Total-Chunks", fmt.Sprintf("%d", (totalSize+chunkSize-1)/chunkSize)) // 总分片数量
req.Header.Set("X-File-MD5", completeMD5)
req.Header.Set("X-File-Size", fmt.Sprintf("%d", totalSize))
client := &http.Client{}
resp, err := client.Do(req)
if err != nil || resp.StatusCode != http.StatusOK {
log.Printf("[%d] Error uploading chunk %d: %v.\n", resp.StatusCode, currentChunk, err)
return err
}
defer resp.Body.Close()
transferSize += n
fmt.Printf("---> %d request done. currentChunk=%d, transferSize=%d, total=%d.\n", currentChunk, n, transferSize, totalSize)
currentChunk++
}
return nil
}服务端分片接收和下载
func handleChunkUpload(w http.ResponseWriter, r *http.Request) {
fileName := r.Header.Get("X-File-Name")
chunkNumberStr := r.Header.Get("X-Chunk-Number")
totalChunksStr := r.Header.Get("X-Total-Chunks")
expectedMD5 := r.Header.Get("X-File-MD5")
totalSizeStr := r.Header.Get("X-File-Size")
fmt.Printf("正在处理上传的文件：fileSize=%s, chunk=%s/%s\n", totalSizeStr, chunkNumberStr, totalChunksStr)
totalSize, err := strconv.Atoi(totalSizeStr)
if err != nil {
http.Error(w, "Invalid file size", http.StatusBadRequest)
return
}
chunkNumber, err := strconv.Atoi(chunkNumberStr)
if err != nil {
http.Error(w, "invalid chunk number", http.StatusBadRequest)
return
}
totalChunks, err := strconv.Atoi(totalChunksStr)
if err != nil {
http.Error(w, "invalid total chunks", http.StatusBadRequest)
return
}
tempFileName := filepath.Join(uploadDir, fmt.Sprintf("%s.chunk.%d", fileName, chunkNumber))
fmt.Println("当前第 ", chunkNumber, "片 临时文件名：", tempFileName)
tmpFile, err := os.Create(tempFileName)
if err != nil {
fmt.Printf("failed to create temp file: %s\n", err.Error())
http.Error(w, fmt.Sprintf("failed to create temp file: %s", err.Error()), http.StatusInternalServerError)
return
}
defer tmpFile.Close()
md5hash := md5.New()
_, err = io.Copy(io.MultiWriter(tmpFile, md5hash), r.Body)
if err != nil {
fmt.Printf("failed to copy tmpFile: %s\n", err.Error())
http.Error(w, "failed to write chunk data", http.StatusInternalServerError)
return
}
actualMD5 := fmt.Sprintf("%x", md5hash.Sum(nil))
fmt.Printf("actualMD5=%s, expectedMD5=%s. ==%v\n", actualMD5, expectedMD5, actualMD5 == expectedMD5)
w.WriteHeader(http.StatusOK)
fmt.Fprintf(w, "%s", "success")
if chunkNumber == totalChunks {
go merge(totalSize, totalChunks, fileName, expectedMD5)
}
w.WriteHeader(http.StatusOK)
}分片传输完成后，服务端完成合并和校验
func merge(fileSize, totalChunks int, fileName, expectedMD5 string) {
fmt.Println("所有已传完，准备合并。 totalChunks = ", totalChunks)
outputPath := path.Join(downloadDir, "result.exe")
outputFile, err := os.Create(outputPath)
if err != nil {
fmt.Println("create failed: ", err)
return
}
defer outputFile.Close()
var filePaths []string
// 生成每个文件的路径
for i := 1; i <= totalChunks; i++ {
tempName := fmt.Sprintf("%s.chunk.%d", fileName, i)
filePath := filepath.Join(uploadDir, tempName)
filePaths = append(filePaths, filePath)
}
for i, filePath := range filePaths {
fmt.Printf("第 %d 个临时文件路径为：%s.\n", i+1, filePath)
inputFile, openErr := os.Open(filePath)
if openErr != nil {
fmt.Println("打开失败：", openErr)
return
}
defer inputFile.Close()
_, copyErr := io.Copy(outputFile, inputFile)
if copyErr != nil {
fmt.Println("copy 失败：", copyErr)
return
}
fmt.Printf("第 %d 个文件已合并.\n", i+1)
info, statErr := outputFile.Stat()
if statErr != nil {
fmt.Println("获取文件信息失败：", statErr)
return
}
fmt.Printf("当前outputFile size=%d, fileSize=%d.\n", info.Size(), fileSize)
}
outputFile.Seek(0, 0)
actualMD5, err := GetMd5ByFile(outputFile)
if err != nil {
fmt.Println("获取文件md5失败：", err)
return
}
if actualMD5 != expectedMD5 {
fmt.Println("校验失败：", actualMD5, expectedMD5)
return
}
info, err := outputFile.Stat()
if err != nil {
fmt.Println("获取文件信息失败：", err)
return
}
fmt.Printf("合并完成。文件md5=%s. resultSize=%d, fileSize=%d.\n", actualMD5, info.Size(), fileSize)
}工具函数
func GetMd5ByFile(file *os.File) (string, error) {
md5Hash := md5.New()
if _, err := io.Copy(md5Hash, file); err != nil {
return "", err
}
hashInBytes := md5Hash.Sum(nil)
file.Seek(0, 0)
return fmt.Sprintf("%x", hashInBytes), nil
}注意
1、在完成合并后，临时文件可删除，尤其是在并发时注意删除临时文件的时机，不能删除的过早，可自行添加删除逻辑。
2、基于代码中双方的数据，服务端可添加缓存记
完整输出
正在处理上传的文件：fileSize=148260984, chunk=1/15
当前第 1 片 临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.1
actualMD5=2bcc1c1a609a20c15fe8b0d64874e3b4, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:45 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 1 request done. currentChunk=10485760, transferSize=10485760, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=2/15
当前第 2 片 临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.2
actualMD5=1c2fbb73080cb86db44f2814078d2609, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:45 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 2 request done. currentChunk=10485760, transferSize=20971520, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=3/15
当前第 3 片 临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.3
actualMD5=f13fe6748c5048e8d21635186e8ddd54, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:46 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 3 request done. currentChunk=10485760, transferSize=31457280, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=4/15
当前第 4 片 临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.4
actualMD5=507bac5d8884abe68098360414ae3c47, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:46 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 4 request done. currentChunk=10485760, transferSize=41943040, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=5/15
当前第 5 片 临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.5
actualMD5=92e4cf595991d77330c69a7d91052783, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:46 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 5 request done. currentChunk=10485760, transferSize=52428800, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=6/15
当前第 6 片 临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.6
actualMD5=74b84f9ba86a6b4648ce9f06026f9819, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:47 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 6 request done. currentChunk=10485760, transferSize=62914560, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=7/15
当前第 7 片 临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.7
actualMD5=ca6d4fae23ee61b54a5c18cdc1137702, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:47 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 7 request done. currentChunk=10485760, transferSize=73400320, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=8/15
当前第 8 片 临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.8
actualMD5=e24234da9d55c4e577cdfa9ab932170b, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:47 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 8 request done. currentChunk=10485760, transferSize=83886080, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=9/15
当前第 9 片 临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.9
actualMD5=c7697c3cbd8a145bb921704bdc83692b, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:48 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 9 request done. currentChunk=10485760, transferSize=94371840, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=10/15
当前第 10 片 临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.10
actualMD5=bc647d44a4b094e00c27d67c521f63d1, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:48 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 10 request done. currentChunk=10485760, transferSize=104857600, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=11/15
当前第 11 片 临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.11
actualMD5=57edc6d0e665279325d25d4238085475, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:48 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 11 request done. currentChunk=10485760, transferSize=115343360, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=12/15
当前第 12 片 临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.12
actualMD5=74c7b12cdf056b8812af19bdf96528d6, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:49 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 12 request done. currentChunk=10485760, transferSize=125829120, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=13/15
当前第 13 片 临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.13
actualMD5=9c6e375efa255425828d16235210b956, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:49 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 13 request done. currentChunk=10485760, transferSize=136314880, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=14/15
当前第 14 片 临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.14
actualMD5=d118498b6056355e84ea46b35369d988, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
2023/12/09 14:45:49 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 14 request done. currentChunk=10485760, transferSize=146800640, total=148260984.
正在处理上传的文件：fileSize=148260984, chunk=15/15
当前第 15 片 临时文件名： D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.15
actualMD5=4b0c19522a6b242ddfed4b4d6c5dda54, expectedMD5=ade9fb7e919b2642191de90325b66cc6. ==false
所有已传完，准备合并。 totalChunks = 15
2023/12/09 14:45:50 http: superfluous response.WriteHeader call from main.handleChunkUpload (server.go:78)
---> 15 request done. currentChunk=1460344, transferSize=148260984, total=148260984.
第 1 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.1.
第 1 个文件已合并.
当前outputFile size=10485760, fileSize=148260984.
第 2 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.2.
第 2 个文件已合并.
当前outputFile size=20971520, fileSize=148260984.
第 8 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.8.
第 8 个文件已合并.
当前outputFile size=83886080, fileSize=148260984.
第 9 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.9.
第 9 个文件已合并.
当前outputFile size=94371840, fileSize=148260984.
第 10 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.10.
第 10 个文件已合并.
当前outputFile size=104857600, fileSize=148260984.
第 11 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.11.
第 11 个文件已合并.
当前outputFile size=115343360, fileSize=148260984.
第 12 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.12.
第 12 个文件已合并.
当前outputFile size=125829120, fileSize=148260984.
第 13 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.13.
第 13 个文件已合并.
当前outputFile size=136314880, fileSize=148260984.
第 14 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.14.
第 14 个文件已合并.
当前outputFile size=146800640, fileSize=148260984.
第 15 个临时文件路径为：D:\debug_path\dst\jdk-11.0.20_windows-x64_bin.exe.chunk.15.
第 15 个文件已合并.
当前outputFile size=148260984, fileSize=148260984.
合并完成。文件md5=ade9fb7e919b2642191de90325b66cc6. resultSize=148260984, fileSize=148260984.录每次接收到后的文件位置，便于后续处理；同时上传方也每次都能知道上传的进度。
完整输出
文章来源:https://blog.csdn.net/HYZX_9987/article/details/135552472
本文来自互联网用户投稿，该文观点仅代表作者本人，不代表本站立场。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。 如若内容造成侵权/违法违规/事实不符，请联系我的编程经验分享网邮箱：chenni525@qq.com进行投诉反馈，一经查实，立即删除！
本文来自互联网用户投稿，该文观点仅代表作者本人，不代表本站立场。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。 如若内容造成侵权/违法违规/事实不符，请联系我的编程经验分享网邮箱：chenni525@qq.com进行投诉反馈，一经查实，立即删除！