egtyl/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix merging artifact chunks error when minio storage basepath is set (#28555) (#28568)

Browse source 
Backport #28555 by @fuxiaohei

Related to  https://github.com/go-gitea/gitea/issues/28279

When merging artifact chunks, it lists chunks from storage. When storage
is minio, chunk's path contains `MINIO_BASE_PATH` that makes merging
break.

<del>So trim the `MINIO_BASE_PATH` when handle chunks.</del>

Update the chunk file's basename to retain necessary information. It
ensures that the directory in the chunk's path remains unaffected.

Co-authored-by: FuXiaoHei <fuxiaohei@vip.qq.com>
This commit is contained in:
Giteabot 2023-12-21 15:38:39 +08:00 committed by GitHub
parent 47f9b3f484
commit 8ca32dc873
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 9 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
routers/api/actions/artifacts_chunks.go
View file

@ -25,10 +25,11 @@ func saveUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext,
contentRange := ctx.Req.Header.Get("Content-Range") contentRange := ctx.Req.Header.Get("Content-Range")
start, end, length := int64(0), int64(0), int64(0) start, end, length := int64(0), int64(0), int64(0)
if _, err := fmt.Sscanf(contentRange, "bytes %d-%d/%d", &start, &end, &length); err != nil { if _, err := fmt.Sscanf(contentRange, "bytes %d-%d/%d", &start, &end, &length); err != nil {
log.Warn("parse content range error: %v, content-range: %s", err, contentRange)
return -1, fmt.Errorf("parse content range error: %v", err) return -1, fmt.Errorf("parse content range error: %v", err)
} }
// build chunk store path // build chunk store path
storagePath := fmt.Sprintf("tmp%d/%d-%d-%d.chunk", runID, artifact.ID, start, end) storagePath := fmt.Sprintf("tmp%d/%d-%d-%d-%d.chunk", runID, runID, artifact.ID, start, end)
// use io.TeeReader to avoid reading all body to md5 sum. // use io.TeeReader to avoid reading all body to md5 sum.
// it writes data to hasher after reading end // it writes data to hasher after reading end
// if hash is not matched, delete the read-end result // if hash is not matched, delete the read-end result
@ -57,6 +58,7 @@ func saveUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext,
} }
type chunkFileItem struct { type chunkFileItem struct {
RunID int64
ArtifactID int64 ArtifactID int64
Start int64 Start int64
End int64 End int64
@ -66,9 +68,12 @@ type chunkFileItem struct {
func listChunksByRunID(st storage.ObjectStorage, runID int64) (map[int64][]*chunkFileItem, error) { func listChunksByRunID(st storage.ObjectStorage, runID int64) (map[int64][]*chunkFileItem, error) {
storageDir := fmt.Sprintf("tmp%d", runID) storageDir := fmt.Sprintf("tmp%d", runID)
var chunks []*chunkFileItem var chunks []*chunkFileItem
if err := st.IterateObjects(storageDir, func(path string, obj storage.Object) error { if err := st.IterateObjects(storageDir, func(fpath string, obj storage.Object) error {
item := chunkFileItem{Path: path} baseName := filepath.Base(fpath)
if _, err := fmt.Sscanf(path, filepath.Join(storageDir, "%d-%d-%d.chunk"), &item.ArtifactID, &item.Start, &item.End); err != nil { // when read chunks from storage, it only contains storage dir and basename,
// no matter the subdirectory setting in storage config
item := chunkFileItem{Path: storageDir + "/" + baseName}
if _, err := fmt.Sscanf(baseName, "%d-%d-%d-%d.chunk", &item.RunID, &item.ArtifactID, &item.Start, &item.End); err != nil {
return fmt.Errorf("parse content range error: %v", err) return fmt.Errorf("parse content range error: %v", err)
} }
chunks = append(chunks, &item) chunks = append(chunks, &item)