test multi

cmd/main.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"gitea.tbdevent.eu/ilbinek/reforger_utils"
+)
+
+const DEBUG = false
+
+func main() {
+	addonID := "61EB463A41FE5274"
+	version := "0.1.166"
+
+	assetsReply, err := reforger_utils.DoAssetsRequest(addonID, version)
+	if err != nil {
+		panic(err)
+	}
+
+	downloadDir := filepath.Join("downloads", addonID)
+	if DEBUG {
+		if err := os.MkdirAll(downloadDir, os.ModePerm); err != nil {
+			panic(err)
+		}
+	}
+
+	bytesReply, err := json.MarshalIndent(assetsReply, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+
+	if DEBUG {
+		if err := os.WriteFile(filepath.Join(downloadDir, "assets.json"), bytesReply, os.ModePerm); err != nil {
+			panic(err)
+		}
+	}
+
+	for _, asset := range assetsReply {
+		for _, file := range asset.Files {
+			manifest, err := reforger_utils.GetManifest(file.ManifestPath)
+			if err != nil {
+				panic(err)
+			}
+
+			if DEBUG {
+				// save manifest
+				var manifestBuf bytes.Buffer
+				manifestEncoder := json.NewEncoder(&manifestBuf)
+				manifestEncoder.SetIndent("", "  ")
+				if err := manifestEncoder.Encode(manifest); err != nil {
+					panic(err)
+				}
+
+				if err := os.WriteFile(filepath.Join(downloadDir, file.Name+".manifest.json"), manifestBuf.Bytes(), os.ModePerm); err != nil {
+					panic(err)
+				}
+			}
+			
+			downedFile, err := reforger_utils.Download(manifest, downloadDir, file.Name)
+			if err != nil {
+				panic(err)
+			}
+			
+			fmt.Println("Downloaded", file.Name, len(downedFile), "bytes")
+
+			if file.Name[len(file.Name)-4:] != ".pak" {
+				continue
+			}
+
+			pakInfo, err := reforger_utils.GetPakFileInfo(&downedFile)
+			if err != nil {
+				panic(err)
+			}
+
+			for _, entry := range pakInfo {
+				fmt.Printf(" - %s (offset: %d, size: %d)\n", entry.Name, entry.Offset, entry.Size)
+				if DEBUG {
+					entryData := downedFile[entry.Offset : entry.Offset+entry.Size]
+					path := filepath.Join("downloads", addonID, "extracted", entry.Name)
+					if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
+						panic(err)
+					}
+					if err := os.WriteFile(path, entryData, os.ModePerm); err != nil {
+						panic(err)
+					}
+				}
+			}
+		}
+	}
+}

util.go

@@ -122,9 +122,7 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro
 
 	// Start worker goroutines
 	for i := 0; i < DownThreads; i++ {
-		wg.Add(1)
+		wg.Go(func() {
-		go func() {
-			defer wg.Done()
 			for fragment := range fragmentsChan {
 				url := transformShaToURL(fragment.Sha512, fragment.Size)
 				content, err := getContent(url)
@@ -138,15 +136,18 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro
 				mu.Lock()
 				// Safely write to the shared slice and update progress
 				for _, offset := range fragment.Offsets {
-					if offset+len(content) > len(ret) {
-						// Handle case where fragment is too large for the buffer
-						fmt.Printf("Fragment too large for buffer at offset %d\n", offset)
-						continue
-					}
 					copy(ret[offset:offset+len(content)], content)
 				}
 				downloaded += len(content) * len(fragment.Offsets)
 
+				// Track downloaded fragments
+				for _, offset := range fragment.Offsets {
+					trackFrag = append(trackFrag, Frag{
+						Offset: offset,
+						Size:   len(content),
+					})
+				}
+
 				// Print progress
 				downedMB := float32(downloaded) / 1024.0 / 1024.0
 				percent := float32(downloaded) / float32(manifest.Size) * 100.0
@@ -163,7 +164,7 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro
 					mu.Unlock()
 				}
 			}
-		}()
+		})
 	}
 
 	// Send fragments to the channel