multithread

cmd/main.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"gitea.tbdevent.eu/ilbinek/reforger_utils"
+)
+
+const DEBUG = true
+
+func main() {
+	addonID := "BADC0DEDABBEDA5E"
+	version := "0.12.4573"
+
+	assetsReply, err := reforger_utils.DoAssetsRequest(addonID, version)
+	if err != nil {
+		panic(err)
+	}
+
+	downloadDir := filepath.Join("downloads", addonID)
+	if DEBUG {
+		if err := os.MkdirAll(downloadDir, os.ModePerm); err != nil {
+			panic(err)
+		}
+	}
+
+	bytesReply, err := json.MarshalIndent(assetsReply, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+
+	if DEBUG {
+		if err := os.WriteFile(filepath.Join(downloadDir, "assets.json"), bytesReply, os.ModePerm); err != nil {
+			panic(err)
+		}
+	}
+
+	for _, asset := range assetsReply {
+		for _, file := range asset.Files {
+			manifest, err := reforger_utils.GetManifest(file.ManifestPath)
+			if err != nil {
+				panic(err)
+			}
+
+			if DEBUG {
+				// save manifest
+				var manifestBuf bytes.Buffer
+				manifestEncoder := json.NewEncoder(&manifestBuf)
+				manifestEncoder.SetIndent("", "  ")
+				if err := manifestEncoder.Encode(manifest); err != nil {
+					panic(err)
+				}
+
+				if err := os.WriteFile(filepath.Join(downloadDir, file.Name+".manifest.json"), manifestBuf.Bytes(), os.ModePerm); err != nil {
+					panic(err)
+				}
+			}
+			
+			downedFile, err := reforger_utils.Download(manifest, downloadDir, file.Name)
+			if err != nil {
+				panic(err)
+			}
+
+			if err := os.WriteFile(filepath.Join(downloadDir, file.Name), downedFile, os.ModePerm); err != nil {
+				panic(err)
+			}
+			
+			fmt.Println("Downloaded", file.Name, len(downedFile), "bytes")
+
+			if file.Name[len(file.Name)-4:] != ".pak" {
+				continue
+			}
+
+			pakInfo, err := reforger_utils.GetPakFileInfo(&downedFile)
+			if err != nil {
+				panic(err)
+			}
+
+			for _, entry := range pakInfo {
+				fmt.Printf(" - %s (offset: %d, size: %d)\n", entry.Name, entry.Offset, entry.Size)
+				if DEBUG {
+					entryData := downedFile[entry.Offset : entry.Offset+entry.Size]
+					path := filepath.Join("downloads", addonID, "extracted", entry.Name)
+					if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
+						panic(err)
+					}
+					if err := os.WriteFile(path, entryData, os.ModePerm); err != nil {
+						panic(err)
+					}
+				}
+			}
+		}
+	}
+}

structs.go

@@ -61,11 +61,13 @@ type Manifest struct {
 		Size    int    `json:"size"`
 		Offsets []int  `json:"offsets"`
 	} `json:"remainder"`
-	Fragments []struct {
+	Fragments []Fragment `json:"fragments"`
-		Sha512  string `json:"sha512"`
+}
-		Size    int    `json:"size"`
+
-		Offsets []int  `json:"offsets"`
+type Fragment struct {
-	} `json:"fragments"`
+	Sha512  string `json:"sha512"`
+	Size    int    `json:"size"`
+	Offsets []int  `json:"offsets"`
 }
 
 type RDBHeader struct {

util.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
+	"sync"
 
 	"golang.org/x/net/proxy"
 )
@@ -135,28 +136,61 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro
 	// start downloading chunks
 	ret := make([]byte, manifest.Size)
 	totalMB := float32(manifest.Size) / 1024.0 / 1024.0
-	for _, fragment := range manifest.Fragments {
-		url := transformShaToURL(fragment.Sha512, fragment.Size)
-		content, err := getContent(url)
-		if err != nil {
-			return nil, err
-		}
-		downloaded += len(content) * len(fragment.Offsets)
-		for _, offset := range fragment.Offsets {
-			trackFrag = append(trackFrag, Frag{Offset: offset, Size: len(content)})
-			copy(ret[offset:offset+len(content)], content)
-		}
 
-		if Debug {
+	// A channel to signal the main goroutine to check for errors
-			// save fragment to file in the fragments/ directory
+	doneChan := make(chan bool)
-			if err := os.WriteFile(filepath.Join(downloadDir, "fragments", filen, fmt.Sprintf("%d.%s.%d.bytes", fragment.Offsets[0], fragment.Sha512, fragment.Size)), content, os.ModePerm); err != nil {
+
-				return nil, err
+	var wg sync.WaitGroup
+	wg.Add(DownThreads)
+	var mu sync.Mutex
+
+	fragChan := make(chan Fragment, DownThreads)
+	errChan := make(chan error, DownThreads)
+
+	// Start worker threads
+	for i := 0; i < DownThreads; i++ {
+		go func() {
+			defer wg.Done()
+			for fragment := range fragChan {
+				url := transformShaToURL(fragment.Sha512, fragment.Size)
+				content, err := getContent(url)
+				if err != nil {
+					errChan <- err
+					return
+				}
+				mu.Lock()
+				downloaded += len(content) * len(fragment.Offsets)
+				for _, offset := range fragment.Offsets {
+					trackFrag = append(trackFrag, Frag{Offset: offset, Size: len(content)})
+					copy(ret[offset:offset+len(content)], content)
+				}
+				percent := float32(downloaded) / float32(manifest.Size) * 100.0
+				downedMB := float32(downloaded) / 1024.0 / 1024.0
+				fmt.Printf("\r\033[32mDownloaded\033[0m %fMB/%fMB (\033[36m%.2f%%\033[0m) of \033[33m%s\033[0m", downedMB, totalMB, percent, filen)
+				mu.Unlock()
 			}
-		}
+		}()
+	}
 
-		percent := float32(downloaded) / float32(manifest.Size) * 100.0
+	// A separate goroutine to wait for all workers to finish and then close the error channel
-		downedMB := float32(downloaded) / 1024.0 / 1024.0
+	go func() {
-		fmt.Printf("\r\033[32mDownloaded\033[0m %fMB/%fMB (\033[36m%.2f%%\033[0m) of \033[33m%s\033[0m", downedMB, totalMB, percent, filen)
+		wg.Wait()
+		close(doneChan)
+	}()
+
+	// Send fragments to workers
+	for _, fragment := range manifest.Fragments {
+		fragChan <- fragment
+	}
+	close(fragChan)
+
+	// Wait for workers to finish or an error to occur
+	select {
+	case <-doneChan:
+		// All workers finished successfully
+	case err := <-errChan:
+		// An error occurred
+		return nil, err
 	}
 
 	// sort trackFrag by Offset