diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..15f0c35 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,98 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "gitea.tbdevent.eu/ilbinek/reforger_utils" +) + +const DEBUG = true + +func main() { + addonID := "BADC0DEDABBEDA5E" + version := "0.12.4573" + + assetsReply, err := reforger_utils.DoAssetsRequest(addonID, version) + if err != nil { + panic(err) + } + + downloadDir := filepath.Join("downloads", addonID) + if DEBUG { + if err := os.MkdirAll(downloadDir, os.ModePerm); err != nil { + panic(err) + } + } + + bytesReply, err := json.MarshalIndent(assetsReply, "", " ") + if err != nil { + panic(err) + } + + if DEBUG { + if err := os.WriteFile(filepath.Join(downloadDir, "assets.json"), bytesReply, os.ModePerm); err != nil { + panic(err) + } + } + + for _, asset := range assetsReply { + for _, file := range asset.Files { + manifest, err := reforger_utils.GetManifest(file.ManifestPath) + if err != nil { + panic(err) + } + + if DEBUG { + // save manifest + var manifestBuf bytes.Buffer + manifestEncoder := json.NewEncoder(&manifestBuf) + manifestEncoder.SetIndent("", " ") + if err := manifestEncoder.Encode(manifest); err != nil { + panic(err) + } + + if err := os.WriteFile(filepath.Join(downloadDir, file.Name+".manifest.json"), manifestBuf.Bytes(), os.ModePerm); err != nil { + panic(err) + } + } + + downedFile, err := reforger_utils.Download(manifest, downloadDir, file.Name) + if err != nil { + panic(err) + } + + if err := os.WriteFile(filepath.Join(downloadDir, file.Name), downedFile, os.ModePerm); err != nil { + panic(err) + } + + fmt.Println("Downloaded", file.Name, len(downedFile), "bytes") + + if file.Name[len(file.Name)-4:] != ".pak" { + continue + } + + pakInfo, err := reforger_utils.GetPakFileInfo(&downedFile) + if err != nil { + panic(err) + } + + for _, entry := range pakInfo { + fmt.Printf(" - %s (offset: %d, size: %d)\n", entry.Name, entry.Offset, entry.Size) + if DEBUG { + entryData := downedFile[entry.Offset : entry.Offset+entry.Size] + path := filepath.Join("downloads", addonID, "extracted", entry.Name) + if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil { + panic(err) + } + if err := os.WriteFile(path, entryData, os.ModePerm); err != nil { + panic(err) + } + } + } + } + } +} diff --git a/structs.go b/structs.go index 9d4c505..5c0f50e 100644 --- a/structs.go +++ b/structs.go @@ -61,11 +61,13 @@ type Manifest struct { Size int `json:"size"` Offsets []int `json:"offsets"` } `json:"remainder"` - Fragments []struct { - Sha512 string `json:"sha512"` - Size int `json:"size"` - Offsets []int `json:"offsets"` - } `json:"fragments"` + Fragments []Fragment `json:"fragments"` +} + +type Fragment struct { + Sha512 string `json:"sha512"` + Size int `json:"size"` + Offsets []int `json:"offsets"` } type RDBHeader struct { diff --git a/util.go b/util.go index 4154b85..0db18ba 100644 --- a/util.go +++ b/util.go @@ -9,6 +9,7 @@ import ( "path/filepath" "slices" "strings" + "sync" "golang.org/x/net/proxy" ) @@ -135,28 +136,61 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro // start downloading chunks ret := make([]byte, manifest.Size) totalMB := float32(manifest.Size) / 1024.0 / 1024.0 - for _, fragment := range manifest.Fragments { - url := transformShaToURL(fragment.Sha512, fragment.Size) - content, err := getContent(url) - if err != nil { - return nil, err - } - downloaded += len(content) * len(fragment.Offsets) - for _, offset := range fragment.Offsets { - trackFrag = append(trackFrag, Frag{Offset: offset, Size: len(content)}) - copy(ret[offset:offset+len(content)], content) - } - if Debug { - // save fragment to file in the fragments/ directory - if err := os.WriteFile(filepath.Join(downloadDir, "fragments", filen, fmt.Sprintf("%d.%s.%d.bytes", fragment.Offsets[0], fragment.Sha512, fragment.Size)), content, os.ModePerm); err != nil { - return nil, err + // A channel to signal the main goroutine to check for errors + doneChan := make(chan bool) + + var wg sync.WaitGroup + wg.Add(DownThreads) + var mu sync.Mutex + + fragChan := make(chan Fragment, DownThreads) + errChan := make(chan error, DownThreads) + + // Start worker threads + for i := 0; i < DownThreads; i++ { + go func() { + defer wg.Done() + for fragment := range fragChan { + url := transformShaToURL(fragment.Sha512, fragment.Size) + content, err := getContent(url) + if err != nil { + errChan <- err + return + } + mu.Lock() + downloaded += len(content) * len(fragment.Offsets) + for _, offset := range fragment.Offsets { + trackFrag = append(trackFrag, Frag{Offset: offset, Size: len(content)}) + copy(ret[offset:offset+len(content)], content) + } + percent := float32(downloaded) / float32(manifest.Size) * 100.0 + downedMB := float32(downloaded) / 1024.0 / 1024.0 + fmt.Printf("\r\033[32mDownloaded\033[0m %fMB/%fMB (\033[36m%.2f%%\033[0m) of \033[33m%s\033[0m", downedMB, totalMB, percent, filen) + mu.Unlock() } - } + }() + } - percent := float32(downloaded) / float32(manifest.Size) * 100.0 - downedMB := float32(downloaded) / 1024.0 / 1024.0 - fmt.Printf("\r\033[32mDownloaded\033[0m %fMB/%fMB (\033[36m%.2f%%\033[0m) of \033[33m%s\033[0m", downedMB, totalMB, percent, filen) + // A separate goroutine to wait for all workers to finish and then close the error channel + go func() { + wg.Wait() + close(doneChan) + }() + + // Send fragments to workers + for _, fragment := range manifest.Fragments { + fragChan <- fragment + } + close(fragChan) + + // Wait for workers to finish or an error to occur + select { + case <-doneChan: + // All workers finished successfully + case err := <-errChan: + // An error occurred + return nil, err } // sort trackFrag by Offset