1 Commits

Author SHA1 Message Date
ilbinek
f05dd6ee26 Simplify Download to single-threaded flow 2026-04-10 17:58:07 +02:00

69
util.go
View File

@@ -9,7 +9,6 @@ import (
"path/filepath" "path/filepath"
"slices" "slices"
"strings" "strings"
"sync"
"golang.org/x/net/proxy" "golang.org/x/net/proxy"
) )
@@ -17,7 +16,6 @@ import (
var Debug = false var Debug = false
var Tor = false var Tor = false
var Socks5Address = "127.0.0.1:9050" var Socks5Address = "127.0.0.1:9050"
var DownThreads = 8
func getHttpClient() (*http.Client, error) { func getHttpClient() (*http.Client, error) {
if Tor { if Tor {
@@ -138,60 +136,21 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro
ret := make([]byte, manifest.Size) ret := make([]byte, manifest.Size)
totalMB := float32(manifest.Size) / 1024.0 / 1024.0 totalMB := float32(manifest.Size) / 1024.0 / 1024.0
// A channel to signal the main goroutine to check for errors // Download all fragments serially to keep execution simple and deterministic.
doneChan := make(chan bool)
var wg sync.WaitGroup
wg.Add(DownThreads)
var mu sync.Mutex
fragChan := make(chan Fragment, DownThreads)
errChan := make(chan error, DownThreads)
// Start worker threads
for i := 0; i < DownThreads; i++ {
go func() {
defer wg.Done()
for fragment := range fragChan {
url := transformShaToURL(fragment.Sha512, fragment.Size)
content, err := getContent(url)
if err != nil {
errChan <- err
return
}
mu.Lock()
downloaded += len(*content) * len(fragment.Offsets)
for _, offset := range fragment.Offsets {
trackFrag = append(trackFrag, Frag{Offset: offset, Size: len(*content)})
copy(ret[offset:offset+len(*content)], *content)
}
percent := float32(downloaded) / float32(manifest.Size) * 100.0
downedMB := float32(downloaded) / 1024.0 / 1024.0
fmt.Printf("\r\033[32mDownloaded\033[0m %fMB/%fMB (\033[36m%.2f%%\033[0m) of \033[33m%s\033[0m", downedMB, totalMB, percent, filen)
mu.Unlock()
}
}()
}
// A separate goroutine to wait for all workers to finish and then close the error channel
go func() {
wg.Wait()
close(doneChan)
}()
// Send fragments to workers
for _, fragment := range manifest.Fragments { for _, fragment := range manifest.Fragments {
fragChan <- fragment url := transformShaToURL(fragment.Sha512, fragment.Size)
} content, err := getContent(url)
close(fragChan) if err != nil {
return nil, err
// Wait for workers to finish or an error to occur }
select { downloaded += len(*content) * len(fragment.Offsets)
case <-doneChan: for _, offset := range fragment.Offsets {
// All workers finished successfully trackFrag = append(trackFrag, Frag{Offset: offset, Size: len(*content)})
case err := <-errChan: copy(ret[offset:offset+len(*content)], *content)
// An error occurred }
return nil, err percent := float32(downloaded) / float32(manifest.Size) * 100.0
downedMB := float32(downloaded) / 1024.0 / 1024.0
fmt.Printf("\r\033[32mDownloaded\033[0m %fMB/%fMB (\033[36m%.2f%%\033[0m) of \033[33m%s\033[0m", downedMB, totalMB, percent, filen)
} }
// sort trackFrag by Offset // sort trackFrag by Offset