Simplify Download to single-threaded flow
This commit is contained in:
69
util.go
69
util.go
@@ -9,7 +9,6 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
|
|
||||||
"golang.org/x/net/proxy"
|
"golang.org/x/net/proxy"
|
||||||
)
|
)
|
||||||
@@ -17,7 +16,6 @@ import (
|
|||||||
var Debug = false
|
var Debug = false
|
||||||
var Tor = false
|
var Tor = false
|
||||||
var Socks5Address = "127.0.0.1:9050"
|
var Socks5Address = "127.0.0.1:9050"
|
||||||
var DownThreads = 8
|
|
||||||
|
|
||||||
func getHttpClient() (*http.Client, error) {
|
func getHttpClient() (*http.Client, error) {
|
||||||
if Tor {
|
if Tor {
|
||||||
@@ -138,60 +136,21 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro
|
|||||||
ret := make([]byte, manifest.Size)
|
ret := make([]byte, manifest.Size)
|
||||||
totalMB := float32(manifest.Size) / 1024.0 / 1024.0
|
totalMB := float32(manifest.Size) / 1024.0 / 1024.0
|
||||||
|
|
||||||
// A channel to signal the main goroutine to check for errors
|
// Download all fragments serially to keep execution simple and deterministic.
|
||||||
doneChan := make(chan bool)
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
wg.Add(DownThreads)
|
|
||||||
var mu sync.Mutex
|
|
||||||
|
|
||||||
fragChan := make(chan Fragment, DownThreads)
|
|
||||||
errChan := make(chan error, DownThreads)
|
|
||||||
|
|
||||||
// Start worker threads
|
|
||||||
for i := 0; i < DownThreads; i++ {
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
for fragment := range fragChan {
|
|
||||||
url := transformShaToURL(fragment.Sha512, fragment.Size)
|
|
||||||
content, err := getContent(url)
|
|
||||||
if err != nil {
|
|
||||||
errChan <- err
|
|
||||||
return
|
|
||||||
}
|
|
||||||
mu.Lock()
|
|
||||||
downloaded += len(*content) * len(fragment.Offsets)
|
|
||||||
for _, offset := range fragment.Offsets {
|
|
||||||
trackFrag = append(trackFrag, Frag{Offset: offset, Size: len(*content)})
|
|
||||||
copy(ret[offset:offset+len(*content)], *content)
|
|
||||||
}
|
|
||||||
percent := float32(downloaded) / float32(manifest.Size) * 100.0
|
|
||||||
downedMB := float32(downloaded) / 1024.0 / 1024.0
|
|
||||||
fmt.Printf("\r\033[32mDownloaded\033[0m %fMB/%fMB (\033[36m%.2f%%\033[0m) of \033[33m%s\033[0m", downedMB, totalMB, percent, filen)
|
|
||||||
mu.Unlock()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
|
|
||||||
// A separate goroutine to wait for all workers to finish and then close the error channel
|
|
||||||
go func() {
|
|
||||||
wg.Wait()
|
|
||||||
close(doneChan)
|
|
||||||
}()
|
|
||||||
|
|
||||||
// Send fragments to workers
|
|
||||||
for _, fragment := range manifest.Fragments {
|
for _, fragment := range manifest.Fragments {
|
||||||
fragChan <- fragment
|
url := transformShaToURL(fragment.Sha512, fragment.Size)
|
||||||
}
|
content, err := getContent(url)
|
||||||
close(fragChan)
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
// Wait for workers to finish or an error to occur
|
}
|
||||||
select {
|
downloaded += len(*content) * len(fragment.Offsets)
|
||||||
case <-doneChan:
|
for _, offset := range fragment.Offsets {
|
||||||
// All workers finished successfully
|
trackFrag = append(trackFrag, Frag{Offset: offset, Size: len(*content)})
|
||||||
case err := <-errChan:
|
copy(ret[offset:offset+len(*content)], *content)
|
||||||
// An error occurred
|
}
|
||||||
return nil, err
|
percent := float32(downloaded) / float32(manifest.Size) * 100.0
|
||||||
|
downedMB := float32(downloaded) / 1024.0 / 1024.0
|
||||||
|
fmt.Printf("\r\033[32mDownloaded\033[0m %fMB/%fMB (\033[36m%.2f%%\033[0m) of \033[33m%s\033[0m", downedMB, totalMB, percent, filen)
|
||||||
}
|
}
|
||||||
|
|
||||||
// sort trackFrag by Offset
|
// sort trackFrag by Offset
|
||||||
|
|||||||
Reference in New Issue
Block a user