test multi

README.md

@@ -1,76 +0,0 @@
-# reforger_utils
-
-Utilities for downloading and processing Arma Reforger workshop content.
-
-## Install
-
-This module is hosted on a private Gitea instance.
-
-Set private module settings once:
-
-```powershell
-go env -w GOPRIVATE=gitea.tbdevent.eu
-```
-
-Then install a version:
-
-```powershell
-go get gitea.tbdevent.eu/ilbinek/reforger_utils@v0.0.15
-```
-
-## Features
-
-- Download asset metadata and manifests from BI APIs.
-- Download and rebuild fragmented files.
-- Parse PAK file entries.
-- Optional SOCKS5/Tor support for HTTP requests.
-
-## Basic Usage
-
-```go
-package main
-
-import (
-	"fmt"
-
-	"gitea.tbdevent.eu/ilbinek/reforger_utils"
-)
-
-func main() {
-	assets, err := reforger_utils.DoAssetsRequest("BADC0DEDABBEDA5E", "0.12.4573")
-	if err != nil {
-		panic(err)
-	}
-
-	fmt.Printf("found %d assets\n", len(assets))
-}
-```
-
-## Logging
-
-The library uses structured slog logging and writes JSON logs to stdout by default.
-This is container-friendly (including Dokploy) because logs are newline-based and do not overwrite previous output.
-
-You can inject your own logger:
-
-```go
-package main
-
-import (
-	"log/slog"
-	"os"
-
-	"gitea.tbdevent.eu/ilbinek/reforger_utils"
-)
-
-func main() {
-	logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
-	reforger_utils.SetLogger(logger)
-}
-```
-
-## Development
-
-```powershell
-go test ./...
-```

cmd/main.go

@@ -10,11 +10,11 @@ import (
 	"gitea.tbdevent.eu/ilbinek/reforger_utils"
 )
 
-const DEBUG = true
+const DEBUG = false
 
 func main() {
-	addonID := "BADC0DEDABBEDA5E"
+	addonID := "61EB463A41FE5274"
-	version := "0.12.4573"
+	version := "0.1.166"
 
 	assetsReply, err := reforger_utils.DoAssetsRequest(addonID, version)
 	if err != nil {
@@ -65,10 +65,6 @@ func main() {
 				panic(err)
 			}
 			
-			if err := os.WriteFile(filepath.Join(downloadDir, file.Name), downedFile, os.ModePerm); err != nil {
-				panic(err)
-			}
-			
 			fmt.Println("Downloaded", file.Name, len(downedFile), "bytes")
 
 			if file.Name[len(file.Name)-4:] != ".pak" {

go.mod

@@ -1,5 +1,3 @@
 module gitea.tbdevent.eu/ilbinek/reforger_utils
 
 go 1.25.0
-
-require golang.org/x/net v0.44.0 // indirect

go.sum

@@ -1,2 +0,0 @@
-golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
-golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=

logger.go

@@ -1,36 +0,0 @@
-package reforger_utils
-
-import (
-	"log/slog"
-	"os"
-	"sync"
-)
-
-var (
-	loggerMu sync.RWMutex
-	logger   = newDefaultLogger()
-)
-
-func newDefaultLogger() *slog.Logger {
-	return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
-}
-
-// SetLogger configures the package logger.
-// Passing nil resets it to the default JSON logger writing to stdout.
-func SetLogger(l *slog.Logger) {
-	loggerMu.Lock()
-	defer loggerMu.Unlock()
-
-	if l == nil {
-		logger = newDefaultLogger()
-		return
-	}
-
-	logger = l
-}
-
-func getLogger() *slog.Logger {
-	loggerMu.RLock()
-	defer loggerMu.RUnlock()
-	return logger
-}

util.go

@@ -9,25 +9,11 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
-
+	"sync"
-	"golang.org/x/net/proxy"
 )
 
 var Debug = false
-var Tor = false
+var DownThreads = 8
-var Socks5Address = "127.0.0.1:9050"
-
-func getHttpClient() (*http.Client, error) {
-	if Tor {
-		dialer, err := proxy.SOCKS5("tcp", Socks5Address, nil, proxy.Direct)
-		if err != nil {
-			return nil, err
-		}
-		transport := &http.Transport{Dial: dialer.Dial}
-		return &http.Client{Transport: transport}, nil
-	}
-	return http.DefaultClient, nil
-}
 
 func DoAssetsRequest(addonID, version string) (AssetsReply, error) {
 	url := "https://api-ar-workshop.bistudio.com/workshop-api/api/v3.0/s2s/assets/download-list"
@@ -43,19 +29,16 @@ func DoAssetsRequest(addonID, version string) (AssetsReply, error) {
 	req.Header.Add("user-agent", "Arma Reforger/1.4.0.53 (Headless; Windows)")
 	req.Header.Add("content-length", fmt.Sprintf("%d", len(body)))
 
-	client, err := getHttpClient()
+	res, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return AssetsReply{}, err
-	}
-	res, err := client.Do(req)
 	if err != nil {
 		return AssetsReply{}, err
 	}
 	defer res.Body.Close()
 
 	if res.StatusCode != 200 {
+		// print entire body
 		body, _ := io.ReadAll(res.Body)
-		getLogger().Error("assets request failed", "status", res.Status, "addon_id", addonID, "version", version, "body", string(body))
+		fmt.Println(string(body))
 		return AssetsReply{}, fmt.Errorf("bad status: %s", res.Status)
 	}
 
@@ -63,7 +46,7 @@ func DoAssetsRequest(addonID, version string) (AssetsReply, error) {
 	decoder := json.NewDecoder(res.Body)
 	var assetsReply AssetsReply
 	if err := decoder.Decode(&assetsReply); err != nil {
-		return AssetsReply{}, fmt.Errorf("failed to decode assets response: %w", err)
+		panic(err)
 	}
 
 	return assetsReply, nil
@@ -76,11 +59,7 @@ func GetManifest(manifestUrl string) (Manifest, error) {
 		return Manifest{}, err
 	}
 
-	client, err := getHttpClient()
+	res, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return Manifest{}, err
-	}
-	res, err := client.Do(req)
 	if err != nil {
 		return Manifest{}, err
 	}
@@ -93,7 +72,7 @@ func GetManifest(manifestUrl string) (Manifest, error) {
 	// Decode the JSON response
 	var manifest Manifest
 	if err := json.NewDecoder(res.Body).Decode(&manifest); err != nil {
-		return Manifest{}, fmt.Errorf("failed to decode manifest response: %w", err)
+		panic(err)
 	}
 
 	return manifest, nil
@@ -118,8 +97,8 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro
 		if err != nil {
 			return nil, err
 		}
-		remContent = *content
+		remContent = content
-		downloaded += len(*content)
+		downloaded += len(content)
 
 		if Debug {
 			// save remainder to file in the fragments/ directory, prefix with gap
@@ -134,30 +113,75 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro
 	// start downloading chunks
 	ret := make([]byte, manifest.Size)
 	totalMB := float32(manifest.Size) / 1024.0 / 1024.0
-	lastLoggedPercent := float32(-1)
-	log := getLogger().With("file", filen)
-	log.Info("download started", "size_bytes", manifest.Size)
 
-	// Download all fragments serially to keep execution simple and deterministic.
+	var wg sync.WaitGroup
-	for idx, fragment := range manifest.Fragments {
+	var mu sync.Mutex
-		url := transformShaToURL(fragment.Sha512, fragment.Size)
+
-		content, err := getContent(url)
+	fragmentsChan := make(chan Fragment, len(manifest.Fragments))
-		if err != nil {
+	errChan := make(chan error, DownThreads)
-			return nil, err
+
-		}
+	// Start worker goroutines
-		downloaded += len(*content) * len(fragment.Offsets)
+	for i := 0; i < DownThreads; i++ {
-		for _, offset := range fragment.Offsets {
+		wg.Go(func() {
-			trackFrag = append(trackFrag, Frag{Offset: offset, Size: len(*content)})
+			for fragment := range fragmentsChan {
-			copy(ret[offset:offset+len(*content)], *content)
+				url := transformShaToURL(fragment.Sha512, fragment.Size)
-		}
+				content, err := getContent(url)
-		percent := float32(downloaded) / float32(manifest.Size) * 100.0
+				if err != nil {
-		downedMB := float32(downloaded) / 1024.0 / 1024.0
+					// Handle error, maybe by sending it to an error channel
-		if percent-lastLoggedPercent >= 5 || idx == len(manifest.Fragments)-1 {
+					// For now, we'll just skip and print.
-			log.Info("download progress", "downloaded_mb", downedMB, "total_mb", totalMB, "percent", percent)
+					fmt.Printf("Error downloading fragment %s: %v\n", fragment.Sha512, err)
-			lastLoggedPercent = percent
+					errChan <- err
-		}
+				}
+
+				mu.Lock()
+				// Safely write to the shared slice and update progress
+				for _, offset := range fragment.Offsets {
+					copy(ret[offset:offset+len(content)], content)
+				}
+				downloaded += len(content) * len(fragment.Offsets)
+
+				// Track downloaded fragments
+				for _, offset := range fragment.Offsets {
+					trackFrag = append(trackFrag, Frag{
+						Offset: offset,
+						Size:   len(content),
+					})
+				}
+
+				// Print progress
+				downedMB := float32(downloaded) / 1024.0 / 1024.0
+				percent := float32(downloaded) / float32(manifest.Size) * 100.0
+				fmt.Printf("\r\033[32mDownloaded\033[0m %.2fMB/%.2fMB (\033[36m%.2f%%\033[0m) of \033[33m%s\033[0m", downedMB, totalMB, percent, filen)
+
+				mu.Unlock()
+
+				// The Debug section can be safely moved here as well
+				if os.Getenv("DEBUG") == "true" { // Using env var for a cleaner check
+					mu.Lock() // Re-lock for file write
+					if err := os.WriteFile(filepath.Join(downloadDir, "fragments", filen, fmt.Sprintf("%d.%s.%d.bytes", fragment.Offsets[0], fragment.Sha512, fragment.Size)), content, os.ModePerm); err != nil {
+						fmt.Printf("Error saving file: %v\n", err)
+					}
+					mu.Unlock()
+				}
+			}
+		})
 	}
 
+	// Send fragments to the channel
+	for _, fragment := range manifest.Fragments {
+		fragmentsChan <- fragment
+	}
+	close(fragmentsChan) // Close the channel to signal workers no more tasks are coming
+
+	// Wait for all workers to finish
+	wg.Wait()
+
+	// Check for errors
+	if len(errChan) > 0 {
+		return nil, <-errChan // Return the first error encountered
+	}
+	close(errChan)
+
 	// sort trackFrag by Offset
 	slices.SortFunc(trackFrag, func(a, b Frag) int {
 		return a.Offset - b.Offset
@@ -171,7 +195,7 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro
 			// gap detected
 			gapSize := frag.Offset - curPos
 			if currentRem+gapSize > len(remContent) {
-				return nil, fmt.Errorf("not enough remainder content to fill gap in %s", filen)
+				panic("not enough remainder content to fill gap")
 			}
 			copy(ret[curPos:curPos+gapSize], remContent[currentRem:currentRem+gapSize])
 			remContent = remContent[gapSize:]
@@ -181,24 +205,19 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro
 
 	// Append rest of remainder content
 	copy(ret[curPos:], remContent)
-	log.Info("download completed", "size_bytes", len(ret))
+	fmt.Println()
 
 	return ret, nil
 }
 
-func getContent(url string) (*[]byte, error) {
+func getContent(url string) ([]byte, error) {
-	client, err := getHttpClient()
+	resp, err := http.Get(url)
-	if err != nil {
-		return nil, err
-	}
-	resp, err := client.Get(url)
 	if err != nil {
 		return nil, err
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != 200 {
-		getLogger().Error("content request failed", "status", resp.Status, "url", url)
 		return nil, fmt.Errorf("bad status: %s", resp.Status)
 	}
 
@@ -207,7 +226,7 @@ func getContent(url string) (*[]byte, error) {
 		return nil, err
 	}
 
-	return &body, nil
+	return body, nil
 }
 
 func transformShaToURL(sha512s string, size int) string {