Add README with install and logging docs

README.md

@@ -0,0 +1,76 @@
+# reforger_utils
+
+Utilities for downloading and processing Arma Reforger workshop content.
+
+## Install
+
+This module is hosted on a private Gitea instance.
+
+Set private module settings once:
+
+```powershell
+go env -w GOPRIVATE=gitea.tbdevent.eu
+```
+
+Then install a version:
+
+```powershell
+go get gitea.tbdevent.eu/ilbinek/reforger_utils@v0.0.15
+```
+
+## Features
+
+- Download asset metadata and manifests from BI APIs.
+- Download and rebuild fragmented files.
+- Parse PAK file entries.
+- Optional SOCKS5/Tor support for HTTP requests.
+
+## Basic Usage
+
+```go
+package main
+
+import (
+	"fmt"
+
+	"gitea.tbdevent.eu/ilbinek/reforger_utils"
+)
+
+func main() {
+	assets, err := reforger_utils.DoAssetsRequest("BADC0DEDABBEDA5E", "0.12.4573")
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("found %d assets\n", len(assets))
+}
+```
+
+## Logging
+
+The library uses structured slog logging and writes JSON logs to stdout by default.
+This is container-friendly (including Dokploy) because logs are newline-based and do not overwrite previous output.
+
+You can inject your own logger:
+
+```go
+package main
+
+import (
+	"log/slog"
+	"os"
+
+	"gitea.tbdevent.eu/ilbinek/reforger_utils"
+)
+
+func main() {
+	logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
+	reforger_utils.SetLogger(logger)
+}
+```
+
+## Development
+
+```powershell
+go test ./...
+```

cmd/main.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"gitea.tbdevent.eu/ilbinek/reforger_utils"
+)
+
+const DEBUG = true
+
+func main() {
+	addonID := "BADC0DEDABBEDA5E"
+	version := "0.12.4573"
+
+	assetsReply, err := reforger_utils.DoAssetsRequest(addonID, version)
+	if err != nil {
+		panic(err)
+	}
+
+	downloadDir := filepath.Join("downloads", addonID)
+	if DEBUG {
+		if err := os.MkdirAll(downloadDir, os.ModePerm); err != nil {
+			panic(err)
+		}
+	}
+
+	bytesReply, err := json.MarshalIndent(assetsReply, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+
+	if DEBUG {
+		if err := os.WriteFile(filepath.Join(downloadDir, "assets.json"), bytesReply, os.ModePerm); err != nil {
+			panic(err)
+		}
+	}
+
+	for _, asset := range assetsReply {
+		for _, file := range asset.Files {
+			manifest, err := reforger_utils.GetManifest(file.ManifestPath)
+			if err != nil {
+				panic(err)
+			}
+
+			if DEBUG {
+				// save manifest
+				var manifestBuf bytes.Buffer
+				manifestEncoder := json.NewEncoder(&manifestBuf)
+				manifestEncoder.SetIndent("", "  ")
+				if err := manifestEncoder.Encode(manifest); err != nil {
+					panic(err)
+				}
+
+				if err := os.WriteFile(filepath.Join(downloadDir, file.Name+".manifest.json"), manifestBuf.Bytes(), os.ModePerm); err != nil {
+					panic(err)
+				}
+			}
+			
+			downedFile, err := reforger_utils.Download(manifest, downloadDir, file.Name)
+			if err != nil {
+				panic(err)
+			}
+
+			if err := os.WriteFile(filepath.Join(downloadDir, file.Name), downedFile, os.ModePerm); err != nil {
+				panic(err)
+			}
+			
+			fmt.Println("Downloaded", file.Name, len(downedFile), "bytes")
+
+			if file.Name[len(file.Name)-4:] != ".pak" {
+				continue
+			}
+
+			pakInfo, err := reforger_utils.GetPakFileInfo(&downedFile)
+			if err != nil {
+				panic(err)
+			}
+
+			for _, entry := range pakInfo {
+				fmt.Printf(" - %s (offset: %d, size: %d)\n", entry.Name, entry.Offset, entry.Size)
+				if DEBUG {
+					entryData := downedFile[entry.Offset : entry.Offset+entry.Size]
+					path := filepath.Join("downloads", addonID, "extracted", entry.Name)
+					if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
+						panic(err)
+					}
+					if err := os.WriteFile(path, entryData, os.ModePerm); err != nil {
+						panic(err)
+					}
+				}
+			}
+		}
+	}
+}

go.mod

@@ -1,3 +1,5 @@
 module gitea.tbdevent.eu/ilbinek/reforger_utils
 
 go 1.25.0
+
+require golang.org/x/net v0.44.0 // indirect

go.sum

@@ -0,0 +1,2 @@
+golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
+golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=

logger.go

@@ -0,0 +1,36 @@
+package reforger_utils
+
+import (
+	"log/slog"
+	"os"
+	"sync"
+)
+
+var (
+	loggerMu sync.RWMutex
+	logger   = newDefaultLogger()
+)
+
+func newDefaultLogger() *slog.Logger {
+	return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
+}
+
+// SetLogger configures the package logger.
+// Passing nil resets it to the default JSON logger writing to stdout.
+func SetLogger(l *slog.Logger) {
+	loggerMu.Lock()
+	defer loggerMu.Unlock()
+
+	if l == nil {
+		logger = newDefaultLogger()
+		return
+	}
+
+	logger = l
+}
+
+func getLogger() *slog.Logger {
+	loggerMu.RLock()
+	defer loggerMu.RUnlock()
+	return logger
+}

pakUtils.go

@@ -146,8 +146,8 @@ func parseDirectory(r *bytes.Reader, path string, dataBlockOffset int64) ([]PakE
 	return entries, totalSize, nil
 }
 
-func GetPakFileInfo(data []byte) ([]PakEntry, error) {
+func GetPakFileInfo(data *[]byte) ([]PakEntry, error) {
-	r := bytes.NewReader(data)
+	r := bytes.NewReader(*data)
 
 	// Read FORM header
 	if err := readString(r, "FORM"); err != nil {

structs.go

@@ -61,11 +61,13 @@ type Manifest struct {
 		Size    int    `json:"size"`
 		Offsets []int  `json:"offsets"`
 	} `json:"remainder"`
-	Fragments []struct {
+	Fragments []Fragment `json:"fragments"`
+}
+
+type Fragment struct {
 	Sha512  string `json:"sha512"`
 	Size    int    `json:"size"`
 	Offsets []int  `json:"offsets"`
-	} `json:"fragments"`
 }
 
 type RDBHeader struct {

util.go

@@ -9,9 +9,25 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
+
+	"golang.org/x/net/proxy"
 )
 
 var Debug = false
+var Tor = false
+var Socks5Address = "127.0.0.1:9050"
+
+func getHttpClient() (*http.Client, error) {
+	if Tor {
+		dialer, err := proxy.SOCKS5("tcp", Socks5Address, nil, proxy.Direct)
+		if err != nil {
+			return nil, err
+		}
+		transport := &http.Transport{Dial: dialer.Dial}
+		return &http.Client{Transport: transport}, nil
+	}
+	return http.DefaultClient, nil
+}
 
 func DoAssetsRequest(addonID, version string) (AssetsReply, error) {
 	url := "https://api-ar-workshop.bistudio.com/workshop-api/api/v3.0/s2s/assets/download-list"
@@ -27,24 +43,27 @@ func DoAssetsRequest(addonID, version string) (AssetsReply, error) {
 	req.Header.Add("user-agent", "Arma Reforger/1.4.0.53 (Headless; Windows)")
 	req.Header.Add("content-length", fmt.Sprintf("%d", len(body)))
 
-	res, err := http.DefaultClient.Do(req)
+	client, err := getHttpClient()
+	if err != nil {
+		return AssetsReply{}, err
+	}
+	res, err := client.Do(req)
 	if err != nil {
 		return AssetsReply{}, err
 	}
 	defer res.Body.Close()
 
 	if res.StatusCode != 200 {
-		// print entire body
 		body, _ := io.ReadAll(res.Body)
-		fmt.Println(string(body))
+		getLogger().Error("assets request failed", "status", res.Status, "addon_id", addonID, "version", version, "body", string(body))
-		panic("bad status: " + res.Status)
+		return AssetsReply{}, fmt.Errorf("bad status: %s", res.Status)
 	}
 
 	// Process response
 	decoder := json.NewDecoder(res.Body)
 	var assetsReply AssetsReply
 	if err := decoder.Decode(&assetsReply); err != nil {
-		panic(err)
+		return AssetsReply{}, fmt.Errorf("failed to decode assets response: %w", err)
 	}
 
 	return assetsReply, nil
@@ -57,7 +76,11 @@ func GetManifest(manifestUrl string) (Manifest, error) {
 		return Manifest{}, err
 	}
 
-	res, err := http.DefaultClient.Do(req)
+	client, err := getHttpClient()
+	if err != nil {
+		return Manifest{}, err
+	}
+	res, err := client.Do(req)
 	if err != nil {
 		return Manifest{}, err
 	}
@@ -70,7 +93,7 @@ func GetManifest(manifestUrl string) (Manifest, error) {
 	// Decode the JSON response
 	var manifest Manifest
 	if err := json.NewDecoder(res.Body).Decode(&manifest); err != nil {
-		panic(err)
+		return Manifest{}, fmt.Errorf("failed to decode manifest response: %w", err)
 	}
 
 	return manifest, nil
@@ -80,7 +103,7 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro
 
 	if Debug {
 		// create the directory if it doesn't exist
-		if err := os.MkdirAll(downloadDir + "/fragments/" + filen, os.ModePerm); err != nil {
+		if err := os.MkdirAll(downloadDir+"/fragments/"+filen, os.ModePerm); err != nil {
 			return nil, err
 		}
 	}
@@ -89,14 +112,14 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro
 	downloaded := 0
 	remContent := make([]byte, 0)
 	currentRem := 0
-	if (manifest.Remainder.Size > 0) {
+	if manifest.Remainder.Size > 0 {
 		url := transformShaToURL(manifest.Remainder.Sha512, manifest.Remainder.Size)
 		content, err := getContent(url)
 		if err != nil {
 			return nil, err
 		}
-		remContent = content
+		remContent = *content
-		downloaded += len(content)
+		downloaded += len(*content)
 
 		if Debug {
 			// save remainder to file in the fragments/ directory, prefix with gap
@@ -109,29 +132,30 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro
 	trackFrag := make([]Frag, 0)
 
 	// start downloading chunks
-	//currentOffset := 0
 	ret := make([]byte, manifest.Size)
-	for _, fragment := range manifest.Fragments {
+	totalMB := float32(manifest.Size) / 1024.0 / 1024.0
+	lastLoggedPercent := float32(-1)
+	log := getLogger().With("file", filen)
+	log.Info("download started", "size_bytes", manifest.Size)
+
+	// Download all fragments serially to keep execution simple and deterministic.
+	for idx, fragment := range manifest.Fragments {
 		url := transformShaToURL(fragment.Sha512, fragment.Size)
 		content, err := getContent(url)
 		if err != nil {
 			return nil, err
 		}
-		downloaded += len(content) * len(fragment.Offsets)
+		downloaded += len(*content) * len(fragment.Offsets)
 		for _, offset := range fragment.Offsets {
-			trackFrag = append(trackFrag, Frag{Offset: offset, Size: len(content)})
+			trackFrag = append(trackFrag, Frag{Offset: offset, Size: len(*content)})
-			copy(ret[offset:offset+len(content)], content)
+			copy(ret[offset:offset+len(*content)], *content)
 		}
-
-		if Debug {
-			// save fragment to file in the fragments/ directory
-			if err := os.WriteFile(filepath.Join(downloadDir, "fragments", filen, fmt.Sprintf("%d.%s.%d.bytes", fragment.Offsets[0], fragment.Sha512, fragment.Size)), content, os.ModePerm); err != nil {
-				return nil, err
-			}
-		}
-		
 		percent := float32(downloaded) / float32(manifest.Size) * 100.0
-        fmt.Printf("\r\033[32mDownloaded\033[0m %d/%d (\033[36m%.2f%%\033[0m) of \033[33m%s\033[0m", downloaded, manifest.Size, percent, filen)
+		downedMB := float32(downloaded) / 1024.0 / 1024.0
+		if percent-lastLoggedPercent >= 5 || idx == len(manifest.Fragments)-1 {
+			log.Info("download progress", "downloaded_mb", downedMB, "total_mb", totalMB, "percent", percent)
+			lastLoggedPercent = percent
+		}
 	}
 
 	// sort trackFrag by Offset
@@ -141,12 +165,13 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro
 
 	// check for gaps in trackFrag and fill with remainder content
 	curPos := 0
+
 	for _, frag := range trackFrag {
 		if frag.Offset > curPos {
 			// gap detected
 			gapSize := frag.Offset - curPos
-			if currentRem + gapSize > len(remContent) {
+			if currentRem+gapSize > len(remContent) {
-				panic("not enough remainder content to fill gap")
+				return nil, fmt.Errorf("not enough remainder content to fill gap in %s", filen)
 			}
 			copy(ret[curPos:curPos+gapSize], remContent[currentRem:currentRem+gapSize])
 			remContent = remContent[gapSize:]
@@ -156,19 +181,24 @@ func Download(manifest Manifest, downloadDir string, filen string) ([]byte, erro
 
 	// Append rest of remainder content
 	copy(ret[curPos:], remContent)
-	fmt.Println()
+	log.Info("download completed", "size_bytes", len(ret))
 
 	return ret, nil
 }
 
-func getContent(url string) ([]byte, error) {
+func getContent(url string) (*[]byte, error) {
-	resp, err := http.Get(url)
+	client, err := getHttpClient()
+	if err != nil {
+		return nil, err
+	}
+	resp, err := client.Get(url)
 	if err != nil {
 		return nil, err
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != 200 {
+		getLogger().Error("content request failed", "status", resp.Status, "url", url)
 		return nil, fmt.Errorf("bad status: %s", resp.Status)
 	}
 
@@ -177,7 +207,7 @@ func getContent(url string) ([]byte, error) {
 		return nil, err
 	}
 
-	return body, nil
+	return &body, nil
 }
 
 func transformShaToURL(sha512s string, size int) string {