feat: assets download utils

next/assets/extract.go

@@ -85,7 +85,7 @@ func archiveType(data []byte) uint {
 		Logger.Debugf("Data detected as xz format")
 		return xzArchive
 	default:
-		Logger.Debugf("Data detected as non-archive format -> %s", mime)
+		Logger.Debugf("Data detected as non-archive format -> `%s`", mime)
 		return notArchive
 	}
 }

next/assets/remote.go

@@ -0,0 +1,124 @@
+package assets
+
+import (
+	. "XProxy/next/logger"
+	"bytes"
+	"github.com/andybalholm/brotli"
+	"github.com/go-http-utils/headers"
+	"github.com/klauspost/compress/flate"
+	"github.com/klauspost/compress/gzip"
+	"io"
+	"net/http"
+	"net/url"
+)
+
+// broltiDecode handles brolti encoding in http responses.
+func broltiDecode(stream io.Reader) ([]byte, error) {
+	var buffer bytes.Buffer
+	_, err := io.Copy(&buffer, brotli.NewReader(stream))
+	if err != nil {
+		Logger.Errorf("Failed to decode http responses with brolti encoding -> %v", err)
+		return nil, err
+	}
+	return buffer.Bytes(), nil
+}
+
+// gzipDecode handles gzip encoding in http responses.
+func gzipDecode(stream io.Reader) ([]byte, error) {
+	reader, err := gzip.NewReader(stream)
+	if err != nil {
+		Logger.Errorf("Failed to decode http responses with gzip encoding -> %v", err)
+		return nil, err
+	}
+
+	var buffer bytes.Buffer
+	_, err = io.Copy(&buffer, reader)
+	if err != nil {
+		Logger.Errorf("Failed to handle gzip reader -> %v", err)
+		return nil, err
+	}
+	return buffer.Bytes(), nil
+}
+
+// deflateDecode handles deflate encoding in http responses.
+func deflateDecode(stream io.Reader) ([]byte, error) {
+	var buffer bytes.Buffer
+	_, err := io.Copy(&buffer, flate.NewReader(stream))
+	if err != nil {
+		Logger.Errorf("Failed to decode http responses with deflate encoding -> %v", err)
+		return nil, err
+	}
+	return buffer.Bytes(), nil
+}
+
+// nonDecode handles plain encoding in http responses.
+func nonDecode(stream io.Reader) ([]byte, error) {
+	var buffer bytes.Buffer
+	_, err := io.Copy(&buffer, stream)
+	if err != nil {
+		Logger.Errorf("Failed to read http responses -> %v", err)
+		return nil, err
+	}
+	return buffer.Bytes(), nil
+}
+
+// createClient build http client based on http or socks proxy url.
+func createClient(remoteUrl string, proxyUrl string) (http.Client, error) {
+	if proxyUrl == "" {
+		Logger.Infof("Downloading `%s` without proxy", remoteUrl)
+		return http.Client{}, nil
+	}
+	Logger.Infof("Downloading `%s` via `%s`", remoteUrl, proxyUrl)
+	proxy, err := url.Parse(proxyUrl)
+	if err != nil {
+		Logger.Errorf("Invalid proxy url `%s` -> %v", proxyUrl, err)
+		return http.Client{}, err
+	}
+	return http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyURL(proxy),
+		},
+	}, nil
+}
+
+// Download obtains resource file from the remote server and supports proxy.
+func Download(url string, proxy string) ([]byte, error) {
+	client, err := createClient(url, proxy)
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		Logger.Errorf("Failed to create http request -> %v", err)
+		return nil, err
+	}
+	req.Header.Set(headers.AcceptEncoding, "gzip, deflate, br")
+	resp, err := client.Do(req)
+	if err != nil {
+		Logger.Errorf("Failed to execute http request -> %v", err)
+		return nil, err
+	}
+	defer resp.Body.Close()
+	Logger.Debugf("Remote data downloaded successfully")
+
+	var content []byte
+	switch resp.Header.Get(headers.ContentEncoding) {
+	case "br":
+		Logger.Debugf("Downloaded content using brolti encoding")
+		content, err = broltiDecode(resp.Body)
+	case "gzip":
+		Logger.Debugf("Downloaded content using gzip encoding")
+		content, err = gzipDecode(resp.Body)
+	case "deflate":
+		Logger.Debugf("Downloaded content using deflate encoding")
+		content, err = deflateDecode(resp.Body)
+	default:
+		content, err = nonDecode(resp.Body)
+	}
+	if err != nil {
+		return nil, err
+	}
+	Logger.Debugf("Download `%s` successfully -> %d bytes", url, len(content))
+	return content, nil
+}

next/main.go

@@ -3,7 +3,6 @@ package main
 import (
 	"XProxy/next/assets"
 	. "XProxy/next/logger"
-	"os"
 )
 
 const gzSample = "/root/XProxy/LICENSE.gz"
@@ -14,13 +13,19 @@ func main() {
 
 	//raw, _ := os.ReadFile(gzSample)
 	//raw, _ := os.ReadFile(bz2Sample)
-	raw, _ := os.ReadFile(xzSample)
-	Logger.Debugf("data len -> %d", len(raw))
-	ret, err := assets.Extract(raw)
-	if err != nil {
-		Logger.Debugf("extract error -> %v", err)
-	}
-	Logger.Debugf("extract ok -> len = %d", len(ret))
+	//raw, _ := os.ReadFile(xzSample)
+	//Logger.Debugf("data len -> %d", len(raw))
+	//ret, err := assets.Extract(raw)
+	//if err != nil {
+	//	Logger.Debugf("extract error -> %v", err)
+	//}
+	//Logger.Debugf("extract ok -> len = %d", len(ret))
 	//os.WriteFile("demo.data", ret, 0777)
 
+	//raw, _ := assets.Download("https://cdn.dnomd343.top/v2ray-rules-dat/geosite.dat", "")
+	//raw, _ := assets.Download("https://cdn.dnomd343.top/v2ray-rules-dat/geosite.dat", "socks5://192.168.2.2:1084")
+	raw, _ := assets.Download("https://cdn.dnomd343.top/v2ray-rules-dat/geosite.dat.xz", "")
+	ret, _ := assets.Extract(raw)
+	Logger.Debugf("content size -> %d", len(ret))
+
 }