Browse Source

update: adapt logger to assets module

feature/assets
Dnomd343 1 year ago
parent
commit
0b02b55a0e
  1. 10
      go.mod
  2. 8
      go.sum
  3. 11
      next/assets/assets.go
  4. 36
      next/assets/extract.go
  5. 36
      next/assets/remote.go
  6. 28
      next/main.go

10
go.mod

@ -5,7 +5,9 @@ go 1.21.0
require ( require (
github.com/BurntSushi/toml v1.3.2 github.com/BurntSushi/toml v1.3.2
github.com/andybalholm/brotli v1.0.5 github.com/andybalholm/brotli v1.0.5
github.com/gabriel-vasile/mimetype v1.4.2
github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a
github.com/gookit/color v1.5.4
github.com/klauspost/compress v1.16.7 github.com/klauspost/compress v1.16.7
github.com/robfig/cron v1.2.0 github.com/robfig/cron v1.2.0
github.com/sirupsen/logrus v1.9.3 github.com/sirupsen/logrus v1.9.3
@ -15,14 +17,8 @@ require (
) )
require ( require (
github.com/fatih/color v1.15.0 // indirect
github.com/gookit/color v1.5.4 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect go.uber.org/multierr v1.10.0 // indirect
github.com/stretchr/testify v1.8.4 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/net v0.8.0 // indirect golang.org/x/net v0.8.0 // indirect
golang.org/x/sys v0.10.0 // indirect golang.org/x/sys v0.10.0 // indirect
) )

8
go.sum

@ -11,6 +11,8 @@ github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno= github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno=
github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw= github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw=
github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@ -25,10 +27,12 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=

11
next/assets/assets.go

@ -0,0 +1,11 @@
package assets
import "XProxy/next/logger"
func Demo() {
raw, _ := download("https://cdn.dnomd343.top/v2ray-rules-dat/geosite.dat", "")
//raw, _ := download("https://cdn.dnomd343.top/v2ray-rules-dat/geosite.dat", "socks5://192.168.2.2:1084")
//raw, _ := download("https://cdn.dnomd343.top/v2ray-rules-dat/geosite.dat.xz", "")
ret, _ := tryExtract(raw)
logger.Debugf("content size -> %d", len(ret))
}

36
next/assets/extract.go

@ -1,7 +1,7 @@
package assets package assets
import ( import (
. "XProxy/next/logger" "XProxy/next/logger"
"bytes" "bytes"
"compress/bzip2" "compress/bzip2"
"github.com/gabriel-vasile/mimetype" "github.com/gabriel-vasile/mimetype"
@ -19,10 +19,10 @@ const (
// gzipExtract use to extract independent gzip archive data. // gzipExtract use to extract independent gzip archive data.
func gzipExtract(data []byte) ([]byte, error) { func gzipExtract(data []byte) ([]byte, error) {
Logger.Debugf("Start extracting gzip archive -> %d bytes", len(data)) logger.Debugf("Start extracting gzip archive -> %d bytes", len(data))
reader, err := gzip.NewReader(bytes.NewReader(data)) reader, err := gzip.NewReader(bytes.NewReader(data))
if err != nil { if err != nil {
Logger.Errorf("Failed to extract gzip archive -> %v", err) logger.Errorf("Failed to extract gzip archive -> %v", err)
return nil, err return nil, err
} }
defer reader.Close() defer reader.Close()
@ -30,44 +30,44 @@ func gzipExtract(data []byte) ([]byte, error) {
var buffer bytes.Buffer var buffer bytes.Buffer
size, err := io.Copy(&buffer, reader) size, err := io.Copy(&buffer, reader)
if err != nil { if err != nil {
Logger.Errorf("Failed to handle gzip archive -> %v", err) logger.Errorf("Failed to handle gzip archive -> %v", err)
return nil, err return nil, err
} }
Logger.Debugf("Extracted gzip archive successfully -> %d bytes", size) logger.Debugf("Extracted gzip archive successfully -> %d bytes", size)
return buffer.Bytes(), nil return buffer.Bytes(), nil
} }
// bzip2Extract use to extract independent bzip2 archive data. // bzip2Extract use to extract independent bzip2 archive data.
func bzip2Extract(data []byte) ([]byte, error) { func bzip2Extract(data []byte) ([]byte, error) {
Logger.Debugf("Start extracting bzip2 archive -> %d bytes", len(data)) logger.Debugf("Start extracting bzip2 archive -> %d bytes", len(data))
reader := bzip2.NewReader(bytes.NewReader(data)) reader := bzip2.NewReader(bytes.NewReader(data))
var buffer bytes.Buffer var buffer bytes.Buffer
size, err := io.Copy(&buffer, reader) size, err := io.Copy(&buffer, reader)
if err != nil { if err != nil {
Logger.Errorf("Failed to extract bzip2 archive -> %v", err) logger.Errorf("Failed to extract bzip2 archive -> %v", err)
return nil, err return nil, err
} }
Logger.Debugf("Extracted bzip2 archive successfully -> %d bytes", size) logger.Debugf("Extracted bzip2 archive successfully -> %d bytes", size)
return buffer.Bytes(), nil return buffer.Bytes(), nil
} }
// xzExtract use to extract independent xz archive data. // xzExtract use to extract independent xz archive data.
func xzExtract(data []byte) ([]byte, error) { func xzExtract(data []byte) ([]byte, error) {
Logger.Debugf("Start extracting xz archive -> %d bytes", len(data)) logger.Debugf("Start extracting xz archive -> %d bytes", len(data))
reader, err := xz.NewReader(bytes.NewReader(data)) reader, err := xz.NewReader(bytes.NewReader(data))
if err != nil { if err != nil {
Logger.Errorf("Failed to extract xz archive -> %v", err) logger.Errorf("Failed to extract xz archive -> %v", err)
return nil, err return nil, err
} }
var buffer bytes.Buffer var buffer bytes.Buffer
size, err := io.Copy(&buffer, reader) size, err := io.Copy(&buffer, reader)
if err != nil { if err != nil {
Logger.Errorf("Failed to handle xz archive -> %v", err) logger.Errorf("Failed to handle xz archive -> %v", err)
return nil, err return nil, err
} }
Logger.Debugf("Extracted xz archive successfully -> %d bytes", size) logger.Debugf("Extracted xz archive successfully -> %d bytes", size)
return buffer.Bytes(), nil return buffer.Bytes(), nil
} }
@ -76,23 +76,23 @@ func archiveType(data []byte) uint {
mime := mimetype.Detect(data) mime := mimetype.Detect(data)
switch mime.String() { switch mime.String() {
case "application/gzip": case "application/gzip":
Logger.Debugf("Data detected as gzip format") logger.Debugf("Data detected as gzip format")
return gzipArchive return gzipArchive
case "application/x-bzip2": case "application/x-bzip2":
Logger.Debugf("Data detected as bzip2 format") logger.Debugf("Data detected as bzip2 format")
return bzip2Archive return bzip2Archive
case "application/x-xz": case "application/x-xz":
Logger.Debugf("Data detected as xz format") logger.Debugf("Data detected as xz format")
return xzArchive return xzArchive
default: default:
Logger.Debugf("Data detected as non-archive format -> `%s`", mime) logger.Debugf("Data detected as non-archive format -> `%s`", mime)
return notArchive return notArchive
} }
} }
// Extract will try to extract the data as a compressed format, and will // tryExtract will try to extract the data as a compressed format, and will
// return the original data if it cannot be determined. // return the original data if it cannot be determined.
func Extract(data []byte) ([]byte, error) { func tryExtract(data []byte) ([]byte, error) {
switch archiveType(data) { switch archiveType(data) {
case gzipArchive: case gzipArchive:
return gzipExtract(data) return gzipExtract(data)

36
next/assets/remote.go

@ -1,7 +1,7 @@
package assets package assets
import ( import (
. "XProxy/next/logger" "XProxy/next/logger"
"bytes" "bytes"
"github.com/andybalholm/brotli" "github.com/andybalholm/brotli"
"github.com/go-http-utils/headers" "github.com/go-http-utils/headers"
@ -17,7 +17,7 @@ func broltiDecode(stream io.Reader) ([]byte, error) {
var buffer bytes.Buffer var buffer bytes.Buffer
_, err := io.Copy(&buffer, brotli.NewReader(stream)) _, err := io.Copy(&buffer, brotli.NewReader(stream))
if err != nil { if err != nil {
Logger.Errorf("Failed to decode http responses with brolti encoding -> %v", err) logger.Errorf("Failed to decode http responses with brolti encoding -> %v", err)
return nil, err return nil, err
} }
return buffer.Bytes(), nil return buffer.Bytes(), nil
@ -27,14 +27,14 @@ func broltiDecode(stream io.Reader) ([]byte, error) {
func gzipDecode(stream io.Reader) ([]byte, error) { func gzipDecode(stream io.Reader) ([]byte, error) {
reader, err := gzip.NewReader(stream) reader, err := gzip.NewReader(stream)
if err != nil { if err != nil {
Logger.Errorf("Failed to decode http responses with gzip encoding -> %v", err) logger.Errorf("Failed to decode http responses with gzip encoding -> %v", err)
return nil, err return nil, err
} }
var buffer bytes.Buffer var buffer bytes.Buffer
_, err = io.Copy(&buffer, reader) _, err = io.Copy(&buffer, reader)
if err != nil { if err != nil {
Logger.Errorf("Failed to handle gzip reader -> %v", err) logger.Errorf("Failed to handle gzip reader -> %v", err)
return nil, err return nil, err
} }
return buffer.Bytes(), nil return buffer.Bytes(), nil
@ -45,7 +45,7 @@ func deflateDecode(stream io.Reader) ([]byte, error) {
var buffer bytes.Buffer var buffer bytes.Buffer
_, err := io.Copy(&buffer, flate.NewReader(stream)) _, err := io.Copy(&buffer, flate.NewReader(stream))
if err != nil { if err != nil {
Logger.Errorf("Failed to decode http responses with deflate encoding -> %v", err) logger.Errorf("Failed to decode http responses with deflate encoding -> %v", err)
return nil, err return nil, err
} }
return buffer.Bytes(), nil return buffer.Bytes(), nil
@ -56,7 +56,7 @@ func nonDecode(stream io.Reader) ([]byte, error) {
var buffer bytes.Buffer var buffer bytes.Buffer
_, err := io.Copy(&buffer, stream) _, err := io.Copy(&buffer, stream)
if err != nil { if err != nil {
Logger.Errorf("Failed to read http responses -> %v", err) logger.Errorf("Failed to read http responses -> %v", err)
return nil, err return nil, err
} }
return buffer.Bytes(), nil return buffer.Bytes(), nil
@ -65,13 +65,13 @@ func nonDecode(stream io.Reader) ([]byte, error) {
// createClient build http client based on http or socks proxy url. // createClient build http client based on http or socks proxy url.
func createClient(remoteUrl string, proxyUrl string) (http.Client, error) { func createClient(remoteUrl string, proxyUrl string) (http.Client, error) {
if proxyUrl == "" { if proxyUrl == "" {
Logger.Infof("Downloading `%s` without proxy", remoteUrl) logger.Infof("Downloading `%s` without proxy", remoteUrl)
return http.Client{}, nil return http.Client{}, nil
} }
Logger.Infof("Downloading `%s` via `%s`", remoteUrl, proxyUrl) logger.Infof("Downloading `%s` via `%s`", remoteUrl, proxyUrl)
proxy, err := url.Parse(proxyUrl) proxy, err := url.Parse(proxyUrl)
if err != nil { if err != nil {
Logger.Errorf("Invalid proxy url `%s` -> %v", proxyUrl, err) logger.Errorf("Invalid proxy url `%s` -> %v", proxyUrl, err)
return http.Client{}, err return http.Client{}, err
} }
return http.Client{ return http.Client{
@ -81,8 +81,8 @@ func createClient(remoteUrl string, proxyUrl string) (http.Client, error) {
}, nil }, nil
} }
// Download obtains resource file from the remote server and supports proxy. // download obtains resource file from the remote server and supports proxy.
func Download(url string, proxy string) ([]byte, error) { func download(url string, proxy string) ([]byte, error) {
client, err := createClient(url, proxy) client, err := createClient(url, proxy)
if err != nil { if err != nil {
return nil, err return nil, err
@ -90,28 +90,28 @@ func Download(url string, proxy string) ([]byte, error) {
req, err := http.NewRequest("GET", url, nil) req, err := http.NewRequest("GET", url, nil)
if err != nil { if err != nil {
Logger.Errorf("Failed to create http request -> %v", err) logger.Errorf("Failed to create http request -> %v", err)
return nil, err return nil, err
} }
req.Header.Set(headers.AcceptEncoding, "gzip, deflate, br") req.Header.Set(headers.AcceptEncoding, "gzip, deflate, br")
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
Logger.Errorf("Failed to execute http request -> %v", err) logger.Errorf("Failed to execute http request -> %v", err)
return nil, err return nil, err
} }
defer resp.Body.Close() defer resp.Body.Close()
Logger.Debugf("Remote data downloaded successfully") logger.Debugf("Remote data downloaded successfully")
var content []byte var content []byte
switch resp.Header.Get(headers.ContentEncoding) { switch resp.Header.Get(headers.ContentEncoding) {
case "br": case "br":
Logger.Debugf("Downloaded content using brolti encoding") logger.Debugf("Downloaded content using brolti encoding")
content, err = broltiDecode(resp.Body) content, err = broltiDecode(resp.Body)
case "gzip": case "gzip":
Logger.Debugf("Downloaded content using gzip encoding") logger.Debugf("Downloaded content using gzip encoding")
content, err = gzipDecode(resp.Body) content, err = gzipDecode(resp.Body)
case "deflate": case "deflate":
Logger.Debugf("Downloaded content using deflate encoding") logger.Debugf("Downloaded content using deflate encoding")
content, err = deflateDecode(resp.Body) content, err = deflateDecode(resp.Body)
default: default:
content, err = nonDecode(resp.Body) content, err = nonDecode(resp.Body)
@ -119,6 +119,6 @@ func Download(url string, proxy string) ([]byte, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
Logger.Debugf("Download `%s` successfully -> %d bytes", url, len(content)) logger.Debugf("Download `%s` successfully -> %d bytes", url, len(content))
return content, nil return content, nil
} }

28
next/main.go

@ -1,31 +1,7 @@
package main package main
import ( import "XProxy/next/assets"
"XProxy/next/assets"
"XProxy/next/logger"
)
const gzSample = "/root/XProxy/LICENSE.gz"
const xzSample = "/root/XProxy/LICENSE.xz"
const bz2Sample = "/root/XProxy/LICENSE.bz2"
func main() { func main() {
assets.Demo()
//raw, _ := os.ReadFile(gzSample)
//raw, _ := os.ReadFile(bz2Sample)
//raw, _ := os.ReadFile(xzSample)
//Logger.Debugf("data len -> %d", len(raw))
//ret, err := assets.Extract(raw)
//if err != nil {
// Logger.Debugf("extract error -> %v", err)
//}
//Logger.Debugf("extract ok -> len = %d", len(ret))
//os.WriteFile("demo.data", ret, 0777)
//raw, _ := assets.Download("https://cdn.dnomd343.top/v2ray-rules-dat/geosite.dat", "")
//raw, _ := assets.Download("https://cdn.dnomd343.top/v2ray-rules-dat/geosite.dat", "socks5://192.168.2.2:1084")
raw, _ := assets.Download("https://cdn.dnomd343.top/v2ray-rules-dat/geosite.dat.xz", "")
ret, _ := assets.Extract(raw)
logger.Debugf("content size -> %d", len(ret))
} }

Loading…
Cancel
Save