From c436b0375d0c9ca4a50d261513d73af5ce0baac8 Mon Sep 17 00:00:00 2001 From: Dnomd343 Date: Sun, 3 Sep 2023 14:12:37 +0800 Subject: [PATCH] feat: assets extract module --- go.mod | 2 + go.sum | 4 ++ next/assets/demo.go | 86 --------------------------------- next/assets/extract.go | 106 +++++++++++++++++++++++++++++++++++++++++ next/main.go | 20 +++++++- 5 files changed, 130 insertions(+), 88 deletions(-) delete mode 100644 next/assets/demo.go create mode 100644 next/assets/extract.go diff --git a/go.mod b/go.mod index dd03c6e..5782fe3 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,9 @@ require ( ) require ( + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/stretchr/testify v1.8.4 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.8.0 // indirect golang.org/x/sys v0.10.0 // indirect ) diff --git a/go.sum b/go.sum index 3268f8a..88a55d6 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno= github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= @@ -29,6 +31,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/next/assets/demo.go b/next/assets/demo.go deleted file mode 100644 index a72a416..0000000 --- a/next/assets/demo.go +++ /dev/null @@ -1,86 +0,0 @@ -package assets - -import ( - . "XProxy/next/logger" - "bytes" - "compress/bzip2" - "compress/gzip" - "fmt" - "github.com/ulikunitz/xz" - "io" - "os" -) - -const gzSample = "/root/XProxy/LICENSE.gz" -const xzSample = "/root/XProxy/LICENSE.xz" -const bz2Sample = "/root/XProxy/LICENSE.bz2" - -func gzipExtract(content io.Reader) ([]byte, error) { - Logger.Debugf("Start extracting gzip archive") - reader, err := gzip.NewReader(content) - if err != nil { - Logger.Errorf("Failed to extract gzip archive -> %v", err) - return nil, err - } - defer reader.Close() - - var buffer bytes.Buffer - size, err := io.Copy(&buffer, reader) - if err != nil { - Logger.Errorf("Failed to handle gzip archive -> %v", err) - return nil, err - } - Logger.Debugf("Successfully extracted gzip archive -> %d bytes", size) - return buffer.Bytes(), nil -} - -func bzip2Extract(content io.Reader) ([]byte, error) { - Logger.Debugf("Start extracting bzip2 archive") - reader := bzip2.NewReader(content) - - var buffer bytes.Buffer - size, err := io.Copy(&buffer, reader) - if err != nil { - Logger.Errorf("Failed to extract bzip2 archive -> %v", err) - return nil, err - } - Logger.Debugf("Successfully extracted bzip2 archive -> %d bytes", size) - return buffer.Bytes(), nil -} - -func xzExtract(content io.Reader) ([]byte, error) { - Logger.Debugf("Start extracting xz archive") - reader, err := xz.NewReader(content) - if err != nil { - Logger.Errorf("Failed to extract xz archive -> %v", err) - return nil, err - } - - var buffer bytes.Buffer - size, err := io.Copy(&buffer, reader) - if err != nil { - Logger.Errorf("Failed to handle xz archive -> %v", err) - return nil, err - } - Logger.Debugf("Successfully extracted xz archive -> %d bytes", size) - return buffer.Bytes(), nil -} - -func Demo() { - Logger.Infof("Assets demo begin") - - //fp, err := os.Open(gzSample) - //fp, err := os.Open(bz2Sample) - fp, err := os.Open(xzSample) - if err != nil { - fmt.Println("open failed") - } - defer fp.Close() - - //gzipExtract(fp) - //bzip2Extract(fp) - xzExtract(fp) - - //fp.Name() - -} diff --git a/next/assets/extract.go b/next/assets/extract.go new file mode 100644 index 0000000..1dd2933 --- /dev/null +++ b/next/assets/extract.go @@ -0,0 +1,106 @@ +package assets + +import ( + . "XProxy/next/logger" + "bytes" + "compress/bzip2" + "github.com/gabriel-vasile/mimetype" + "github.com/klauspost/compress/gzip" + "github.com/ulikunitz/xz" + "io" +) + +const ( + notArchive = iota + gzipArchive + bzip2Archive + xzArchive +) + +// gzipExtract use to extract independent gzip archive data. +func gzipExtract(data []byte) ([]byte, error) { + Logger.Debugf("Start extracting gzip archive -> %d bytes", len(data)) + reader, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + Logger.Errorf("Failed to extract gzip archive -> %v", err) + return nil, err + } + defer reader.Close() + + var buffer bytes.Buffer + size, err := io.Copy(&buffer, reader) + if err != nil { + Logger.Errorf("Failed to handle gzip archive -> %v", err) + return nil, err + } + Logger.Debugf("Extracted gzip archive successfully -> %d bytes", size) + return buffer.Bytes(), nil +} + +// bzip2Extract use to extract independent bzip2 archive data. +func bzip2Extract(data []byte) ([]byte, error) { + Logger.Debugf("Start extracting bzip2 archive -> %d bytes", len(data)) + reader := bzip2.NewReader(bytes.NewReader(data)) + + var buffer bytes.Buffer + size, err := io.Copy(&buffer, reader) + if err != nil { + Logger.Errorf("Failed to extract bzip2 archive -> %v", err) + return nil, err + } + Logger.Debugf("Extracted bzip2 archive successfully -> %d bytes", size) + return buffer.Bytes(), nil +} + +// xzExtract use to extract independent xz archive data. +func xzExtract(data []byte) ([]byte, error) { + Logger.Debugf("Start extracting xz archive -> %d bytes", len(data)) + reader, err := xz.NewReader(bytes.NewReader(data)) + if err != nil { + Logger.Errorf("Failed to extract xz archive -> %v", err) + return nil, err + } + + var buffer bytes.Buffer + size, err := io.Copy(&buffer, reader) + if err != nil { + Logger.Errorf("Failed to handle xz archive -> %v", err) + return nil, err + } + Logger.Debugf("Extracted xz archive successfully -> %d bytes", size) + return buffer.Bytes(), nil +} + +// archiveType use to determine the type of archive file. +func archiveType(data []byte) uint { + mime := mimetype.Detect(data) + switch mime.String() { + case "application/gzip": + Logger.Debugf("Data detected as gzip format") + return gzipArchive + case "application/x-bzip2": + Logger.Debugf("Data detected as bzip2 format") + return bzip2Archive + case "application/x-xz": + Logger.Debugf("Data detected as xz format") + return xzArchive + default: + Logger.Debugf("Data detected as non-archive format -> %s", mime) + return notArchive + } +} + +// Extract will try to extract the data as a compressed format, and will +// return the original data if it cannot be determined. +func Extract(data []byte) ([]byte, error) { + switch archiveType(data) { + case gzipArchive: + return gzipExtract(data) + case bzip2Archive: + return bzip2Extract(data) + case xzArchive: + return xzExtract(data) + default: + return data, nil + } +} diff --git a/next/main.go b/next/main.go index bddddfc..85439be 100644 --- a/next/main.go +++ b/next/main.go @@ -2,9 +2,25 @@ package main import ( "XProxy/next/assets" - _ "XProxy/next/logger" + . "XProxy/next/logger" + "os" ) +const gzSample = "/root/XProxy/LICENSE.gz" +const xzSample = "/root/XProxy/LICENSE.xz" +const bz2Sample = "/root/XProxy/LICENSE.bz2" + func main() { - assets.Demo() + + //raw, _ := os.ReadFile(gzSample) + //raw, _ := os.ReadFile(bz2Sample) + raw, _ := os.ReadFile(xzSample) + Logger.Debugf("data len -> %d", len(raw)) + ret, err := assets.Extract(raw) + if err != nil { + Logger.Debugf("extract error -> %v", err) + } + Logger.Debugf("extract ok -> len = %d", len(ret)) + //os.WriteFile("demo.data", ret, 0777) + }