feat: assets extract module

go.mod

@@ -15,7 +15,9 @@ require (
 )
 
 require (
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
 	golang.org/x/sys v0.10.0 // indirect
 )

go.sum

@@ -7,6 +7,8 @@ github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
+github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
 github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno=
 github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw=
 github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
@@ -29,6 +31,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
 go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
+golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
+golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

next/assets/demo.go

@@ -1,86 +0,0 @@
-package assets
-
-import (
-	. "XProxy/next/logger"
-	"bytes"
-	"compress/bzip2"
-	"compress/gzip"
-	"fmt"
-	"github.com/ulikunitz/xz"
-	"io"
-	"os"
-)
-
-const gzSample = "/root/XProxy/LICENSE.gz"
-const xzSample = "/root/XProxy/LICENSE.xz"
-const bz2Sample = "/root/XProxy/LICENSE.bz2"
-
-func gzipExtract(content io.Reader) ([]byte, error) {
-	Logger.Debugf("Start extracting gzip archive")
-	reader, err := gzip.NewReader(content)
-	if err != nil {
-		Logger.Errorf("Failed to extract gzip archive -> %v", err)
-		return nil, err
-	}
-	defer reader.Close()
-
-	var buffer bytes.Buffer
-	size, err := io.Copy(&buffer, reader)
-	if err != nil {
-		Logger.Errorf("Failed to handle gzip archive -> %v", err)
-		return nil, err
-	}
-	Logger.Debugf("Successfully extracted gzip archive -> %d bytes", size)
-	return buffer.Bytes(), nil
-}
-
-func bzip2Extract(content io.Reader) ([]byte, error) {
-	Logger.Debugf("Start extracting bzip2 archive")
-	reader := bzip2.NewReader(content)
-
-	var buffer bytes.Buffer
-	size, err := io.Copy(&buffer, reader)
-	if err != nil {
-		Logger.Errorf("Failed to extract bzip2 archive -> %v", err)
-		return nil, err
-	}
-	Logger.Debugf("Successfully extracted bzip2 archive -> %d bytes", size)
-	return buffer.Bytes(), nil
-}
-
-func xzExtract(content io.Reader) ([]byte, error) {
-	Logger.Debugf("Start extracting xz archive")
-	reader, err := xz.NewReader(content)
-	if err != nil {
-		Logger.Errorf("Failed to extract xz archive -> %v", err)
-		return nil, err
-	}
-
-	var buffer bytes.Buffer
-	size, err := io.Copy(&buffer, reader)
-	if err != nil {
-		Logger.Errorf("Failed to handle xz archive -> %v", err)
-		return nil, err
-	}
-	Logger.Debugf("Successfully extracted xz archive -> %d bytes", size)
-	return buffer.Bytes(), nil
-}
-
-func Demo() {
-	Logger.Infof("Assets demo begin")
-
-	//fp, err := os.Open(gzSample)
-	//fp, err := os.Open(bz2Sample)
-	fp, err := os.Open(xzSample)
-	if err != nil {
-		fmt.Println("open failed")
-	}
-	defer fp.Close()
-
-	//gzipExtract(fp)
-	//bzip2Extract(fp)
-	xzExtract(fp)
-
-	//fp.Name()
-
-}

next/assets/extract.go

@@ -0,0 +1,106 @@
+package assets
+
+import (
+	. "XProxy/next/logger"
+	"bytes"
+	"compress/bzip2"
+	"github.com/gabriel-vasile/mimetype"
+	"github.com/klauspost/compress/gzip"
+	"github.com/ulikunitz/xz"
+	"io"
+)
+
+const (
+	notArchive = iota
+	gzipArchive
+	bzip2Archive
+	xzArchive
+)
+
+// gzipExtract use to extract independent gzip archive data.
+func gzipExtract(data []byte) ([]byte, error) {
+	Logger.Debugf("Start extracting gzip archive -> %d bytes", len(data))
+	reader, err := gzip.NewReader(bytes.NewReader(data))
+	if err != nil {
+		Logger.Errorf("Failed to extract gzip archive -> %v", err)
+		return nil, err
+	}
+	defer reader.Close()
+
+	var buffer bytes.Buffer
+	size, err := io.Copy(&buffer, reader)
+	if err != nil {
+		Logger.Errorf("Failed to handle gzip archive -> %v", err)
+		return nil, err
+	}
+	Logger.Debugf("Extracted gzip archive successfully -> %d bytes", size)
+	return buffer.Bytes(), nil
+}
+
+// bzip2Extract use to extract independent bzip2 archive data.
+func bzip2Extract(data []byte) ([]byte, error) {
+	Logger.Debugf("Start extracting bzip2 archive -> %d bytes", len(data))
+	reader := bzip2.NewReader(bytes.NewReader(data))
+
+	var buffer bytes.Buffer
+	size, err := io.Copy(&buffer, reader)
+	if err != nil {
+		Logger.Errorf("Failed to extract bzip2 archive -> %v", err)
+		return nil, err
+	}
+	Logger.Debugf("Extracted bzip2 archive successfully -> %d bytes", size)
+	return buffer.Bytes(), nil
+}
+
+// xzExtract use to extract independent xz archive data.
+func xzExtract(data []byte) ([]byte, error) {
+	Logger.Debugf("Start extracting xz archive -> %d bytes", len(data))
+	reader, err := xz.NewReader(bytes.NewReader(data))
+	if err != nil {
+		Logger.Errorf("Failed to extract xz archive -> %v", err)
+		return nil, err
+	}
+
+	var buffer bytes.Buffer
+	size, err := io.Copy(&buffer, reader)
+	if err != nil {
+		Logger.Errorf("Failed to handle xz archive -> %v", err)
+		return nil, err
+	}
+	Logger.Debugf("Extracted xz archive successfully -> %d bytes", size)
+	return buffer.Bytes(), nil
+}
+
+// archiveType use to determine the type of archive file.
+func archiveType(data []byte) uint {
+	mime := mimetype.Detect(data)
+	switch mime.String() {
+	case "application/gzip":
+		Logger.Debugf("Data detected as gzip format")
+		return gzipArchive
+	case "application/x-bzip2":
+		Logger.Debugf("Data detected as bzip2 format")
+		return bzip2Archive
+	case "application/x-xz":
+		Logger.Debugf("Data detected as xz format")
+		return xzArchive
+	default:
+		Logger.Debugf("Data detected as non-archive format -> %s", mime)
+		return notArchive
+	}
+}
+
+// Extract will try to extract the data as a compressed format, and will
+// return the original data if it cannot be determined.
+func Extract(data []byte) ([]byte, error) {
+	switch archiveType(data) {
+	case gzipArchive:
+		return gzipExtract(data)
+	case bzip2Archive:
+		return bzip2Extract(data)
+	case xzArchive:
+		return xzExtract(data)
+	default:
+		return data, nil
+	}
+}

next/main.go

@@ -2,9 +2,25 @@ package main
 
 import (
 	"XProxy/next/assets"
-	_ "XProxy/next/logger"
+	. "XProxy/next/logger"
+	"os"
 )
 
+const gzSample = "/root/XProxy/LICENSE.gz"
+const xzSample = "/root/XProxy/LICENSE.xz"
+const bz2Sample = "/root/XProxy/LICENSE.bz2"
+
 func main() {
-	assets.Demo()
+
+	//raw, _ := os.ReadFile(gzSample)
+	//raw, _ := os.ReadFile(bz2Sample)
+	raw, _ := os.ReadFile(xzSample)
+	Logger.Debugf("data len -> %d", len(raw))
+	ret, err := assets.Extract(raw)
+	if err != nil {
+		Logger.Debugf("extract error -> %v", err)
+	}
+	Logger.Debugf("extract ok -> len = %d", len(ret))
+	//os.WriteFile("demo.data", ret, 0777)
+
 }