Browse Source

feat: assets extract module

feature/assets
Dnomd343 1 year ago
parent
commit
c436b0375d
  1. 2
      go.mod
  2. 4
      go.sum
  3. 86
      next/assets/demo.go
  4. 106
      next/assets/extract.go
  5. 20
      next/main.go

2
go.mod

@ -15,7 +15,9 @@ require (
) )
require ( require (
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/stretchr/testify v1.8.4 // indirect github.com/stretchr/testify v1.8.4 // indirect
go.uber.org/multierr v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/sys v0.10.0 // indirect golang.org/x/sys v0.10.0 // indirect
) )

4
go.sum

@ -7,6 +7,8 @@ github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno= github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno=
github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw= github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw=
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
@ -29,6 +31,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

86
next/assets/demo.go

@ -1,86 +0,0 @@
package assets
import (
. "XProxy/next/logger"
"bytes"
"compress/bzip2"
"compress/gzip"
"fmt"
"github.com/ulikunitz/xz"
"io"
"os"
)
const gzSample = "/root/XProxy/LICENSE.gz"
const xzSample = "/root/XProxy/LICENSE.xz"
const bz2Sample = "/root/XProxy/LICENSE.bz2"
func gzipExtract(content io.Reader) ([]byte, error) {
Logger.Debugf("Start extracting gzip archive")
reader, err := gzip.NewReader(content)
if err != nil {
Logger.Errorf("Failed to extract gzip archive -> %v", err)
return nil, err
}
defer reader.Close()
var buffer bytes.Buffer
size, err := io.Copy(&buffer, reader)
if err != nil {
Logger.Errorf("Failed to handle gzip archive -> %v", err)
return nil, err
}
Logger.Debugf("Successfully extracted gzip archive -> %d bytes", size)
return buffer.Bytes(), nil
}
func bzip2Extract(content io.Reader) ([]byte, error) {
Logger.Debugf("Start extracting bzip2 archive")
reader := bzip2.NewReader(content)
var buffer bytes.Buffer
size, err := io.Copy(&buffer, reader)
if err != nil {
Logger.Errorf("Failed to extract bzip2 archive -> %v", err)
return nil, err
}
Logger.Debugf("Successfully extracted bzip2 archive -> %d bytes", size)
return buffer.Bytes(), nil
}
func xzExtract(content io.Reader) ([]byte, error) {
Logger.Debugf("Start extracting xz archive")
reader, err := xz.NewReader(content)
if err != nil {
Logger.Errorf("Failed to extract xz archive -> %v", err)
return nil, err
}
var buffer bytes.Buffer
size, err := io.Copy(&buffer, reader)
if err != nil {
Logger.Errorf("Failed to handle xz archive -> %v", err)
return nil, err
}
Logger.Debugf("Successfully extracted xz archive -> %d bytes", size)
return buffer.Bytes(), nil
}
func Demo() {
Logger.Infof("Assets demo begin")
//fp, err := os.Open(gzSample)
//fp, err := os.Open(bz2Sample)
fp, err := os.Open(xzSample)
if err != nil {
fmt.Println("open failed")
}
defer fp.Close()
//gzipExtract(fp)
//bzip2Extract(fp)
xzExtract(fp)
//fp.Name()
}

106
next/assets/extract.go

@ -0,0 +1,106 @@
package assets
import (
. "XProxy/next/logger"
"bytes"
"compress/bzip2"
"github.com/gabriel-vasile/mimetype"
"github.com/klauspost/compress/gzip"
"github.com/ulikunitz/xz"
"io"
)
const (
notArchive = iota
gzipArchive
bzip2Archive
xzArchive
)
// gzipExtract use to extract independent gzip archive data.
func gzipExtract(data []byte) ([]byte, error) {
Logger.Debugf("Start extracting gzip archive -> %d bytes", len(data))
reader, err := gzip.NewReader(bytes.NewReader(data))
if err != nil {
Logger.Errorf("Failed to extract gzip archive -> %v", err)
return nil, err
}
defer reader.Close()
var buffer bytes.Buffer
size, err := io.Copy(&buffer, reader)
if err != nil {
Logger.Errorf("Failed to handle gzip archive -> %v", err)
return nil, err
}
Logger.Debugf("Extracted gzip archive successfully -> %d bytes", size)
return buffer.Bytes(), nil
}
// bzip2Extract use to extract independent bzip2 archive data.
func bzip2Extract(data []byte) ([]byte, error) {
Logger.Debugf("Start extracting bzip2 archive -> %d bytes", len(data))
reader := bzip2.NewReader(bytes.NewReader(data))
var buffer bytes.Buffer
size, err := io.Copy(&buffer, reader)
if err != nil {
Logger.Errorf("Failed to extract bzip2 archive -> %v", err)
return nil, err
}
Logger.Debugf("Extracted bzip2 archive successfully -> %d bytes", size)
return buffer.Bytes(), nil
}
// xzExtract use to extract independent xz archive data.
func xzExtract(data []byte) ([]byte, error) {
Logger.Debugf("Start extracting xz archive -> %d bytes", len(data))
reader, err := xz.NewReader(bytes.NewReader(data))
if err != nil {
Logger.Errorf("Failed to extract xz archive -> %v", err)
return nil, err
}
var buffer bytes.Buffer
size, err := io.Copy(&buffer, reader)
if err != nil {
Logger.Errorf("Failed to handle xz archive -> %v", err)
return nil, err
}
Logger.Debugf("Extracted xz archive successfully -> %d bytes", size)
return buffer.Bytes(), nil
}
// archiveType use to determine the type of archive file.
func archiveType(data []byte) uint {
mime := mimetype.Detect(data)
switch mime.String() {
case "application/gzip":
Logger.Debugf("Data detected as gzip format")
return gzipArchive
case "application/x-bzip2":
Logger.Debugf("Data detected as bzip2 format")
return bzip2Archive
case "application/x-xz":
Logger.Debugf("Data detected as xz format")
return xzArchive
default:
Logger.Debugf("Data detected as non-archive format -> %s", mime)
return notArchive
}
}
// Extract will try to extract the data as a compressed format, and will
// return the original data if it cannot be determined.
func Extract(data []byte) ([]byte, error) {
switch archiveType(data) {
case gzipArchive:
return gzipExtract(data)
case bzip2Archive:
return bzip2Extract(data)
case xzArchive:
return xzExtract(data)
default:
return data, nil
}
}

20
next/main.go

@ -2,9 +2,25 @@ package main
import ( import (
"XProxy/next/assets" "XProxy/next/assets"
_ "XProxy/next/logger" . "XProxy/next/logger"
"os"
) )
const gzSample = "/root/XProxy/LICENSE.gz"
const xzSample = "/root/XProxy/LICENSE.xz"
const bz2Sample = "/root/XProxy/LICENSE.bz2"
func main() { func main() {
assets.Demo()
//raw, _ := os.ReadFile(gzSample)
//raw, _ := os.ReadFile(bz2Sample)
raw, _ := os.ReadFile(xzSample)
Logger.Debugf("data len -> %d", len(raw))
ret, err := assets.Extract(raw)
if err != nil {
Logger.Debugf("extract error -> %v", err)
}
Logger.Debugf("extract ok -> len = %d", len(ret))
//os.WriteFile("demo.data", ret, 0777)
} }

Loading…
Cancel
Save