add open, write
This commit is contained in:
parent
9edb886a15
commit
17896c3cc4
GPG key ID:
E46B5FEA35B22FF9
6 changed files with 202 additions and 3 deletions
15
Taskfile.yml
Normal file
15
Taskfile.yml
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
version: 3
|
||||
|
||||
tasks:
|
||||
minio-run:
|
||||
desc: Start MinIO for testing
|
||||
cmds:
|
||||
- rm -rf ./test/minio && mkdir -p ./test/minio
|
||||
- docker run -p 9000:9000 -p 9001:9001 --rm --name storage-testing -v ./test/minio:/data -e "MINIO_ROOT_USER=root" -e "MINIO_ROOT_PASSWORD=password123" quay.io/minio/minio server /data --console-address ":9001"
|
||||
- rm -rf ./test/minio
|
||||
test:
|
||||
desc: Run library tests
|
||||
cmds:
|
||||
- rm -rf ./test/local
|
||||
- go test ./... -v
|
||||
- rm -rf ./test/local
|
||||
3
file.go
3
file.go
|
|
@ -2,7 +2,6 @@ package fsys
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"io"
|
||||
"mime"
|
||||
|
|
@ -41,7 +40,7 @@ func (f *File) Write(p []byte) (n int, err error) {
|
|||
mimeType = "application/octet-stream"
|
||||
}
|
||||
|
||||
_, err = f.storage.s3Client.PutObject(context.Background(), f.storage.config.S3BucketName, f.Name, bytes.NewBuffer(fileContent), int64(len(fileContent)), minio.PutObjectOptions{
|
||||
_, err = f.storage.s3Client.PutObject(f.storage.ctx, f.storage.config.S3BucketName, f.Name, bytes.NewBuffer(fileContent), int64(len(fileContent)), minio.PutObjectOptions{
|
||||
ContentType: mimeType,
|
||||
})
|
||||
if err != nil {
|
||||
|
|
|
|||
7
go.mod
7
go.mod
|
|
@ -2,7 +2,10 @@ module egtyl.xyz/omnibill/fsys
|
|||
|
||||
go 1.23.2
|
||||
|
||||
require github.com/minio/minio-go/v7 v7.0.80
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/goccy/go-json v0.10.3 // indirect
|
||||
|
|
@ -10,10 +13,12 @@ require (
|
|||
github.com/klauspost/compress v1.17.11 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/minio/minio-go/v7 v7.0.80 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/stretchr/testify v1.9.0 // indirect
|
||||
golang.org/x/crypto v0.28.0 // indirect
|
||||
golang.org/x/net v0.30.0 // indirect
|
||||
golang.org/x/sys v0.26.0 // indirect
|
||||
golang.org/x/text v0.19.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
|
|
|||
9
go.sum
9
go.sum
|
|
@ -1,3 +1,5 @@
|
|||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
|
|
@ -15,8 +17,12 @@ github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
|||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.80 h1:2mdUHXEykRdY/BigLt3Iuu1otL0JTogT0Nmltg0wujk=
|
||||
github.com/minio/minio-go/v7 v7.0.80/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
|
||||
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
|
||||
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
|
||||
|
|
@ -26,3 +32,6 @@ golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
|
|||
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
|
||||
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
|
|
|||
109
storage.go
109
storage.go
|
|
@ -1,9 +1,15 @@
|
|||
package fsys
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -15,6 +21,7 @@ var (
|
|||
type Storage struct {
|
||||
config *Config
|
||||
s3Client *minio.Client
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
|
|
@ -35,6 +42,7 @@ type Config struct {
|
|||
func New(config *Config) (*Storage, error) {
|
||||
newStorage := new(Storage)
|
||||
newStorage.config = config
|
||||
newStorage.ctx = context.Background()
|
||||
|
||||
switch config.Type {
|
||||
case "minio", "s3":
|
||||
|
|
@ -45,11 +53,112 @@ func New(config *Config) (*Storage, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bucketExists, err := s3Client.BucketExists(newStorage.ctx, config.S3BucketName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !bucketExists {
|
||||
if err := s3Client.MakeBucket(newStorage.ctx, config.S3BucketName, minio.MakeBucketOptions{}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
newStorage.s3Client = s3Client
|
||||
case "local":
|
||||
if err := os.MkdirAll(config.Path, 0740); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
default:
|
||||
return nil, ErrInvalidStorageType
|
||||
}
|
||||
|
||||
return newStorage, nil
|
||||
}
|
||||
|
||||
func (s *Storage) Open(name string) (*File, error) {
|
||||
returnFile := new(File)
|
||||
returnFile.storage = s
|
||||
|
||||
if s.s3Client != nil {
|
||||
object, err := s.s3Client.GetObject(s.ctx, s.config.S3BucketName, name, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
errResp := minio.ToErrorResponse(err)
|
||||
if errResp.StatusCode == http.StatusNotFound {
|
||||
return nil, ErrFileNotFound
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
objectStat, err := object.Stat()
|
||||
if err != nil {
|
||||
errResp := minio.ToErrorResponse(err)
|
||||
if errResp.StatusCode == http.StatusNotFound {
|
||||
return nil, ErrFileNotFound
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
returnFile.Name = objectStat.Key
|
||||
returnFile.file = object
|
||||
} else {
|
||||
if _, err := os.Stat(filepath.Join(s.config.Path, name)); err != nil && os.IsNotExist(err) {
|
||||
return nil, ErrFileNotFound
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
file, err := os.Open(filepath.Join(s.config.Path, name))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
returnFile.file = file
|
||||
|
||||
cutStr, _ := strings.CutPrefix(file.Name(), filepath.Clean(s.config.Path)+"/")
|
||||
|
||||
returnFile.Name = cutStr
|
||||
}
|
||||
|
||||
return returnFile, nil
|
||||
}
|
||||
|
||||
func (s *Storage) Read(name string) ([]byte, error) {
|
||||
if s.s3Client != nil {
|
||||
object, err := s.s3Client.GetObject(s.ctx, s.config.S3BucketName, name, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
errResp := minio.ToErrorResponse(err)
|
||||
if errResp.StatusCode == http.StatusNotFound {
|
||||
return nil, ErrFileNotFound
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var objectBytes []byte
|
||||
if _, err := object.Read(objectBytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return objectBytes, nil
|
||||
} else {
|
||||
return os.ReadFile(filepath.Join(s.config.Path, name))
|
||||
}
|
||||
}
|
||||
|
||||
type WriteOptions struct {
|
||||
S3Tags map[string]string
|
||||
}
|
||||
|
||||
func (s *Storage) Write(name string, data []byte, opts WriteOptions) error {
|
||||
if s.s3Client != nil {
|
||||
if _, err := s.s3Client.PutObject(s.ctx, s.config.S3BucketName, name, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{
|
||||
UserTags: opts.S3Tags,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
} else {
|
||||
return os.WriteFile(filepath.Join(s.config.Path, name), data, 0640)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
62
storage_test.go
Normal file
62
storage_test.go
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
package fsys
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStorageLocal(t *testing.T) {
|
||||
|
||||
stor, err := New(&Config{
|
||||
Type: "local",
|
||||
Path: "./test/local",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
t.Log("== Write ==")
|
||||
err = stor.Write("test.txt", []byte("hi"), WriteOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
t.Log("== Open ==")
|
||||
file, err := stor.Open("test.txt")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "test.txt", file.Name)
|
||||
|
||||
fileContent, err := io.ReadAll(file)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, []byte("hi"), fileContent)
|
||||
|
||||
err = file.Close()
|
||||
assert.NoError(t, err)
|
||||
|
||||
}
|
||||
|
||||
func TestStorageS3(t *testing.T) {
|
||||
|
||||
stor, err := New(&Config{
|
||||
Type: "s3",
|
||||
S3BucketName: "test",
|
||||
S3AccessID: "root",
|
||||
S3AccessKey: "password123",
|
||||
S3Endpoint: "127.0.0.1:9000",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
t.Log("== Write ==")
|
||||
err = stor.Write("test.txt", []byte("hi"), WriteOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
t.Log("== Open ==")
|
||||
file, err := stor.Open("test.txt")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "test.txt", file.Name)
|
||||
|
||||
fileContent, err := io.ReadAll(file)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, []byte("hi"), fileContent)
|
||||
|
||||
err = file.Close()
|
||||
assert.NoError(t, err)
|
||||
|
||||
}
|
||||
Loading…
Reference in a new issue