diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 0000000..690df0f --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,15 @@ +version: 3 + +tasks: + minio-run: + desc: Start MinIO for testing + cmds: + - rm -rf ./test/minio && mkdir -p ./test/minio + - docker run -p 9000:9000 -p 9001:9001 --rm --name storage-testing -v ./test/minio:/data -e "MINIO_ROOT_USER=root" -e "MINIO_ROOT_PASSWORD=password123" quay.io/minio/minio server /data --console-address ":9001" + - rm -rf ./test/minio + test: + desc: Run library tests + cmds: + - rm -rf ./test/local + - go test ./... -v + - rm -rf ./test/local \ No newline at end of file diff --git a/file.go b/file.go index 9f6850b..d0bf357 100644 --- a/file.go +++ b/file.go @@ -2,7 +2,6 @@ package fsys import ( "bytes" - "context" "github.com/minio/minio-go/v7" "io" "mime" @@ -41,7 +40,7 @@ func (f *File) Write(p []byte) (n int, err error) { mimeType = "application/octet-stream" } - _, err = f.storage.s3Client.PutObject(context.Background(), f.storage.config.S3BucketName, f.Name, bytes.NewBuffer(fileContent), int64(len(fileContent)), minio.PutObjectOptions{ + _, err = f.storage.s3Client.PutObject(f.storage.ctx, f.storage.config.S3BucketName, f.Name, bytes.NewBuffer(fileContent), int64(len(fileContent)), minio.PutObjectOptions{ ContentType: mimeType, }) if err != nil { diff --git a/go.mod b/go.mod index bb9e1a1..6032d35 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,10 @@ module egtyl.xyz/omnibill/fsys go 1.23.2 +require github.com/minio/minio-go/v7 v7.0.80 + require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/goccy/go-json v0.10.3 // indirect @@ -10,10 +13,12 @@ require ( github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.80 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rs/xid v1.6.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect golang.org/x/crypto v0.28.0 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.19.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 12e4fa7..3cdb8bd 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= @@ -15,8 +17,12 @@ github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.80 h1:2mdUHXEykRdY/BigLt3Iuu1otL0JTogT0Nmltg0wujk= github.com/minio/minio-go/v7 v7.0.80/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= @@ -26,3 +32,6 @@ golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/storage.go b/storage.go index 2e723c1..45b8693 100644 --- a/storage.go +++ b/storage.go @@ -1,9 +1,15 @@ package fsys import ( + "bytes" + "context" "errors" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" + "net/http" + "os" + "path/filepath" + "strings" ) var ( @@ -15,6 +21,7 @@ var ( type Storage struct { config *Config s3Client *minio.Client + ctx context.Context } type Config struct { @@ -35,6 +42,7 @@ type Config struct { func New(config *Config) (*Storage, error) { newStorage := new(Storage) newStorage.config = config + newStorage.ctx = context.Background() switch config.Type { case "minio", "s3": @@ -45,11 +53,112 @@ func New(config *Config) (*Storage, error) { if err != nil { return nil, err } + + bucketExists, err := s3Client.BucketExists(newStorage.ctx, config.S3BucketName) + if err != nil { + return nil, err + } + + if !bucketExists { + if err := s3Client.MakeBucket(newStorage.ctx, config.S3BucketName, minio.MakeBucketOptions{}); err != nil { + return nil, err + } + } + newStorage.s3Client = s3Client case "local": + if err := os.MkdirAll(config.Path, 0740); err != nil { + return nil, err + } default: return nil, ErrInvalidStorageType } return newStorage, nil } + +func (s *Storage) Open(name string) (*File, error) { + returnFile := new(File) + returnFile.storage = s + + if s.s3Client != nil { + object, err := s.s3Client.GetObject(s.ctx, s.config.S3BucketName, name, minio.GetObjectOptions{}) + if err != nil { + errResp := minio.ToErrorResponse(err) + if errResp.StatusCode == http.StatusNotFound { + return nil, ErrFileNotFound + } else { + return nil, err + } + } + + objectStat, err := object.Stat() + if err != nil { + errResp := minio.ToErrorResponse(err) + if errResp.StatusCode == http.StatusNotFound { + return nil, ErrFileNotFound + } else { + return nil, err + } + } + + returnFile.Name = objectStat.Key + returnFile.file = object + } else { + if _, err := os.Stat(filepath.Join(s.config.Path, name)); err != nil && os.IsNotExist(err) { + return nil, ErrFileNotFound + } else if err != nil { + return nil, err + } + file, err := os.Open(filepath.Join(s.config.Path, name)) + if err != nil { + return nil, err + } + returnFile.file = file + + cutStr, _ := strings.CutPrefix(file.Name(), filepath.Clean(s.config.Path)+"/") + + returnFile.Name = cutStr + } + + return returnFile, nil +} + +func (s *Storage) Read(name string) ([]byte, error) { + if s.s3Client != nil { + object, err := s.s3Client.GetObject(s.ctx, s.config.S3BucketName, name, minio.GetObjectOptions{}) + if err != nil { + errResp := minio.ToErrorResponse(err) + if errResp.StatusCode == http.StatusNotFound { + return nil, ErrFileNotFound + } else { + return nil, err + } + } + + var objectBytes []byte + if _, err := object.Read(objectBytes); err != nil { + return nil, err + } + return objectBytes, nil + } else { + return os.ReadFile(filepath.Join(s.config.Path, name)) + } +} + +type WriteOptions struct { + S3Tags map[string]string +} + +func (s *Storage) Write(name string, data []byte, opts WriteOptions) error { + if s.s3Client != nil { + if _, err := s.s3Client.PutObject(s.ctx, s.config.S3BucketName, name, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ + UserTags: opts.S3Tags, + }); err != nil { + return err + } + return nil + } else { + return os.WriteFile(filepath.Join(s.config.Path, name), data, 0640) + } +} diff --git a/storage_test.go b/storage_test.go new file mode 100644 index 0000000..ae89b97 --- /dev/null +++ b/storage_test.go @@ -0,0 +1,62 @@ +package fsys + +import ( + "github.com/stretchr/testify/assert" + "io" + "testing" +) + +func TestStorageLocal(t *testing.T) { + + stor, err := New(&Config{ + Type: "local", + Path: "./test/local", + }) + assert.NoError(t, err) + + t.Log("== Write ==") + err = stor.Write("test.txt", []byte("hi"), WriteOptions{}) + assert.NoError(t, err) + + t.Log("== Open ==") + file, err := stor.Open("test.txt") + assert.NoError(t, err) + assert.Equal(t, "test.txt", file.Name) + + fileContent, err := io.ReadAll(file) + assert.NoError(t, err) + assert.Equal(t, []byte("hi"), fileContent) + + err = file.Close() + assert.NoError(t, err) + +} + +func TestStorageS3(t *testing.T) { + + stor, err := New(&Config{ + Type: "s3", + S3BucketName: "test", + S3AccessID: "root", + S3AccessKey: "password123", + S3Endpoint: "127.0.0.1:9000", + }) + assert.NoError(t, err) + + t.Log("== Write ==") + err = stor.Write("test.txt", []byte("hi"), WriteOptions{}) + assert.NoError(t, err) + + t.Log("== Open ==") + file, err := stor.Open("test.txt") + assert.NoError(t, err) + assert.Equal(t, "test.txt", file.Name) + + fileContent, err := io.ReadAll(file) + assert.NoError(t, err) + assert.Equal(t, []byte("hi"), fileContent) + + err = file.Close() + assert.NoError(t, err) + +}