From 98ee01a2d5d64d218fba4e29526a870da03e85ba Mon Sep 17 00:00:00 2001 From: Shane C Date: Wed, 20 Nov 2024 10:19:07 -0500 Subject: [PATCH] add readdir function, fix writing files on local filesystem if within folder. --- go.mod | 6 +++-- go.sum | 1 + storage.go | 66 +++++++++++++++++++++++++++++++++++++++++++++++-- storage_test.go | 33 +++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 6032d35..c108c37 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,10 @@ module egtyl.xyz/omnibill/fsys go 1.23.2 -require github.com/minio/minio-go/v7 v7.0.80 +require ( + github.com/minio/minio-go/v7 v7.0.80 + github.com/stretchr/testify v1.9.0 +) require ( github.com/davecgh/go-spew v1.1.1 // indirect @@ -15,7 +18,6 @@ require ( github.com/minio/md5-simd v1.1.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rs/xid v1.6.0 // indirect - github.com/stretchr/testify v1.9.0 // indirect golang.org/x/crypto v0.28.0 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/sys v0.26.0 // indirect diff --git a/go.sum b/go.sum index 3cdb8bd..13ad7af 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,7 @@ golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/storage.go b/storage.go index 2d7f2c2..c729175 100644 --- a/storage.go +++ b/storage.go @@ -19,6 +19,7 @@ var ( ErrInvalidStorageType = errors.New("invalid storage type") ErrInvalidPath = errors.New("invalid path") ErrFileNotFound = errors.New("file not found") + ErrFolderNotFound = errors.New("folder not found") ) type Storage struct { @@ -182,6 +183,17 @@ func (s *Storage) Write(name string, data []byte, opts WriteOptions) error { } return nil } else { + if fileInfo, err := os.Stat(filepath.Join(s.config.Path, filepath.Dir(name))); err != nil && errors.Is(err, fs.ErrNotExist) { + if err := os.MkdirAll(filepath.Join(s.config.Path, filepath.Dir(name)), 0740); err != nil { + return err + } + } else if err != nil { + return err + } else { + if !fileInfo.IsDir() { + return ErrInvalidPath + } + } return os.WriteFile(filepath.Join(s.config.Path, name), data, 0600) } } @@ -205,7 +217,8 @@ type FileInfo struct { Size int64 ModTime time.Time IsDir bool - Mode os.FileMode + // The file permissions, will always be 0777 when using S3. + Mode os.FileMode } // Stat grabs the file info from the Storage interface. @@ -228,7 +241,7 @@ func (s *Storage) Stat(name string) (*FileInfo, error) { fileInfo.Name = objInfo.Key fileInfo.Size = objInfo.Size fileInfo.ModTime = objInfo.LastModified - fileInfo.Mode = 0777 // Used as a fake perm. Doesn't actually do anything. + fileInfo.Mode = fs.FileMode(0777) // Used as a fake perm. Doesn't actually do anything. } else { fInfo, err := os.Stat(filepath.Join(s.config.Path, name)) if err != nil { @@ -301,3 +314,52 @@ func (s *Storage) Move(name string, dest string) error { return os.Rename(filepath.Join(s.config.Path, name), filepath.Join(s.config.Path, dest)) } } + +// ReadDir reads all files within a certain directory. +// It takes a folder name as its parameter. +// The function returns an array of FileInfo, and an error, if any. +func (s *Storage) ReadDir(name string) ([]FileInfo, error) { + var fileInfo []FileInfo + if s.s3Client != nil { + for object := range s.s3Client.ListObjects(s.ctx, s.config.S3BucketName, minio.ListObjectsOptions{ + Prefix: name + "/", + }) { + if object.Err != nil { + minioErr := minio.ToErrorResponse(object.Err) + if minioErr.Code == "NoSuchKey" { + return nil, ErrFolderNotFound + } + return nil, object.Err + } + fileInfo = append(fileInfo, FileInfo{ + Name: object.Key, + Size: object.Size, + ModTime: object.LastModified, + Mode: fs.FileMode(0777), + IsDir: object.Size == 0, + }) + } + } else { + dirInfo, err := os.ReadDir(filepath.Join(s.config.Path, name)) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, ErrFolderNotFound + } + return nil, err + } + for _, fInfo := range dirInfo { + info, err := fInfo.Info() + if err != nil { + return nil, err + } + fileInfo = append(fileInfo, FileInfo{ + Name: info.Name(), + Size: info.Size(), + ModTime: info.ModTime(), + Mode: info.Mode(), + IsDir: info.IsDir(), + }) + } + } + return fileInfo, nil +} diff --git a/storage_test.go b/storage_test.go index 17062ff..58bf5c9 100644 --- a/storage_test.go +++ b/storage_test.go @@ -16,9 +16,16 @@ func TestStorageLocal(t *testing.T) { assert.NoError(t, err) t.Log("== Write ==") + err = stor.Write("test.txt", []byte("hi"), WriteOptions{}) assert.NoError(t, err) + err = stor.Write("testing/1.txt", []byte("hi"), WriteOptions{}) + assert.NoError(t, err) + err = stor.Write("testing/2.txt", []byte("hi"), WriteOptions{}) + assert.NoError(t, err) + err = stor.Write("testing/test/2.txt", []byte("hi"), WriteOptions{}) + assert.NoError(t, err) t.Log("== Stat ==") fileInfo, err := stor.Stat("test.txt") assert.NoError(t, err) @@ -31,6 +38,12 @@ func TestStorageLocal(t *testing.T) { assert.NoError(t, err) assert.Equal(t, []byte("hi"), fileContent) + t.Log("== ReadDir ==") + + files, err := stor.ReadDir("testing") + assert.NoError(t, err) + assert.Equal(t, 3, len(files)) + t.Log("== Copy ==") err = stor.Copy("test.txt", "test2.txt") assert.NoError(t, err) @@ -81,10 +94,19 @@ func TestStorageS3(t *testing.T) { assert.NoError(t, err) t.Log("== Write ==") + err = stor.Write("test.txt", []byte("hi"), WriteOptions{}) assert.NoError(t, err) + err = stor.Write("testing/1.txt", []byte("hi"), WriteOptions{}) + assert.NoError(t, err) + err = stor.Write("testing/2.txt", []byte("hi"), WriteOptions{}) + assert.NoError(t, err) + err = stor.Write("testing/test/2.txt", []byte("hi"), WriteOptions{}) + assert.NoError(t, err) + t.Log("== Stat ==") + fileInfo, err := stor.Stat("test.txt") assert.NoError(t, err) assert.Equal(t, "test.txt", fileInfo.Name) @@ -92,11 +114,19 @@ func TestStorageS3(t *testing.T) { assert.Equal(t, fs.FileMode(0777), fileInfo.Mode) t.Log("== Read ==") + fileContent, err := stor.Read("test.txt") assert.NoError(t, err) assert.Equal(t, []byte("hi"), fileContent) + t.Log("== ReadDir ==") + + files, err := stor.ReadDir("testing") + assert.NoError(t, err) + assert.Equal(t, 3, len(files)) + t.Log("== Copy ==") + err = stor.Copy("test.txt", "test2.txt") assert.NoError(t, err) @@ -107,6 +137,7 @@ func TestStorageS3(t *testing.T) { assert.Equal(t, int64(len("hi")), copyFileInfo.Size) t.Log("== Move ==") + err = stor.Move("test2.txt", "test3.txt") assert.NoError(t, err) @@ -117,6 +148,7 @@ func TestStorageS3(t *testing.T) { assert.Equal(t, int64(len("hi")), moveFileInfo.Size) t.Log("== Open ==") + file, err := stor.Open("test.txt") assert.NoError(t, err) assert.Equal(t, "test.txt", file.Name) @@ -129,6 +161,7 @@ func TestStorageS3(t *testing.T) { assert.NoError(t, err) t.Log("== Delete ==") + err = stor.Delete("test.txt") assert.NoError(t, err)