package fsys import ( "bytes" "context" "errors" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "io" "io/fs" "net/http" "os" "path/filepath" "strings" "time" ) var ( ErrInvalidStorageType = errors.New("invalid storage type") ErrInvalidPath = errors.New("invalid path") ErrFileNotFound = errors.New("file not found") ErrFolderNotFound = errors.New("folder not found") ) type Storage struct { config Config s3Client *minio.Client ctx context.Context } type Config struct { // Type of storage, can be "local" or "s3" Type string Path string S3Endpoint string S3Location string S3Secure bool S3AccessID string S3AccessKey string S3BucketName string // Override default s3 client. MinioClient *minio.Client } // New creates a new storage interface // It takes Config as its parameter. // The function returns the Storage interface, and an error, if any. func New(config Config) (*Storage, error) { newStorage := new(Storage) newStorage.config = config newStorage.ctx = context.Background() switch config.Type { case "minio", "s3": if config.MinioClient != nil { newStorage.s3Client = config.MinioClient } else { s3Client, err := minio.New(config.S3Endpoint, &minio.Options{ Creds: credentials.NewStaticV4(config.S3AccessID, config.S3AccessKey, ""), Secure: config.S3Secure, }) if err != nil { return nil, err } newStorage.s3Client = s3Client } bucketExists, err := newStorage.s3Client.BucketExists(newStorage.ctx, config.S3BucketName) if err != nil { return nil, err } if !bucketExists { if err := newStorage.s3Client.MakeBucket(newStorage.ctx, config.S3BucketName, minio.MakeBucketOptions{}); err != nil { return nil, err } } case "local": if statInfo, err := os.Stat(config.Path); err != nil && errors.Is(err, fs.ErrNotExist) { if err := os.MkdirAll(config.Path, 0740); err != nil { return nil, err } } else if err != nil { return nil, err } else if !statInfo.IsDir() { return nil, ErrInvalidPath } default: return nil, ErrInvalidStorageType } return newStorage, nil } // Open opens a file within the Storage interface. // It takes a file name as its parameter. // The function returns a File and an error, if any. func (s *Storage) Open(name string) (*File, error) { returnFile := new(File) returnFile.storage = s if s.s3Client != nil { object, err := s.s3Client.GetObject(s.ctx, s.config.S3BucketName, name, minio.GetObjectOptions{}) if err != nil { errResp := minio.ToErrorResponse(err) if errResp.StatusCode == http.StatusNotFound { return nil, ErrFileNotFound } else { return nil, err } } objectStat, err := object.Stat() if err != nil { errResp := minio.ToErrorResponse(err) if errResp.StatusCode == http.StatusNotFound { return nil, ErrFileNotFound } else { return nil, err } } returnFile.Name = objectStat.Key returnFile.file = object } else { if _, err := os.Stat(filepath.Join(s.config.Path, name)); err != nil && errors.Is(err, fs.ErrNotExist) { return nil, ErrFileNotFound } else if err != nil { return nil, err } file, err := os.Open(filepath.Join(s.config.Path, name)) if err != nil { return nil, err } returnFile.file = file cutStr, _ := strings.CutPrefix(file.Name(), filepath.Clean(s.config.Path)+"/") returnFile.Name = cutStr } return returnFile, nil } // Read opens a file within the Storage interfaces and reads the contents. // It takes a file name as its parameter. // The function returns a []byte, or an error, if any. func (s *Storage) Read(name string) ([]byte, error) { if s.s3Client != nil { _, err := s.s3Client.StatObject(s.ctx, s.config.S3BucketName, name, minio.StatObjectOptions{}) if err != nil { errResp := minio.ToErrorResponse(err) if errResp.Code == "NoSuchKey" { return nil, ErrFileNotFound } else { return nil, err } } object, err := s.s3Client.GetObject(s.ctx, s.config.S3BucketName, name, minio.GetObjectOptions{}) if err != nil { return nil, err } objectBytes, err := io.ReadAll(object) if err != nil { return nil, err } return objectBytes, nil } else { return os.ReadFile(filepath.Join(s.config.Path, name)) } } type WriteOptions struct { S3Tags map[string]string } // Write writes to a file within the Storage interface. // It takes a name, a []byte, and WriteOptions as its parameters. // The function returns an error, if any. func (s *Storage) Write(name string, data []byte, opts WriteOptions) error { if s.s3Client != nil { if _, err := s.s3Client.PutObject(s.ctx, s.config.S3BucketName, name, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ UserTags: opts.S3Tags, }); err != nil { return err } return nil } else { if fileInfo, err := os.Stat(filepath.Join(s.config.Path, filepath.Dir(name))); err != nil && errors.Is(err, fs.ErrNotExist) { if err := os.MkdirAll(filepath.Join(s.config.Path, filepath.Dir(name)), 0740); err != nil { return err } } else if err != nil { return err } else { if !fileInfo.IsDir() { return ErrInvalidPath } } return os.WriteFile(filepath.Join(s.config.Path, name), data, 0600) } } // Delete deleted a file within the storage interface. // It takes a file name as its parameter. // The function returns an error, if any. func (s *Storage) Delete(name string) error { if s.s3Client != nil { if err := s.s3Client.RemoveObject(s.ctx, s.config.S3BucketName, name, minio.RemoveObjectOptions{}); err != nil { return err } return nil } else { return os.Remove(filepath.Join(s.config.Path, name)) } } type FileInfo struct { Name string Size int64 ModTime time.Time IsDir bool // The file permissions, will always be 0777 when using S3. Mode os.FileMode } // Stat grabs the file info from the Storage interface. // It takes a file name as its parameter. // The function returns FileInfo, and an error, if any. func (s *Storage) Stat(name string) (*FileInfo, error) { fileInfo := new(FileInfo) if s.s3Client != nil { objInfo, err := s.s3Client.StatObject(s.ctx, s.config.S3BucketName, name, minio.StatObjectOptions{}) if err != nil { errResp := minio.ToErrorResponse(err) if errResp.StatusCode == http.StatusNotFound { return nil, ErrFileNotFound } else { return nil, err } } fileInfo.Name = objInfo.Key fileInfo.Size = objInfo.Size fileInfo.ModTime = objInfo.LastModified fileInfo.Mode = fs.FileMode(0777) // Used as a fake perm. Doesn't actually do anything. } else { fInfo, err := os.Stat(filepath.Join(s.config.Path, name)) if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil, ErrFileNotFound } return nil, err } fileInfo.Name = fInfo.Name() fileInfo.Size = fInfo.Size() fileInfo.ModTime = fInfo.ModTime() fileInfo.Mode = fInfo.Mode() fileInfo.IsDir = fInfo.IsDir() } return fileInfo, nil } // Copy copies a file from one place to another. // It takes the file name, and the destination name as its parameter. // The function returns an error, if any. func (s *Storage) Copy(name string, dest string) error { if s.s3Client != nil { if _, err := s.s3Client.CopyObject(s.ctx, minio.CopyDestOptions{ Bucket: s.config.S3BucketName, Object: dest, }, minio.CopySrcOptions{ Bucket: s.config.S3BucketName, Object: name, }); err != nil { return err } return nil } else { fileContent, err := os.ReadFile(filepath.Join(s.config.Path, name)) if err != nil { return err } if err := os.WriteFile(filepath.Join(s.config.Path, dest), fileContent, 0600); err != nil { return err } return nil } } // Rename is a wrapper over the Move function. func (s *Storage) Rename(name, dest string) error { return s.Move(name, dest) } // Move moves a file from one place to another. // It takes a file name, and the destination name as its parameter. // The function returns an error, if any. func (s *Storage) Move(name string, dest string) error { if s.s3Client != nil { if _, err := s.s3Client.CopyObject(s.ctx, minio.CopyDestOptions{ Bucket: s.config.S3BucketName, Object: dest, }, minio.CopySrcOptions{ Bucket: s.config.S3BucketName, Object: name, }); err != nil { return err } if err := s.s3Client.RemoveObject(s.ctx, s.config.S3BucketName, name, minio.RemoveObjectOptions{}); err != nil { return err } return nil } else { return os.Rename(filepath.Join(s.config.Path, name), filepath.Join(s.config.Path, dest)) } } // ReadDir reads all files within a certain directory. // It takes a folder name as its parameter. // The function returns an array of FileInfo, and an error, if any. func (s *Storage) ReadDir(name string) ([]FileInfo, error) { var fileInfo []FileInfo if s.s3Client != nil { doesExist := false for object := range s.s3Client.ListObjects(s.ctx, s.config.S3BucketName, minio.ListObjectsOptions{ Prefix: name + "/", }) { doesExist = true if object.Err != nil { return nil, object.Err } fileInfo = append(fileInfo, FileInfo{ Name: object.Key, Size: object.Size, ModTime: object.LastModified, Mode: fs.FileMode(0777), IsDir: object.Size == 0, }) } if !doesExist { return nil, ErrFolderNotFound } } else { dirInfo, err := os.ReadDir(filepath.Join(s.config.Path, name)) if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil, ErrFolderNotFound } return nil, err } for _, fInfo := range dirInfo { info, err := fInfo.Info() if err != nil { return nil, err } fileInfo = append(fileInfo, FileInfo{ Name: info.Name(), Size: info.Size(), ModTime: info.ModTime(), Mode: info.Mode(), IsDir: info.IsDir(), }) } } return fileInfo, nil } // CopyDir copies a directory from one place to another. // It takes a folder nae, and the destination folder name as its parameter. // The function returns an error, if any. func (s *Storage) CopyDir(name string, dst string) error { if s.s3Client != nil { var fileInfo []FileInfo for object := range s.s3Client.ListObjects(s.ctx, s.config.S3BucketName, minio.ListObjectsOptions{ Prefix: name + "/", Recursive: true, }) { if object.Err != nil { minioErr := minio.ToErrorResponse(object.Err) if minioErr.Code == "NoSuchKey" { return ErrFolderNotFound } return object.Err } fileInfo = append(fileInfo, FileInfo{ Name: object.Key, Size: object.Size, ModTime: object.LastModified, Mode: fs.FileMode(0777), IsDir: object.Size == 0, }) } for _, object := range fileInfo { cutStr, _ := strings.CutPrefix(object.Name, name+"/") if _, err := s.s3Client.CopyObject(s.ctx, minio.CopyDestOptions{ Bucket: s.config.S3BucketName, Object: dst + "/" + cutStr, }, minio.CopySrcOptions{ Bucket: s.config.S3BucketName, Object: object.Name, }); err != nil { return err } } return nil } else { return os.Rename(filepath.Join(s.config.Path, name), filepath.Join(s.config.Path, dst)) } } // DeleteDir deletes all files within a certain directory. // It takes a folder name as its parameter. // The function returns an error, if any. func (s *Storage) DeleteDir(name string) error { if s.s3Client != nil { objChan := s.s3Client.ListObjects(s.ctx, s.config.S3BucketName, minio.ListObjectsOptions{Prefix: name + "/", Recursive: true}) for remObjErr := range s.s3Client.RemoveObjects(s.ctx, s.config.S3BucketName, objChan, minio.RemoveObjectsOptions{}) { if remObjErr.Err != nil { minioErr := minio.ToErrorResponse(remObjErr.Err) if minioErr.Code == "NoSuchKey" { return ErrFolderNotFound } else { return remObjErr.Err } } } return nil } else { return os.RemoveAll(filepath.Join(s.config.Path, name)) } }