diff --git a/README.md b/README.md index b747d33..6c73cbd 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ # filestore -Simple file store +Простое файловое хранилище diff --git a/httpfs.go b/httpfs.go index 3bc5fd0..a086c36 100644 --- a/httpfs.go +++ b/httpfs.go @@ -7,6 +7,7 @@ import ( "git.company.lan/gopkg/filestore/remote" ) +// Убеждаемся в том, что мы всегда реализуем интерфейс http.FileSystem. var _ http.FileSystem = (*HttpFS)(nil) type HttpFS struct { @@ -18,14 +19,14 @@ type HttpFSOption func(*HttpFS) // WithRemoteStorage func WithRemoteStorage(storage remote.Storage) HttpFSOption { - return func(httpFS *HttpFS) { - httpFS.remoteStorage = storage + return func(f *HttpFS) { + f.remoteStorage = storage } } -// NewHttpFS -func NewHttpFS(dir string, opts ...HttpFSOption) (*HttpFS, error) { - localStorage, err := NewLocalStorage(dir) +// NewHttpFS создаёт новый экземпляр файловой системы. +func NewHttpFS(rootDir string, opts ...HttpFSOption) (*HttpFS, error) { + localStorage, err := NewLocalStorage(rootDir) if err != nil { return nil, err } @@ -42,7 +43,7 @@ func NewHttpFS(dir string, opts ...HttpFSOption) (*HttpFS, error) { return f, nil } -// Open +// Open реализует метод http.FileSystem. func (f *HttpFS) Open(name string) (http.File, error) { name = strings.TrimPrefix(name, "/") if f.remoteStorage != nil { @@ -51,7 +52,7 @@ func (f *HttpFS) Open(name string) (http.File, error) { return f.localStorage.Open(name) } -// Remove +// Remove удаляет файл. func (f *HttpFS) Remove(name string) error { name = strings.TrimPrefix(name, "/") if f.remoteStorage != nil { @@ -60,8 +61,8 @@ func (f *HttpFS) Remove(name string) error { return f.localStorage.Remove(name) } -// LocalStorage +// LocalStorage возвращает указатель на локальное хранилище. func (f *HttpFS) LocalStorage() *LocalStorage { return f.localStorage } -// RemoteStorage +// RemoteStorage возвращает указатель на удалённое хранилище. func (f *HttpFS) RemoteStorage() remote.Storage { return f.remoteStorage } diff --git a/remote/helper.go b/remote/helper.go index 04fe53e..85da9c0 100644 --- a/remote/helper.go +++ b/remote/helper.go @@ -10,7 +10,7 @@ var ( ErrEmptyURL = errors.New("URL cannot be empty") ) -// schemeFromURL returns the scheme from a URL string +// schemeFromURL возвращает схему из строки URL. func schemeFromURL(url string) (string, error) { if url == "" { return "", ErrEmptyURL diff --git a/remote/miniostorage/file.go b/remote/miniostorage/file.go index e7532d1..608c0a2 100644 --- a/remote/miniostorage/file.go +++ b/remote/miniostorage/file.go @@ -1,300 +1,31 @@ package miniostorage import ( - "bytes" - "context" - "errors" - "fmt" - "io" "io/fs" "net/http" - "os" - "sort" - "syscall" - "git.company.lan/gopkg/filestore/remote" "github.com/minio/minio-go/v7" ) -var ( - ErrOutOfRange = errors.New("out of range") - ErrNotSupported = errors.New("doesn't support this operation") -) +// Убеждаемся в том, что мы всегда реализуем интерфейс http.File. +var _ http.File = (*minioFileWrapper)(nil) -var _ remote.File = (*MinioFile)(nil) - -type MinioFile struct { - openFlags int - offset int64 - closed bool - resource *minioFileResource +// minioFileWrapper оборачивает minio.Object для реализации http.File. +type minioFileWrapper struct { + *minio.Object + name string } -// NewMinioFile -func NewMinioFile(ctx context.Context, storage *MinioStorage, openFlags int, fileMode os.FileMode, name string) *MinioFile { - return &MinioFile{ - openFlags: openFlags, - // offset: 0, - // closed: false, - resource: &minioFileResource{ - ctx: ctx, - storage: storage, - name: name, - fileMode: fileMode, - currentIoSize: 0, - offset: 0, - reader: nil, - writer: nil, - }, - } +// Readdir требуется для http.File. Для файлов возвращает ошибку. +func (f *minioFileWrapper) Readdir(count int) ([]fs.FileInfo, error) { + return nil, fs.ErrInvalid } -// Close -func (f *MinioFile) Close() error { - if f.closed { - return os.ErrClosed - } - f.closed = true - return f.resource.Close() -} - -// Read -func (f *MinioFile) Read(p []byte) (int, error) { - if f.closed { - return 0, os.ErrClosed - } - readed, err := f.resource.ReadAt(p, f.offset) - f.offset += int64(readed) - return readed, err -} - -// Seek -func (f *MinioFile) Seek(offset int64, whence int) (int64, error) { - if f.closed { - return 0, os.ErrClosed - } - // Since this is an expensive operation; let's make sure we need it - if (whence == 0 && offset == f.offset) || (whence == 1 && offset == 0) { - return f.offset, nil - } - // Fore the reader/writers to be reopened (at correct offset) - if err := f.Sync(); err != nil { - return 0, err - } - stat, err := f.Stat() - if err != nil { - return 0, nil - } - switch whence { - case io.SeekStart: - f.offset = offset - case io.SeekCurrent: - f.offset += offset - case io.SeekEnd: - f.offset = stat.Size() + offset - } - return f.offset, nil -} - -// Write -func (f *MinioFile) Write(p []byte) (int, error) { - return f.WriteAt(p, f.offset) -} - -// WriteAt -func (f *MinioFile) WriteAt(b []byte, off int64) (int, error) { - if f.closed { - return 0, os.ErrClosed - } - if f.openFlags&os.O_RDONLY != 0 { - return 0, fmt.Errorf("file is opened as read only") - } - written, err := f.resource.WriteAt(b, off) - f.offset += int64(written) - return written, err -} - -// readdirImpl -func (f *MinioFile) readdirImpl(count int) ([]*MinioFileInfo, error) { - if err := f.Sync(); err != nil { - return nil, err - } - ownInfo, err := f.Stat() +// Stat возвращает FileInfo, требуемый для http.File. +func (f *minioFileWrapper) Stat() (fs.FileInfo, error) { + info, err := f.Object.Stat() if err != nil { return nil, err } - if !ownInfo.IsDir() { - return nil, syscall.ENOTDIR - } - var res []*MinioFileInfo - objs := f.resource.storage.client.ListObjects(f.resource.ctx, f.resource.storage.bucket, minio.ListObjectsOptions{ - Recursive: true, - Prefix: f.resource.name, - }) - for obj := range objs { - tmp := NewFileInfoFromAttrs(obj, f.resource.fileMode) - if tmp.Name() == "" { - // neither object.Name, not object.Prefix were present - so let's skip this unknown thing - continue - } - res = append(res, tmp) - } - if count > 0 && len(res) > 0 { - sort.Sort(ByName(res)) - res = res[:count] - } - return res, nil -} - -// Readdir -func (f *MinioFile) Readdir(count int) ([]fs.FileInfo, error) { - fi, err := f.readdirImpl(count) - if err != nil { - return nil, err - } - var res []fs.FileInfo - for _, v := range fi { - res = append(res, v) - } - return res, nil -} - -// Readdirnames -func (f *MinioFile) Readdirnames(n int) ([]string, error) { - fi, err := f.Readdir(n) - if err != nil && err != io.EOF { - return nil, err - } - names := make([]string, len(fi)) - for i, v := range fi { - names[i] = v.Name() - } - return names, err -} - -// Stat -func (f *MinioFile) Stat() (os.FileInfo, error) { - if err := f.Sync(); err != nil { - return nil, err - } - stat, err := f.resource.storage.client.StatObject(f.resource.ctx, f.resource.storage.bucket, f.resource.name, minio.StatObjectOptions{}) - if err != nil { - return nil, err - } - return NewFileInfoFromAttrs(stat, f.resource.fileMode), nil -} - -// Sync -func (f *MinioFile) Sync() error { - return f.resource.maybeCloseIo() -} - -// Truncate -func (f *MinioFile) Truncate(_ int64) error { - return ErrNotSupported -} - -// WriteString -func (f *MinioFile) WriteString(s string) (int, error) { - return f.Write([]byte(s)) -} - -type readerAtCloser interface { - io.ReadCloser - io.ReaderAt -} - -type minioFileResource struct { - ctx context.Context - storage *MinioStorage - name string - fileMode os.FileMode - currentIoSize int64 - offset int64 - reader readerAtCloser - writer io.WriteCloser - closed bool -} - -// Close -func (r *minioFileResource) Close() error { - r.closed = true - return r.maybeCloseIo() -} - -// maybeCloseIo -func (r *minioFileResource) maybeCloseIo() error { - if r.reader != nil { - if err := r.reader.Close(); err != nil { - return fmt.Errorf("error closing reader: %v", err) - } - r.reader = nil - } - if r.writer != nil { - if err := r.writer.Close(); err != nil { - return fmt.Errorf("error closing writer: %v", err) - } - r.writer = nil - } - return nil -} - -// ReadAt -func (r *minioFileResource) ReadAt(p []byte, offset int64) (int, error) { - if cap(p) == 0 { - return 0, nil - } - // Assume that if the reader is open; it is at the correct offset a good performance assumption that we must ensure holds - if offset == r.offset && r.reader != nil { - readed, err := r.reader.ReadAt(p, offset) - r.offset += int64(readed) - return readed, err - } - // If any writers have written anything; commit it first so we can read it back. - if err := r.maybeCloseIo(); err != nil { - return 0, err - } - obj, err := r.storage.client.GetObject(r.ctx, r.storage.bucket, r.name, minio.GetObjectOptions{}) - if err != nil { - return 0, err - } - r.reader = obj - r.offset = offset - readed, err := obj.ReadAt(p, offset) - r.offset += int64(readed) - return readed, err -} - -// WriteAt -func (r *minioFileResource) WriteAt(b []byte, offset int64) (int, error) { - // If the writer is opened and at the correct offset we're good! - if offset == r.offset && r.writer != nil { - written, err := r.writer.Write(b) - r.offset += int64(written) - return written, err - } - // Ensure readers must be re-opened and that if a writer is active at another offset it is first committed before we do a "seek" below - if err := r.maybeCloseIo(); err != nil { - return 0, err - } - // WriteAt to a non existing file - if offset > r.currentIoSize { - return 0, ErrOutOfRange - } - r.offset = offset - buffer := bytes.NewReader(b) - opts := minio.PutObjectOptions{ - ContentType: http.DetectContentType(b), - } - if offset > 0 { - opts.PartSize = uint64(offset) - opts.NumThreads = 8 - opts.ConcurrentStreamParts = false - opts.DisableMultipart = true - } - if _, err := r.storage.client.PutObject(r.ctx, r.storage.bucket, r.name, buffer, buffer.Size(), opts); err != nil { - return 0, err - } - r.offset += int64(buffer.Len()) - return buffer.Len(), nil + return &minioFileInfo{info: info}, nil } diff --git a/remote/miniostorage/fileinfo.go b/remote/miniostorage/fileinfo.go index 36f4b8c..f8e6e87 100644 --- a/remote/miniostorage/fileinfo.go +++ b/remote/miniostorage/fileinfo.go @@ -3,94 +3,31 @@ package miniostorage import ( "io/fs" "os" - "path/filepath" - "strings" "time" "github.com/minio/minio-go/v7" ) -const folderSize = 42 +// Убеждаемся в том, что мы всегда реализуем интерфейс fs.FileInfo. +var _ fs.FileInfo = (*minioFileInfo)(nil) -var _ fs.FileInfo = (*MinioFileInfo)(nil) - -type MinioFileInfo struct { - ETag string - name string - size int64 - updated time.Time - isDir bool - fileMode os.FileMode +// minioFileInfo реализует fs.FileInfo. +type minioFileInfo struct { + info minio.ObjectInfo } -// NewFileInfoFromAttrs -func NewFileInfoFromAttrs(obj minio.ObjectInfo, fileMode os.FileMode) *MinioFileInfo { - res := &MinioFileInfo{ - ETag: obj.ETag, - name: obj.Key, - size: obj.Size, - updated: obj.LastModified, - isDir: false, - fileMode: fileMode, - } - - if res.name == "" { - // deals with them at the moment - //res.name = "folder" - res.size = folderSize - res.isDir = true - } - - return res +func newMinioFileInfo(info minio.ObjectInfo) *minioFileInfo { + return &minioFileInfo{info: info} } -// Name -func (fi *MinioFileInfo) Name() string { - return filepath.Base(filepath.FromSlash(fi.name)) -} +func (f *minioFileInfo) Name() string { return f.info.Key } -// Size -func (fi *MinioFileInfo) Size() int64 { - return fi.size -} +func (f *minioFileInfo) Size() int64 { return f.info.Size } -// Mode -func (fi *MinioFileInfo) Mode() os.FileMode { - if fi.IsDir() { - return os.ModeDir | fi.fileMode - } - return fi.fileMode -} +func (f *minioFileInfo) Mode() os.FileMode { return 0644 } // MinIO не поддерживает права доступа к файлам. Возвращаем значение по умолчанию. -// ModTime -func (fi *MinioFileInfo) ModTime() time.Time { - return fi.updated -} +func (f *minioFileInfo) ModTime() time.Time { return f.info.LastModified.Local() } -// IsDir -func (fi *MinioFileInfo) IsDir() bool { - return fi.isDir -} +func (f *minioFileInfo) IsDir() bool { return f.info.Key[len(f.info.Key)-1] == '/' } -// Sys -func (fi *MinioFileInfo) Sys() any { - return nil -} - -type ByName []*MinioFileInfo - -// Len -func (a ByName) Len() int { return len(a) } - -// Swap -func (a ByName) Swap(i, j int) { - a[i].name, a[j].name = a[j].name, a[i].name - a[i].size, a[j].size = a[j].size, a[i].size - a[i].updated, a[j].updated = a[j].updated, a[i].updated - a[i].isDir, a[j].isDir = a[j].isDir, a[i].isDir -} - -// Less -func (a ByName) Less(i, j int) bool { - return strings.Compare(a[i].Name(), a[j].Name()) == -1 -} +func (f *minioFileInfo) Sys() interface{} { return f.info } diff --git a/remote/miniostorage/helper.go b/remote/miniostorage/helper.go index c20119d..31e62af 100644 --- a/remote/miniostorage/helper.go +++ b/remote/miniostorage/helper.go @@ -2,7 +2,7 @@ package miniostorage import "net/url" -// getUserPassword +// getUserPassword возвращает имя пользователя и пароль. func getUserPassword(u *url.URL) (string, string) { var user, password string if u.User != nil { diff --git a/remote/miniostorage/miniostorage.go b/remote/miniostorage/miniostorage.go index 37417ac..cbdfaab 100644 --- a/remote/miniostorage/miniostorage.go +++ b/remote/miniostorage/miniostorage.go @@ -2,9 +2,9 @@ package miniostorage import ( "context" - "errors" + "io" + "net/http" "net/url" - "os" "strconv" "strings" @@ -14,8 +14,7 @@ import ( "git.company.lan/gopkg/filestore/remote" ) -const defaultFileMode = 0o755 - +// Убеждаемся в том, что мы всегда реализуем интерфейс remote.Storage. var _ remote.Storage = (*MinioStorage)(nil) func init() { @@ -23,21 +22,21 @@ func init() { } type MinioStorage struct { - ctx context.Context - client *minio.Client - bucket string - separator string + ctx context.Context + client *minio.Client + bucket string } -// NewStorage func (s *MinioStorage) NewStorage(ctx context.Context, connString string) (remote.Storage, error) { u, err := url.Parse(connString) if err != nil { return nil, err } + queries := u.Query() username, password := getUserPassword(u) token := queries.Get("token") + opts := &minio.Options{ Creds: credentials.NewStaticV4(username, password, token), Region: "us-east-1", @@ -52,97 +51,76 @@ func (s *MinioStorage) NewStorage(ctx context.Context, connString string) (remot if queries.Has("region") { opts.Region = queries.Get("region") } + client, err := minio.New(u.Host, opts) if err != nil { return nil, err } + s.ctx = ctx s.client = client s.bucket = u.Path[1:] - s.separator = "/" + return s, nil } -// normSeparators will normalize all "\\" and "/" to the provided separator -func (s *MinioStorage) normSeparators(str string) string { - return strings.ReplaceAll(strings.ReplaceAll(str, "\\", s.separator), "/", s.separator) +func (s *MinioStorage) Create(path string, opts ...remote.Option) (io.WriteCloser, error) { + return newMinioWriter(s.ctx, s.client, s.bucket, path, opts...), nil } -// Create -func (s *MinioStorage) Create(name string) (remote.File, error) { - return s.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0) +func (s *MinioStorage) Open(name string) (http.File, error) { + obj, err := s.client.GetObject(s.ctx, s.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + return &minioFileWrapper{Object: obj, name: name}, nil } -// Open -func (s *MinioStorage) Open(name string) (remote.File, error) { - return s.OpenFile(name, os.O_RDONLY, 0) +func (s *MinioStorage) Remove(path string) error { + return s.client.RemoveObject(s.ctx, s.bucket, path, minio.RemoveObjectOptions{}) } -// OpenFile -func (s *MinioStorage) OpenFile(name string, flag int, fileMode os.FileMode) (remote.File, error) { - if flag&os.O_APPEND != 0 { - return nil, errors.New("appending files will lead to trouble") +func (s *MinioStorage) Stat(path string) (remote.FileInfo, error) { + info, err := s.client.StatObject(s.ctx, s.bucket, path, minio.StatObjectOptions{}) + if err != nil { + return nil, err } - name = strings.TrimPrefix(s.normSeparators(name), s.separator) - file := NewMinioFile(s.ctx, s, flag, fileMode, name) - var err error - if flag&os.O_CREATE != 0 { - _, err = file.WriteString("") - } - return file, err + return newMinioFileInfo(info), nil } -// Remove -func (s *MinioStorage) Remove(name string) error { - name = strings.TrimPrefix(s.normSeparators(name), s.separator) - return s.client.RemoveObject(s.ctx, s.bucket, name, minio.RemoveObjectOptions{GovernanceBypass: true}) +func (s *MinioStorage) Exists(path string) (bool, error) { + // Сначала проверяем, является ли путь файлом + if ok, err := s.IsFile(path); err == nil && ok { + return true, nil + } + // Если не файл, то проверяем, является ли путь каталогом + return s.IsDir(path) } -// RemoveAll -func (s *MinioStorage) RemoveAll(path string) error { - path = strings.TrimPrefix(s.normSeparators(path), s.separator) - objectsCh := make(chan minio.ObjectInfo) - go func() { - defer close(objectsCh) - opts := minio.ListObjectsOptions{Prefix: path, Recursive: true} - for object := range s.client.ListObjects(s.ctx, s.bucket, opts) { - if object.Err != nil { - panic(object.Err) - } - objectsCh <- object - } - }() - errorCh := s.client.RemoveObjects(s.ctx, s.bucket, objectsCh, minio.RemoveObjectsOptions{}) - for e := range errorCh { - return errors.New("Failed to remove " + e.ObjectName + ", error: " + e.Err.Error()) +func (s *MinioStorage) IsDir(path string) (bool, error) { + options := minio.ListObjectsOptions{ + Prefix: strings.TrimRight(path, "/") + "/", + Recursive: false, + MaxKeys: 1, } - return nil + objectChan := s.client.ListObjects(s.ctx, s.bucket, options) + object, ok := <-objectChan + if !ok { + return false, nil + } + if object.Err != nil { + return false, object.Err + } + return true, nil } -// Rename -func (s *MinioStorage) Rename(oldName, newName string) error { - if oldName == newName { - return nil +func (s *MinioStorage) IsFile(path string) (bool, error) { + _, err := s.client.StatObject(s.ctx, s.bucket, path, minio.StatObjectOptions{}) + if err == nil { + return true, nil } - oldName = strings.TrimPrefix(s.normSeparators(oldName), s.separator) - newName = strings.TrimPrefix(s.normSeparators(newName), s.separator) - src := minio.CopySrcOptions{ - Bucket: s.bucket, - Object: oldName, + if strings.Contains(err.Error(), "The specified key does not exist.") { + return false, nil } - dst := minio.CopyDestOptions{ - Bucket: s.bucket, - Object: newName, - } - if _, err := s.client.CopyObject(s.ctx, dst, src); err != nil { - return err - } - return s.Remove(oldName) -} - -// Stat -func (s *MinioStorage) Stat(name string) (os.FileInfo, error) { - name = strings.TrimPrefix(s.normSeparators(name), s.separator) - file := NewMinioFile(s.ctx, s, os.O_RDWR, defaultFileMode, name) - return file.Stat() + return false, err } diff --git a/remote/miniostorage/uploader.go b/remote/miniostorage/uploader.go new file mode 100644 index 0000000..63adea8 --- /dev/null +++ b/remote/miniostorage/uploader.go @@ -0,0 +1,23 @@ +package miniostorage + +import ( + "io" + + "git.company.lan/gopkg/filestore/remote" +) + +func (s *MinioStorage) Uploader() remote.Uploader { return s } + +func (s *MinioStorage) Upload(path string, reader io.Reader, opts ...remote.Option) error { + file, err := s.Create(path, opts...) + if err != nil { + return err + } + + if _, err := io.Copy(file, reader); err != nil { + file.Close() + return err + } + + return file.Close() +} diff --git a/remote/miniostorage/writer.go b/remote/miniostorage/writer.go new file mode 100644 index 0000000..c069dd6 --- /dev/null +++ b/remote/miniostorage/writer.go @@ -0,0 +1,89 @@ +package miniostorage + +import ( + "bytes" + "context" + "fmt" + "io" + + "git.company.lan/gopkg/filestore/remote" + "github.com/minio/minio-go/v7" +) + +// Убеждаемся в том, что мы всегда реализуем интерфейс io.WriteCloser. +var _ io.WriteCloser = (*minioWriter)(nil) + +// minioWriter реализует интерфейс io.WriteCloser. +type minioWriter struct { + ctx context.Context + client *minio.Client + bucket string + path string + buffer *bytes.Buffer + metadata remote.Metadata + contentType string +} + +func newMinioWriter(ctx context.Context, client *minio.Client, bucket, path string, opts ...remote.Option) *minioWriter { + o := &remote.Options{} + for _, opt := range opts { + opt(o) + } + + writer := &minioWriter{ + ctx: ctx, + client: client, + bucket: bucket, + path: path, + buffer: bytes.NewBuffer(nil), + } + + if o.ContentType != "" { + writer.contentType = o.ContentType + } + if o.Metadata != nil { + writer.metadata = o.Metadata + } + + return writer +} + +func (w *minioWriter) Write(p []byte) (n int, err error) { + select { + case <-w.ctx.Done(): + return 0, w.ctx.Err() + default: + return w.buffer.Write(p) + } +} + +func (w *minioWriter) Close() error { + select { + case <-w.ctx.Done(): + return w.ctx.Err() + default: + opts := minio.PutObjectOptions{} + + if w.contentType != "" { + opts.ContentType = w.contentType + } + + if w.metadata != nil { + userMetadata := make(map[string]string) + for k, v := range w.metadata { + userMetadata[k] = fmt.Sprintf("%v", v) + } + opts.UserMetadata = userMetadata + } + + _, err := w.client.PutObject( + w.ctx, + w.bucket, + w.path, + bytes.NewReader(w.buffer.Bytes()), + int64(w.buffer.Len()), + opts, + ) + return err + } +} diff --git a/remote/option.go b/remote/option.go new file mode 100644 index 0000000..9f98cf6 --- /dev/null +++ b/remote/option.go @@ -0,0 +1,22 @@ +package remote + +type Option func(*Options) + +type Options struct { + Metadata Metadata + ContentType string +} + +// WithMetadata устанавливает метаданные. +func WithMetadata(metadata Metadata) Option { + return func(o *Options) { + o.Metadata = metadata + } +} + +// WithContentType устанавливает тип файла. +func WithContentType(contentType string) Option { + return func(o *Options) { + o.ContentType = contentType + } +} diff --git a/remote/storage.go b/remote/storage.go index 5894676..609e5c3 100644 --- a/remote/storage.go +++ b/remote/storage.go @@ -4,8 +4,7 @@ import ( "context" "fmt" "io" - "io/fs" - "os" + "net/http" "sync" ) @@ -14,50 +13,67 @@ var ( storages = make(map[string]Storage) ) -type File interface { - io.Closer - io.Reader - io.Seeker - io.Writer - Readdir(count int) ([]fs.FileInfo, error) - Stat() (fs.FileInfo, error) +type Uploader interface { + Upload(path string, reader io.Reader, opts ...Option) error } type Storage interface { NewStorage(ctx context.Context, connString string) (Storage, error) - Create(name string) (File, error) - Open(name string) (File, error) - OpenFile(name string, flag int, fileMode os.FileMode) (File, error) - Remove(name string) error - RemoveAll(path string) error - Rename(oldName, newName string) error - Stat(name string) (os.FileInfo, error) + + // Open реализует метод http.FileSystem. + Open(name string) (http.File, error) + + // Create создаёт файл и возвращает io.WriteCloser. + Create(path string, opts ...Option) (io.WriteCloser, error) + + // Remove удаляет файл. + Remove(path string) error + + // Stat получает информацию о файле/каталоге. + Stat(path string) (FileInfo, error) + + // Exists определяет, существует ли файл или каталог. + Exists(path string) (bool, error) + + // IsDir определяет, является ли путь каталогом. + IsDir(path string) (bool, error) + + // IsFile определяет, является ли путь файлом. + IsFile(path string) (bool, error) + + Uploader() Uploader } -// NewStorage returns a new remote storage instance. +// NewStorage создаёт новый экземпляр удаленного хранилища. func NewStorage(ctx context.Context, connString string) (Storage, error) { scheme, err := schemeFromURL(connString) if err != nil { return nil, err } + storagesMu.RLock() s, ok := storages[scheme] storagesMu.RUnlock() + if !ok { return nil, fmt.Errorf("unknown storage %v (forgotten import?)", scheme) } + return s.NewStorage(ctx, connString) } -// Register globally registers a storage +// Register глобально регистрирует хранилище. func Register(name string, storage Storage) { storagesMu.Lock() defer storagesMu.Unlock() + if storage == nil { panic("Register storage is nil") } + if _, exists := storages[name]; exists { panic("Register called twice for storage " + name) } + storages[name] = storage } diff --git a/remote/types.go b/remote/types.go new file mode 100644 index 0000000..3fe97d2 --- /dev/null +++ b/remote/types.go @@ -0,0 +1,19 @@ +package remote + +import ( + "os" + "time" +) + +// FileInfo реализует интерфейс os.FileInfo. +type FileInfo interface { + Name() string + Size() int64 + Mode() os.FileMode + ModTime() time.Time + IsDir() bool + Sys() interface{} +} + +// Metadata метаданные файла +type Metadata map[string]any diff --git a/storage.go b/storage.go index 63836cd..d76fb37 100644 --- a/storage.go +++ b/storage.go @@ -20,8 +20,8 @@ const tmpfileName = "" // LocalStorage описывает хранилище файлов. type LocalStorage struct { - dir string - permissions os.FileMode + rootDir string + perm os.FileMode mutexes struct { sync.Mutex @@ -33,9 +33,9 @@ type LocalStorage struct { type LocalStorageOption func(*LocalStorage) // WithPermissions -func WithPermissions(permissions os.FileMode) LocalStorageOption { - return func(storage *LocalStorage) { - storage.permissions = permissions +func WithPermissions(perm os.FileMode) LocalStorageOption { + return func(s *LocalStorage) { + s.perm = perm } } @@ -50,19 +50,17 @@ type FileInfo struct { } // NewLocalStorage открывает и возвращает хранилище файлов. -func NewLocalStorage(dir string, opts ...LocalStorageOption) (*LocalStorage, error) { +func NewLocalStorage(rootDir string, opts ...LocalStorageOption) (*LocalStorage, error) { s := &LocalStorage{} - s.dir = dir - s.permissions = 0700 + s.rootDir = rootDir + s.perm = 0700 for _, opt := range opts { - if opt != nil { - opt(s) - } + opt(s) } // Создаём каталог, если он ещё не создан - if err := os.MkdirAll(s.dir, s.permissions); err != nil { + if err := os.MkdirAll(s.rootDir, s.perm); err != nil { return nil, err } @@ -72,7 +70,7 @@ func NewLocalStorage(dir string, opts ...LocalStorageOption) (*LocalStorage, err // Create сохраняет файл в хранилище. В качестве имени файла используется комбинация из двух хешей. func (s *LocalStorage) Create(r io.Reader) (*FileInfo, error) { // Создаём временный файл в корневом каталоге - tmpfile, err := os.CreateTemp(s.dir, "~tmp") + tmpfile, err := os.CreateTemp(s.rootDir, "~tmp") if err != nil { err.(*os.PathError).Path = tmpfileName // Подмениваем имя файла return nil, err @@ -127,7 +125,7 @@ func (s *LocalStorage) Create(r io.Reader) (*FileInfo, error) { } // Если такого файла нет, то создаем для него каталог - if err := os.MkdirAll(filepath.Dir(fi.Location), s.permissions); err != nil { + if err := os.MkdirAll(filepath.Dir(fi.Location), s.perm); err != nil { err.(*os.PathError).Path = fi.Name return nil, err } @@ -212,7 +210,7 @@ func (s *LocalStorage) Remove(name string) error { func (s *LocalStorage) Clean(lifetime time.Duration) error { // Удаляем вообще все файлы, если время жизни не задано if lifetime <= 0 { - files, err := filepath.Glob(filepath.Join(s.dir, "*")) + files, err := filepath.Glob(filepath.Join(s.rootDir, "*")) if err != nil { return err } @@ -226,7 +224,7 @@ func (s *LocalStorage) Clean(lifetime time.Duration) error { // Вычисляем крайнюю дату валидности файлов valid := time.Now().Add(-lifetime) - err := filepath.Walk(s.dir, + err := filepath.Walk(s.rootDir, func(filename string, info os.FileInfo, err error) error { if err != nil { return err @@ -282,7 +280,7 @@ func (s *LocalStorage) GetFullName(name string) string { if len(name) < 27 { return "" } - return filepath.Join(s.dir, name[:1], name[1:3], name[3:]) + return filepath.Join(s.rootDir, name[:1], name[1:3], name[3:]) } // IsExists проверяет: существует ли файл в хранилище?