From e777e25d8737004c37333fc07dfece3fb17783ef Mon Sep 17 00:00:00 2001 From: "S.Solodyagin" Date: Sun, 27 Apr 2025 01:49:21 +0300 Subject: [PATCH] remove prefix, add remote storage --- go.mod | 20 +- go.sum | 29 +++ httpfs.go | 53 +++++ remote/helper.go | 23 +++ remote/miniostorage/file.go | 299 ++++++++++++++++++++++++++++ remote/miniostorage/fileinfo.go | 96 +++++++++ remote/miniostorage/helper.go | 15 ++ remote/miniostorage/miniostorage.go | 148 ++++++++++++++ remote/storage.go | 52 +++++ store.go | 61 +++--- 10 files changed, 759 insertions(+), 37 deletions(-) create mode 100644 go.sum create mode 100644 httpfs.go create mode 100644 remote/helper.go create mode 100644 remote/miniostorage/file.go create mode 100644 remote/miniostorage/fileinfo.go create mode 100644 remote/miniostorage/helper.go create mode 100644 remote/miniostorage/miniostorage.go create mode 100644 remote/storage.go diff --git a/go.mod b/go.mod index b6d8c4c..c7609fa 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,21 @@ module github.com/tenrok/filestore -go 1.23.4 +go 1.23.6 + +require github.com/minio/minio-go/v7 v7.0.90 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/minio/crc64nvme v1.0.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/rs/xid v1.6.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d6b17eb --- /dev/null +++ b/go.sum @@ -0,0 +1,29 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY= +github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.90 h1:TmSj1083wtAD0kEYTx7a5pFsv3iRYMsOJ6A4crjA1lE= +github.com/minio/minio-go/v7 v7.0.90/go.mod h1:uvMUcGrpgeSAAI6+sD3818508nUyMULw94j2Nxku/Go= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= diff --git a/httpfs.go b/httpfs.go new file mode 100644 index 0000000..4bb5bf7 --- /dev/null +++ b/httpfs.go @@ -0,0 +1,53 @@ +package filestore + +import ( + "net/http" + "strings" + + "github.com/tenrok/filestore/remote" +) + +var _ http.FileSystem = (*HttpFS)(nil) + +type HttpFS struct { + store *Store + remoteStorage remote.Storage +} + +type Option func(*HttpFS) + +// WithRemoteStorage +func WithRemoteStorage(storage remote.Storage) Option { + return func(httpFS *HttpFS) { + httpFS.remoteStorage = storage + } +} + +// NewHttpFS +func NewHttpFS(dir string, opts ...Option) (*HttpFS, error) { + store, err := NewStore(dir) + if err != nil { + return nil, err + } + f := &HttpFS{store: store} + for _, opt := range opts { + if opt != nil { + opt(f) + } + } + return f, nil +} + +// Open +func (f *HttpFS) Open(name string) (http.File, error) { + n := strings.TrimPrefix(name, "/") + if f.remoteStorage != nil { + return f.remoteStorage.Open(name) + } + return f.store.Open(n) +} + +// RemoteStorage +func (f *HttpFS) RemoteStorage() remote.Storage { + return f.remoteStorage +} diff --git a/remote/helper.go b/remote/helper.go new file mode 100644 index 0000000..04fe53e --- /dev/null +++ b/remote/helper.go @@ -0,0 +1,23 @@ +package remote + +import ( + "errors" + "strings" +) + +var ( + ErrNoScheme = errors.New("no scheme") + ErrEmptyURL = errors.New("URL cannot be empty") +) + +// schemeFromURL returns the scheme from a URL string +func schemeFromURL(url string) (string, error) { + if url == "" { + return "", ErrEmptyURL + } + i := strings.Index(url, ":") + if i < 1 { + return "", ErrNoScheme + } + return url[:i], nil +} diff --git a/remote/miniostorage/file.go b/remote/miniostorage/file.go new file mode 100644 index 0000000..f5ab04d --- /dev/null +++ b/remote/miniostorage/file.go @@ -0,0 +1,299 @@ +package miniostorage + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "sort" + "syscall" + + "github.com/minio/minio-go/v7" +) + +var ( + ErrOutOfRange = errors.New("out of range") + ErrNotSupported = errors.New("doesn't support this operation") +) + +var _ http.File = (*MinioFile)(nil) + +type MinioFile struct { + openFlags int + offset int64 + closed bool + resource *minioFileResource +} + +// NewMinioFile +func NewMinioFile(ctx context.Context, storage *MinioStorage, openFlags int, fileMode os.FileMode, name string) *MinioFile { + return &MinioFile{ + openFlags: openFlags, + // offset: 0, + // closed: false, + resource: &minioFileResource{ + ctx: ctx, + storage: storage, + name: name, + fileMode: fileMode, + currentIoSize: 0, + offset: 0, + reader: nil, + writer: nil, + }, + } +} + +// Close +func (f *MinioFile) Close() error { + if f.closed { + return os.ErrClosed + } + f.closed = true + return f.resource.Close() +} + +// Read +func (f *MinioFile) Read(p []byte) (int, error) { + if f.closed { + return 0, os.ErrClosed + } + readed, err := f.resource.ReadAt(p, f.offset) + f.offset += int64(readed) + return readed, err +} + +// Seek +func (f *MinioFile) Seek(offset int64, whence int) (int64, error) { + if f.closed { + return 0, os.ErrClosed + } + // Since this is an expensive operation; let's make sure we need it + if (whence == 0 && offset == f.offset) || (whence == 1 && offset == 0) { + return f.offset, nil + } + // Fore the reader/writers to be reopened (at correct offset) + if err := f.Sync(); err != nil { + return 0, err + } + stat, err := f.Stat() + if err != nil { + return 0, nil + } + switch whence { + case io.SeekStart: + f.offset = offset + case io.SeekCurrent: + f.offset += offset + case io.SeekEnd: + f.offset = stat.Size() + offset + } + return f.offset, nil +} + +// Write +func (f *MinioFile) Write(p []byte) (int, error) { + return f.WriteAt(p, f.offset) +} + +// WriteAt +func (f *MinioFile) WriteAt(b []byte, off int64) (int, error) { + if f.closed { + return 0, os.ErrClosed + } + if f.openFlags&os.O_RDONLY != 0 { + return 0, fmt.Errorf("file is opened as read only") + } + written, err := f.resource.WriteAt(b, off) + f.offset += int64(written) + return written, err +} + +// readdirImpl +func (f *MinioFile) readdirImpl(count int) ([]*MinioFileInfo, error) { + if err := f.Sync(); err != nil { + return nil, err + } + ownInfo, err := f.Stat() + if err != nil { + return nil, err + } + if !ownInfo.IsDir() { + return nil, syscall.ENOTDIR + } + var res []*MinioFileInfo + objs := f.resource.storage.client.ListObjects(f.resource.ctx, f.resource.storage.bucket, minio.ListObjectsOptions{ + Recursive: true, + Prefix: f.resource.name, + }) + for obj := range objs { + tmp := NewFileInfoFromAttrs(obj, f.resource.fileMode) + if tmp.Name() == "" { + // neither object.Name, not object.Prefix were present - so let's skip this unknown thing + continue + } + res = append(res, tmp) + } + if count > 0 && len(res) > 0 { + sort.Sort(ByName(res)) + res = res[:count] + } + return res, nil +} + +// Readdir +func (f *MinioFile) Readdir(count int) ([]fs.FileInfo, error) { + fi, err := f.readdirImpl(count) + if err != nil { + return nil, err + } + var res []fs.FileInfo + for _, v := range fi { + res = append(res, v) + } + return res, nil +} + +// Readdirnames +func (f *MinioFile) Readdirnames(n int) ([]string, error) { + fi, err := f.Readdir(n) + if err != nil && err != io.EOF { + return nil, err + } + names := make([]string, len(fi)) + for i, v := range fi { + names[i] = v.Name() + } + return names, err +} + +// Stat +func (f *MinioFile) Stat() (os.FileInfo, error) { + if err := f.Sync(); err != nil { + return nil, err + } + stat, err := f.resource.storage.client.StatObject(f.resource.ctx, f.resource.storage.bucket, f.resource.name, minio.StatObjectOptions{}) + if err != nil { + return nil, err + } + return NewFileInfoFromAttrs(stat, f.resource.fileMode), nil +} + +// Sync +func (f *MinioFile) Sync() error { + return f.resource.maybeCloseIo() +} + +// Truncate +func (f *MinioFile) Truncate(_ int64) error { + return ErrNotSupported +} + +// WriteString +func (f *MinioFile) WriteString(s string) (int, error) { + return f.Write([]byte(s)) +} + +type readerAtCloser interface { + io.ReadCloser + io.ReaderAt +} + +type minioFileResource struct { + ctx context.Context + storage *MinioStorage + name string + fileMode os.FileMode + currentIoSize int64 + offset int64 + reader readerAtCloser + writer io.WriteCloser + closed bool +} + +// Close +func (r *minioFileResource) Close() error { + r.closed = true + return r.maybeCloseIo() +} + +// maybeCloseIo +func (r *minioFileResource) maybeCloseIo() error { + if r.reader != nil { + if err := r.reader.Close(); err != nil { + return fmt.Errorf("error closing reader: %v", err) + } + r.reader = nil + } + if r.writer != nil { + if err := r.writer.Close(); err != nil { + return fmt.Errorf("error closing writer: %v", err) + } + r.writer = nil + } + return nil +} + +// ReadAt +func (r *minioFileResource) ReadAt(p []byte, offset int64) (int, error) { + if cap(p) == 0 { + return 0, nil + } + // Assume that if the reader is open; it is at the correct offset a good performance assumption that we must ensure holds + if offset == r.offset && r.reader != nil { + readed, err := r.reader.ReadAt(p, offset) + r.offset += int64(readed) + return readed, err + } + // If any writers have written anything; commit it first so we can read it back. + if err := r.maybeCloseIo(); err != nil { + return 0, err + } + obj, err := r.storage.client.GetObject(r.ctx, r.storage.bucket, r.name, minio.GetObjectOptions{}) + if err != nil { + return 0, err + } + r.reader = obj + r.offset = offset + readed, err := obj.ReadAt(p, offset) + r.offset += int64(readed) + return readed, err +} + +// WriteAt +func (r *minioFileResource) WriteAt(b []byte, offset int64) (int, error) { + // If the writer is opened and at the correct offset we're good! + if offset == r.offset && r.writer != nil { + written, err := r.writer.Write(b) + r.offset += int64(written) + return written, err + } + // Ensure readers must be re-opened and that if a writer is active at another offset it is first committed before we do a "seek" below + if err := r.maybeCloseIo(); err != nil { + return 0, err + } + // WriteAt to a non existing file + if offset > r.currentIoSize { + return 0, ErrOutOfRange + } + r.offset = offset + buffer := bytes.NewReader(b) + opts := minio.PutObjectOptions{ + ContentType: http.DetectContentType(b), + } + if offset > 0 { + opts.PartSize = uint64(offset) + opts.NumThreads = 8 + opts.ConcurrentStreamParts = false + opts.DisableMultipart = true + } + if _, err := r.storage.client.PutObject(r.ctx, r.storage.bucket, r.name, buffer, buffer.Size(), opts); err != nil { + return 0, err + } + r.offset += int64(buffer.Len()) + return buffer.Len(), nil +} diff --git a/remote/miniostorage/fileinfo.go b/remote/miniostorage/fileinfo.go new file mode 100644 index 0000000..36f4b8c --- /dev/null +++ b/remote/miniostorage/fileinfo.go @@ -0,0 +1,96 @@ +package miniostorage + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + "time" + + "github.com/minio/minio-go/v7" +) + +const folderSize = 42 + +var _ fs.FileInfo = (*MinioFileInfo)(nil) + +type MinioFileInfo struct { + ETag string + name string + size int64 + updated time.Time + isDir bool + fileMode os.FileMode +} + +// NewFileInfoFromAttrs +func NewFileInfoFromAttrs(obj minio.ObjectInfo, fileMode os.FileMode) *MinioFileInfo { + res := &MinioFileInfo{ + ETag: obj.ETag, + name: obj.Key, + size: obj.Size, + updated: obj.LastModified, + isDir: false, + fileMode: fileMode, + } + + if res.name == "" { + // deals with them at the moment + //res.name = "folder" + res.size = folderSize + res.isDir = true + } + + return res +} + +// Name +func (fi *MinioFileInfo) Name() string { + return filepath.Base(filepath.FromSlash(fi.name)) +} + +// Size +func (fi *MinioFileInfo) Size() int64 { + return fi.size +} + +// Mode +func (fi *MinioFileInfo) Mode() os.FileMode { + if fi.IsDir() { + return os.ModeDir | fi.fileMode + } + return fi.fileMode +} + +// ModTime +func (fi *MinioFileInfo) ModTime() time.Time { + return fi.updated +} + +// IsDir +func (fi *MinioFileInfo) IsDir() bool { + return fi.isDir +} + +// Sys +func (fi *MinioFileInfo) Sys() any { + return nil +} + +type ByName []*MinioFileInfo + +// Len +func (a ByName) Len() int { return len(a) } + +// Swap +func (a ByName) Swap(i, j int) { + a[i].name, a[j].name = a[j].name, a[i].name + a[i].size, a[j].size = a[j].size, a[i].size + a[i].updated, a[j].updated = a[j].updated, a[i].updated + a[i].isDir, a[j].isDir = a[j].isDir, a[i].isDir +} + +// Less +func (a ByName) Less(i, j int) bool { + return strings.Compare(a[i].Name(), a[j].Name()) == -1 +} diff --git a/remote/miniostorage/helper.go b/remote/miniostorage/helper.go new file mode 100644 index 0000000..c20119d --- /dev/null +++ b/remote/miniostorage/helper.go @@ -0,0 +1,15 @@ +package miniostorage + +import "net/url" + +// getUserPassword +func getUserPassword(u *url.URL) (string, string) { + var user, password string + if u.User != nil { + user = u.User.Username() + if p, ok := u.User.Password(); ok { + password = p + } + } + return user, password +} diff --git a/remote/miniostorage/miniostorage.go b/remote/miniostorage/miniostorage.go new file mode 100644 index 0000000..7f1963d --- /dev/null +++ b/remote/miniostorage/miniostorage.go @@ -0,0 +1,148 @@ +package miniostorage + +import ( + "context" + "errors" + "net/http" + "net/url" + "os" + "strconv" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/tenrok/filestore/remote" +) + +const defaultFileMode = 0o755 + +var _ remote.Storage = (*MinioStorage)(nil) + +func init() { + remote.Register("minio", &MinioStorage{}) +} + +type MinioStorage struct { + ctx context.Context + client *minio.Client + bucket string + separator string +} + +// NewStorage +func (s *MinioStorage) NewStorage(ctx context.Context, connString string) (remote.Storage, error) { + u, err := url.Parse(connString) + if err != nil { + return nil, err + } + queries := u.Query() + username, password := getUserPassword(u) + token := queries.Get("token") + opts := &minio.Options{ + Creds: credentials.NewStaticV4(username, password, token), + Region: "us-east-1", + } + if queries.Has("secure") { + secure, err := strconv.ParseBool(queries.Get("secure")) + if err != nil { + return nil, err + } + opts.Secure = secure + } + if queries.Has("region") { + opts.Region = queries.Get("region") + } + client, err := minio.New(u.Host, opts) + if err != nil { + return nil, err + } + s.ctx = ctx + s.client = client + s.bucket = u.Path[1:] + s.separator = "/" + return s, nil +} + +// normSeparators will normalize all "\\" and "/" to the provided separator +func (s *MinioStorage) normSeparators(str string) string { + return strings.Replace(strings.Replace(str, "\\", s.separator, -1), "/", s.separator, -1) +} + +// Create +func (s *MinioStorage) Create(name string) (http.File, error) { + return s.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0) +} + +// Open +func (s *MinioStorage) Open(name string) (http.File, error) { + return s.OpenFile(name, os.O_RDONLY, 0) +} + +// OpenFile +func (s *MinioStorage) OpenFile(name string, flag int, fileMode os.FileMode) (http.File, error) { + if flag&os.O_APPEND != 0 { + return nil, errors.New("appending files will lead to trouble") + } + name = strings.TrimPrefix(s.normSeparators(name), s.separator) + file := NewMinioFile(s.ctx, s, flag, fileMode, name) + var err error + if flag&os.O_CREATE != 0 { + _, err = file.WriteString("") + } + return file, err +} + +// Remove +func (s *MinioStorage) Remove(name string) error { + name = strings.TrimPrefix(s.normSeparators(name), s.separator) + return s.client.RemoveObject(s.ctx, s.bucket, name, minio.RemoveObjectOptions{GovernanceBypass: true}) +} + +// RemoveAll +func (s *MinioStorage) RemoveAll(path string) error { + path = strings.TrimPrefix(s.normSeparators(path), s.separator) + objectsCh := make(chan minio.ObjectInfo) + go func() { + defer close(objectsCh) + opts := minio.ListObjectsOptions{Prefix: path, Recursive: true} + for object := range s.client.ListObjects(s.ctx, s.bucket, opts) { + if object.Err != nil { + panic(object.Err) + } + objectsCh <- object + } + }() + errorCh := s.client.RemoveObjects(s.ctx, s.bucket, objectsCh, minio.RemoveObjectsOptions{}) + for e := range errorCh { + return errors.New("Failed to remove " + e.ObjectName + ", error: " + e.Err.Error()) + } + return nil +} + +// Rename +func (s *MinioStorage) Rename(oldName, newName string) error { + if oldName == newName { + return nil + } + oldName = strings.TrimPrefix(s.normSeparators(oldName), s.separator) + newName = strings.TrimPrefix(s.normSeparators(newName), s.separator) + src := minio.CopySrcOptions{ + Bucket: s.bucket, + Object: oldName, + } + dst := minio.CopyDestOptions{ + Bucket: s.bucket, + Object: newName, + } + if _, err := s.client.CopyObject(s.ctx, dst, src); err != nil { + return err + } + return s.Remove(oldName) +} + +// Stat +func (s *MinioStorage) Stat(name string) (os.FileInfo, error) { + name = strings.TrimPrefix(s.normSeparators(name), s.separator) + file := NewMinioFile(s.ctx, s, os.O_RDWR, defaultFileMode, name) + return file.Stat() +} diff --git a/remote/storage.go b/remote/storage.go new file mode 100644 index 0000000..c5c8c65 --- /dev/null +++ b/remote/storage.go @@ -0,0 +1,52 @@ +package remote + +import ( + "context" + "fmt" + "net/http" + "sync" +) + +var ( + storagesMu sync.RWMutex + storages = make(map[string]Storage) +) + +type Storage interface { + NewStorage(ctx context.Context, connString string) (Storage, error) + // Create(name string) (http.File, error) + Open(name string) (http.File, error) + // OpenFile(name string, flag int, fileMode os.FileMode) (http.File, error) + // Remove(name string) error + // RemoveAll(path string) error + // Rename(oldName, newName string) error + // Stat(name string) (os.FileInfo, error) +} + +// NewStorage returns a new remote storage instance. +func NewStorage(ctx context.Context, connString string) (Storage, error) { + scheme, err := schemeFromURL(connString) + if err != nil { + return nil, err + } + storagesMu.RLock() + s, ok := storages[scheme] + storagesMu.RUnlock() + if !ok { + return nil, fmt.Errorf("unknown storage %v (forgotten import?)", scheme) + } + return s.NewStorage(ctx, connString) +} + +// Register globally registers a storage +func Register(name string, storage Storage) { + storagesMu.Lock() + defer storagesMu.Unlock() + if storage == nil { + panic("Register storage is nil") + } + if _, exists := storages[name]; exists { + panic("Register called twice for storage " + name) + } + storages[name] = storage +} diff --git a/store.go b/store.go index 84846bd..af0e60e 100644 --- a/store.go +++ b/store.go @@ -19,7 +19,7 @@ const tmpfileName = "" // Store описывает хранилище файлов. type Store struct { - root string + dir string mutexes struct { sync.Mutex @@ -28,15 +28,6 @@ type Store struct { } } -// Open открывает и возвращает хранилище файлов. -func Open(root string) (*Store, error) { - // Создаём каталог, если он ещё не создан - if err := os.MkdirAll(root, 0700); err != nil { - return nil, err - } - return &Store{root: root}, nil -} - // FileInfo описывает информацию о сохраненном файле. type FileInfo struct { Location string `json:"location"` @@ -47,24 +38,23 @@ type FileInfo struct { MD5 string `json:"md5"` } -// Create сохраняет файл в хранилище. В качестве имени файла используется комбинация из двух хешей. Файл сохраняется в подкаталоге prefix, -// если он задан, но данный prefix не учитывается в возвращаемой информации в имени файла. -func (s *Store) Create(prefix string, r io.Reader) (*FileInfo, error) { - // Добавляем префикс к корню и создаём каталог - root := filepath.Join(s.root, prefix) - if err := os.MkdirAll(root, 0700); err != nil { +// NewStore открывает и возвращает хранилище файлов. +func NewStore(dir string) (*Store, error) { + // Создаём каталог, если он ещё не создан + if err := os.MkdirAll(dir, 0700); err != nil { return nil, err } + return &Store{dir: dir}, nil +} +// Create сохраняет файл в хранилище. В качестве имени файла используется комбинация из двух хешей. +func (s *Store) Create(r io.Reader) (*FileInfo, error) { // Создаём временный файл в корневом каталоге - tmpfile, err := os.CreateTemp(root, "~tmp") + tmpfile, err := os.CreateTemp(s.dir, "~tmp") if err != nil { - err.(*os.PathError).Path = tmpfileName // Подменяем имя файла + err.(*os.PathError).Path = tmpfileName // Подмениваем имя файла return nil, err } - - // В любом случае временный файл должен быть удален, если он не был переименован, т.е. на момент окончания функции существует под - // временным именем defer os.Remove(tmpfile.Name()) // Копируем содержимое во временный файл @@ -77,7 +67,7 @@ func (s *Store) Create(prefix string, r io.Reader) (*FileInfo, error) { err = &os.PathError{Op: "create", Path: tmpfileName, Err: err} return nil, err } - mimetype := http.DetectContentType(data) // Определяем тип содержимого + mimetype := http.DetectContentType(data) // Одновременно с сохранением в файл считаем две хеш-суммы hashCRC32, hashMD5 := crc32.NewIEEE(), md5.New() @@ -92,7 +82,7 @@ func (s *Store) Create(prefix string, r io.Reader) (*FileInfo, error) { sumMD5 := hashMD5.Sum(nil) name := base32.StdEncoding.EncodeToString(append(hashCRC32.Sum(nil), sumMD5...)) fi := &FileInfo{ - Location: s.GetFullName(prefix, name), + Location: s.GetFullName(name), Name: name, Mimetype: mimetype, Size: size, @@ -133,9 +123,9 @@ func (s *Store) Create(prefix string, r io.Reader) (*FileInfo, error) { } // Open открывает файл из каталога. -func (s *Store) Open(prefix, name string) (*os.File, error) { +func (s *Store) Open(name string) (*os.File, error) { // Полное имя для доступа к файлу - fullName := s.GetFullName(prefix, name) + fullName := s.GetFullName(name) if fullName == "" { return nil, os.ErrNotExist } @@ -169,13 +159,13 @@ func (s *Store) Open(prefix, name string) (*os.File, error) { } // Remove удаляет файл из хранилища. -func (s *Store) Remove(prefix, name string) error { +func (s *Store) Remove(name string) error { mu := s.getMutex(name) mu.Lock() defer mu.Unlock() // Полное имя для доступа к файлу - fullName := s.GetFullName(prefix, name) + fullName := s.GetFullName(name) if fullName == "" { return os.ErrNotExist } @@ -186,7 +176,7 @@ func (s *Store) Remove(prefix, name string) error { } // Пытаемся удалить пустые каталоги, если они образовались - for i := 0; i < 2; i++ { + for range 2 { fullName = filepath.Dir(fullName) if err := os.Remove(fullName); err != nil { break // Если не получилось, значит каталог не пустой @@ -200,7 +190,7 @@ func (s *Store) Remove(prefix, name string) error { func (s *Store) Clean(lifetime time.Duration) error { // Удаляем вообще все файлы, если время жизни не задано if lifetime <= 0 { - files, err := filepath.Glob(filepath.Join(s.root, "*")) + files, err := filepath.Glob(filepath.Join(s.dir, "*")) if err != nil { return err } @@ -214,7 +204,7 @@ func (s *Store) Clean(lifetime time.Duration) error { // Вычисляем крайнюю дату валидности файлов valid := time.Now().Add(-lifetime) - err := filepath.Walk(s.root, + err := filepath.Walk(s.dir, func(filename string, info os.FileInfo, err error) error { if err != nil { return err @@ -231,7 +221,7 @@ func (s *Store) Clean(lifetime time.Duration) error { } // Пытаемся удалить пустые каталоги - for i := 0; i < 2; i++ { + for range 2 { filename = filepath.Dir(filename) if err = os.Remove(filename); err != nil { break // Каталог не пустой @@ -265,17 +255,16 @@ func (s *Store) getMutex(name string) *sync.Mutex { } // GetFullName возвращает полный путь к файлу в хранилище. -func (s *Store) GetFullName(prefix, name string) string { +func (s *Store) GetFullName(name string) string { if len(name) < 27 { return "" } - return filepath.Join(s.root, prefix, name[:1], name[1:3], name[3:]) + return filepath.Join(s.dir, name[:1], name[1:3], name[3:]) } // IsExists проверяет: существует ли файл в хранилище? -func (s *Store) IsExists(prefix, name string) bool { - // Полное имя файла - fullName := s.GetFullName(prefix, name) +func (s *Store) IsExists(name string) bool { + fullName := s.GetFullName(name) if fullName == "" { return false }