разработка
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"git.company.lan/gopkg/filestore/remote"
|
||||
)
|
||||
|
||||
// Убеждаемся в том, что мы всегда реализуем интерфейс http.FileSystem.
|
||||
var _ http.FileSystem = (*HttpFS)(nil)
|
||||
|
||||
type HttpFS struct {
|
||||
@@ -18,14 +19,14 @@ type HttpFSOption func(*HttpFS)
|
||||
|
||||
// WithRemoteStorage
|
||||
func WithRemoteStorage(storage remote.Storage) HttpFSOption {
|
||||
return func(httpFS *HttpFS) {
|
||||
httpFS.remoteStorage = storage
|
||||
return func(f *HttpFS) {
|
||||
f.remoteStorage = storage
|
||||
}
|
||||
}
|
||||
|
||||
// NewHttpFS
|
||||
func NewHttpFS(dir string, opts ...HttpFSOption) (*HttpFS, error) {
|
||||
localStorage, err := NewLocalStorage(dir)
|
||||
// NewHttpFS создаёт новый экземпляр файловой системы.
|
||||
func NewHttpFS(rootDir string, opts ...HttpFSOption) (*HttpFS, error) {
|
||||
localStorage, err := NewLocalStorage(rootDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -42,7 +43,7 @@ func NewHttpFS(dir string, opts ...HttpFSOption) (*HttpFS, error) {
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Open
|
||||
// Open реализует метод http.FileSystem.
|
||||
func (f *HttpFS) Open(name string) (http.File, error) {
|
||||
name = strings.TrimPrefix(name, "/")
|
||||
if f.remoteStorage != nil {
|
||||
@@ -51,7 +52,7 @@ func (f *HttpFS) Open(name string) (http.File, error) {
|
||||
return f.localStorage.Open(name)
|
||||
}
|
||||
|
||||
// Remove
|
||||
// Remove удаляет файл.
|
||||
func (f *HttpFS) Remove(name string) error {
|
||||
name = strings.TrimPrefix(name, "/")
|
||||
if f.remoteStorage != nil {
|
||||
@@ -60,8 +61,8 @@ func (f *HttpFS) Remove(name string) error {
|
||||
return f.localStorage.Remove(name)
|
||||
}
|
||||
|
||||
// LocalStorage
|
||||
// LocalStorage возвращает указатель на локальное хранилище.
|
||||
func (f *HttpFS) LocalStorage() *LocalStorage { return f.localStorage }
|
||||
|
||||
// RemoteStorage
|
||||
// RemoteStorage возвращает указатель на удалённое хранилище.
|
||||
func (f *HttpFS) RemoteStorage() remote.Storage { return f.remoteStorage }
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ var (
|
||||
ErrEmptyURL = errors.New("URL cannot be empty")
|
||||
)
|
||||
|
||||
// schemeFromURL returns the scheme from a URL string
|
||||
// schemeFromURL возвращает схему из строки URL.
|
||||
func schemeFromURL(url string) (string, error) {
|
||||
if url == "" {
|
||||
return "", ErrEmptyURL
|
||||
|
||||
+13
-282
@@ -1,300 +1,31 @@
|
||||
package miniostorage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"syscall"
|
||||
|
||||
"git.company.lan/gopkg/filestore/remote"
|
||||
"github.com/minio/minio-go/v7"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrOutOfRange = errors.New("out of range")
|
||||
ErrNotSupported = errors.New("doesn't support this operation")
|
||||
)
|
||||
// Убеждаемся в том, что мы всегда реализуем интерфейс http.File.
|
||||
var _ http.File = (*minioFileWrapper)(nil)
|
||||
|
||||
var _ remote.File = (*MinioFile)(nil)
|
||||
|
||||
type MinioFile struct {
|
||||
openFlags int
|
||||
offset int64
|
||||
closed bool
|
||||
resource *minioFileResource
|
||||
// minioFileWrapper оборачивает minio.Object для реализации http.File.
|
||||
type minioFileWrapper struct {
|
||||
*minio.Object
|
||||
name string
|
||||
}
|
||||
|
||||
// NewMinioFile
|
||||
func NewMinioFile(ctx context.Context, storage *MinioStorage, openFlags int, fileMode os.FileMode, name string) *MinioFile {
|
||||
return &MinioFile{
|
||||
openFlags: openFlags,
|
||||
// offset: 0,
|
||||
// closed: false,
|
||||
resource: &minioFileResource{
|
||||
ctx: ctx,
|
||||
storage: storage,
|
||||
name: name,
|
||||
fileMode: fileMode,
|
||||
currentIoSize: 0,
|
||||
offset: 0,
|
||||
reader: nil,
|
||||
writer: nil,
|
||||
},
|
||||
}
|
||||
// Readdir требуется для http.File. Для файлов возвращает ошибку.
|
||||
func (f *minioFileWrapper) Readdir(count int) ([]fs.FileInfo, error) {
|
||||
return nil, fs.ErrInvalid
|
||||
}
|
||||
|
||||
// Close
|
||||
func (f *MinioFile) Close() error {
|
||||
if f.closed {
|
||||
return os.ErrClosed
|
||||
}
|
||||
f.closed = true
|
||||
return f.resource.Close()
|
||||
}
|
||||
|
||||
// Read
|
||||
func (f *MinioFile) Read(p []byte) (int, error) {
|
||||
if f.closed {
|
||||
return 0, os.ErrClosed
|
||||
}
|
||||
readed, err := f.resource.ReadAt(p, f.offset)
|
||||
f.offset += int64(readed)
|
||||
return readed, err
|
||||
}
|
||||
|
||||
// Seek
|
||||
func (f *MinioFile) Seek(offset int64, whence int) (int64, error) {
|
||||
if f.closed {
|
||||
return 0, os.ErrClosed
|
||||
}
|
||||
// Since this is an expensive operation; let's make sure we need it
|
||||
if (whence == 0 && offset == f.offset) || (whence == 1 && offset == 0) {
|
||||
return f.offset, nil
|
||||
}
|
||||
// Fore the reader/writers to be reopened (at correct offset)
|
||||
if err := f.Sync(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
stat, err := f.Stat()
|
||||
if err != nil {
|
||||
return 0, nil
|
||||
}
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
f.offset = offset
|
||||
case io.SeekCurrent:
|
||||
f.offset += offset
|
||||
case io.SeekEnd:
|
||||
f.offset = stat.Size() + offset
|
||||
}
|
||||
return f.offset, nil
|
||||
}
|
||||
|
||||
// Write
|
||||
func (f *MinioFile) Write(p []byte) (int, error) {
|
||||
return f.WriteAt(p, f.offset)
|
||||
}
|
||||
|
||||
// WriteAt
|
||||
func (f *MinioFile) WriteAt(b []byte, off int64) (int, error) {
|
||||
if f.closed {
|
||||
return 0, os.ErrClosed
|
||||
}
|
||||
if f.openFlags&os.O_RDONLY != 0 {
|
||||
return 0, fmt.Errorf("file is opened as read only")
|
||||
}
|
||||
written, err := f.resource.WriteAt(b, off)
|
||||
f.offset += int64(written)
|
||||
return written, err
|
||||
}
|
||||
|
||||
// readdirImpl
|
||||
func (f *MinioFile) readdirImpl(count int) ([]*MinioFileInfo, error) {
|
||||
if err := f.Sync(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ownInfo, err := f.Stat()
|
||||
// Stat возвращает FileInfo, требуемый для http.File.
|
||||
func (f *minioFileWrapper) Stat() (fs.FileInfo, error) {
|
||||
info, err := f.Object.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ownInfo.IsDir() {
|
||||
return nil, syscall.ENOTDIR
|
||||
}
|
||||
var res []*MinioFileInfo
|
||||
objs := f.resource.storage.client.ListObjects(f.resource.ctx, f.resource.storage.bucket, minio.ListObjectsOptions{
|
||||
Recursive: true,
|
||||
Prefix: f.resource.name,
|
||||
})
|
||||
for obj := range objs {
|
||||
tmp := NewFileInfoFromAttrs(obj, f.resource.fileMode)
|
||||
if tmp.Name() == "" {
|
||||
// neither object.Name, not object.Prefix were present - so let's skip this unknown thing
|
||||
continue
|
||||
}
|
||||
res = append(res, tmp)
|
||||
}
|
||||
if count > 0 && len(res) > 0 {
|
||||
sort.Sort(ByName(res))
|
||||
res = res[:count]
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Readdir
|
||||
func (f *MinioFile) Readdir(count int) ([]fs.FileInfo, error) {
|
||||
fi, err := f.readdirImpl(count)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var res []fs.FileInfo
|
||||
for _, v := range fi {
|
||||
res = append(res, v)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Readdirnames
|
||||
func (f *MinioFile) Readdirnames(n int) ([]string, error) {
|
||||
fi, err := f.Readdir(n)
|
||||
if err != nil && err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
names := make([]string, len(fi))
|
||||
for i, v := range fi {
|
||||
names[i] = v.Name()
|
||||
}
|
||||
return names, err
|
||||
}
|
||||
|
||||
// Stat
|
||||
func (f *MinioFile) Stat() (os.FileInfo, error) {
|
||||
if err := f.Sync(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stat, err := f.resource.storage.client.StatObject(f.resource.ctx, f.resource.storage.bucket, f.resource.name, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewFileInfoFromAttrs(stat, f.resource.fileMode), nil
|
||||
}
|
||||
|
||||
// Sync
|
||||
func (f *MinioFile) Sync() error {
|
||||
return f.resource.maybeCloseIo()
|
||||
}
|
||||
|
||||
// Truncate
|
||||
func (f *MinioFile) Truncate(_ int64) error {
|
||||
return ErrNotSupported
|
||||
}
|
||||
|
||||
// WriteString
|
||||
func (f *MinioFile) WriteString(s string) (int, error) {
|
||||
return f.Write([]byte(s))
|
||||
}
|
||||
|
||||
type readerAtCloser interface {
|
||||
io.ReadCloser
|
||||
io.ReaderAt
|
||||
}
|
||||
|
||||
type minioFileResource struct {
|
||||
ctx context.Context
|
||||
storage *MinioStorage
|
||||
name string
|
||||
fileMode os.FileMode
|
||||
currentIoSize int64
|
||||
offset int64
|
||||
reader readerAtCloser
|
||||
writer io.WriteCloser
|
||||
closed bool
|
||||
}
|
||||
|
||||
// Close
|
||||
func (r *minioFileResource) Close() error {
|
||||
r.closed = true
|
||||
return r.maybeCloseIo()
|
||||
}
|
||||
|
||||
// maybeCloseIo
|
||||
func (r *minioFileResource) maybeCloseIo() error {
|
||||
if r.reader != nil {
|
||||
if err := r.reader.Close(); err != nil {
|
||||
return fmt.Errorf("error closing reader: %v", err)
|
||||
}
|
||||
r.reader = nil
|
||||
}
|
||||
if r.writer != nil {
|
||||
if err := r.writer.Close(); err != nil {
|
||||
return fmt.Errorf("error closing writer: %v", err)
|
||||
}
|
||||
r.writer = nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadAt
|
||||
func (r *minioFileResource) ReadAt(p []byte, offset int64) (int, error) {
|
||||
if cap(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
// Assume that if the reader is open; it is at the correct offset a good performance assumption that we must ensure holds
|
||||
if offset == r.offset && r.reader != nil {
|
||||
readed, err := r.reader.ReadAt(p, offset)
|
||||
r.offset += int64(readed)
|
||||
return readed, err
|
||||
}
|
||||
// If any writers have written anything; commit it first so we can read it back.
|
||||
if err := r.maybeCloseIo(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
obj, err := r.storage.client.GetObject(r.ctx, r.storage.bucket, r.name, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
r.reader = obj
|
||||
r.offset = offset
|
||||
readed, err := obj.ReadAt(p, offset)
|
||||
r.offset += int64(readed)
|
||||
return readed, err
|
||||
}
|
||||
|
||||
// WriteAt
|
||||
func (r *minioFileResource) WriteAt(b []byte, offset int64) (int, error) {
|
||||
// If the writer is opened and at the correct offset we're good!
|
||||
if offset == r.offset && r.writer != nil {
|
||||
written, err := r.writer.Write(b)
|
||||
r.offset += int64(written)
|
||||
return written, err
|
||||
}
|
||||
// Ensure readers must be re-opened and that if a writer is active at another offset it is first committed before we do a "seek" below
|
||||
if err := r.maybeCloseIo(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// WriteAt to a non existing file
|
||||
if offset > r.currentIoSize {
|
||||
return 0, ErrOutOfRange
|
||||
}
|
||||
r.offset = offset
|
||||
buffer := bytes.NewReader(b)
|
||||
opts := minio.PutObjectOptions{
|
||||
ContentType: http.DetectContentType(b),
|
||||
}
|
||||
if offset > 0 {
|
||||
opts.PartSize = uint64(offset)
|
||||
opts.NumThreads = 8
|
||||
opts.ConcurrentStreamParts = false
|
||||
opts.DisableMultipart = true
|
||||
}
|
||||
if _, err := r.storage.client.PutObject(r.ctx, r.storage.bucket, r.name, buffer, buffer.Size(), opts); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
r.offset += int64(buffer.Len())
|
||||
return buffer.Len(), nil
|
||||
return &minioFileInfo{info: info}, nil
|
||||
}
|
||||
|
||||
@@ -3,94 +3,31 @@ package miniostorage
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
)
|
||||
|
||||
const folderSize = 42
|
||||
// Убеждаемся в том, что мы всегда реализуем интерфейс fs.FileInfo.
|
||||
var _ fs.FileInfo = (*minioFileInfo)(nil)
|
||||
|
||||
var _ fs.FileInfo = (*MinioFileInfo)(nil)
|
||||
|
||||
type MinioFileInfo struct {
|
||||
ETag string
|
||||
name string
|
||||
size int64
|
||||
updated time.Time
|
||||
isDir bool
|
||||
fileMode os.FileMode
|
||||
// minioFileInfo реализует fs.FileInfo.
|
||||
type minioFileInfo struct {
|
||||
info minio.ObjectInfo
|
||||
}
|
||||
|
||||
// NewFileInfoFromAttrs
|
||||
func NewFileInfoFromAttrs(obj minio.ObjectInfo, fileMode os.FileMode) *MinioFileInfo {
|
||||
res := &MinioFileInfo{
|
||||
ETag: obj.ETag,
|
||||
name: obj.Key,
|
||||
size: obj.Size,
|
||||
updated: obj.LastModified,
|
||||
isDir: false,
|
||||
fileMode: fileMode,
|
||||
}
|
||||
|
||||
if res.name == "" {
|
||||
// deals with them at the moment
|
||||
//res.name = "folder"
|
||||
res.size = folderSize
|
||||
res.isDir = true
|
||||
}
|
||||
|
||||
return res
|
||||
func newMinioFileInfo(info minio.ObjectInfo) *minioFileInfo {
|
||||
return &minioFileInfo{info: info}
|
||||
}
|
||||
|
||||
// Name
|
||||
func (fi *MinioFileInfo) Name() string {
|
||||
return filepath.Base(filepath.FromSlash(fi.name))
|
||||
}
|
||||
func (f *minioFileInfo) Name() string { return f.info.Key }
|
||||
|
||||
// Size
|
||||
func (fi *MinioFileInfo) Size() int64 {
|
||||
return fi.size
|
||||
}
|
||||
func (f *minioFileInfo) Size() int64 { return f.info.Size }
|
||||
|
||||
// Mode
|
||||
func (fi *MinioFileInfo) Mode() os.FileMode {
|
||||
if fi.IsDir() {
|
||||
return os.ModeDir | fi.fileMode
|
||||
}
|
||||
return fi.fileMode
|
||||
}
|
||||
func (f *minioFileInfo) Mode() os.FileMode { return 0644 } // MinIO не поддерживает права доступа к файлам. Возвращаем значение по умолчанию.
|
||||
|
||||
// ModTime
|
||||
func (fi *MinioFileInfo) ModTime() time.Time {
|
||||
return fi.updated
|
||||
}
|
||||
func (f *minioFileInfo) ModTime() time.Time { return f.info.LastModified.Local() }
|
||||
|
||||
// IsDir
|
||||
func (fi *MinioFileInfo) IsDir() bool {
|
||||
return fi.isDir
|
||||
}
|
||||
func (f *minioFileInfo) IsDir() bool { return f.info.Key[len(f.info.Key)-1] == '/' }
|
||||
|
||||
// Sys
|
||||
func (fi *MinioFileInfo) Sys() any {
|
||||
return nil
|
||||
}
|
||||
|
||||
type ByName []*MinioFileInfo
|
||||
|
||||
// Len
|
||||
func (a ByName) Len() int { return len(a) }
|
||||
|
||||
// Swap
|
||||
func (a ByName) Swap(i, j int) {
|
||||
a[i].name, a[j].name = a[j].name, a[i].name
|
||||
a[i].size, a[j].size = a[j].size, a[i].size
|
||||
a[i].updated, a[j].updated = a[j].updated, a[i].updated
|
||||
a[i].isDir, a[j].isDir = a[j].isDir, a[i].isDir
|
||||
}
|
||||
|
||||
// Less
|
||||
func (a ByName) Less(i, j int) bool {
|
||||
return strings.Compare(a[i].Name(), a[j].Name()) == -1
|
||||
}
|
||||
func (f *minioFileInfo) Sys() interface{} { return f.info }
|
||||
|
||||
@@ -2,7 +2,7 @@ package miniostorage
|
||||
|
||||
import "net/url"
|
||||
|
||||
// getUserPassword
|
||||
// getUserPassword возвращает имя пользователя и пароль.
|
||||
func getUserPassword(u *url.URL) (string, string) {
|
||||
var user, password string
|
||||
if u.User != nil {
|
||||
|
||||
@@ -2,9 +2,9 @@ package miniostorage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -14,8 +14,7 @@ import (
|
||||
"git.company.lan/gopkg/filestore/remote"
|
||||
)
|
||||
|
||||
const defaultFileMode = 0o755
|
||||
|
||||
// Убеждаемся в том, что мы всегда реализуем интерфейс remote.Storage.
|
||||
var _ remote.Storage = (*MinioStorage)(nil)
|
||||
|
||||
func init() {
|
||||
@@ -23,21 +22,21 @@ func init() {
|
||||
}
|
||||
|
||||
type MinioStorage struct {
|
||||
ctx context.Context
|
||||
client *minio.Client
|
||||
bucket string
|
||||
separator string
|
||||
ctx context.Context
|
||||
client *minio.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
// NewStorage
|
||||
func (s *MinioStorage) NewStorage(ctx context.Context, connString string) (remote.Storage, error) {
|
||||
u, err := url.Parse(connString)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
queries := u.Query()
|
||||
username, password := getUserPassword(u)
|
||||
token := queries.Get("token")
|
||||
|
||||
opts := &minio.Options{
|
||||
Creds: credentials.NewStaticV4(username, password, token),
|
||||
Region: "us-east-1",
|
||||
@@ -52,97 +51,76 @@ func (s *MinioStorage) NewStorage(ctx context.Context, connString string) (remot
|
||||
if queries.Has("region") {
|
||||
opts.Region = queries.Get("region")
|
||||
}
|
||||
|
||||
client, err := minio.New(u.Host, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.ctx = ctx
|
||||
s.client = client
|
||||
s.bucket = u.Path[1:]
|
||||
s.separator = "/"
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// normSeparators will normalize all "\\" and "/" to the provided separator
|
||||
func (s *MinioStorage) normSeparators(str string) string {
|
||||
return strings.ReplaceAll(strings.ReplaceAll(str, "\\", s.separator), "/", s.separator)
|
||||
func (s *MinioStorage) Create(path string, opts ...remote.Option) (io.WriteCloser, error) {
|
||||
return newMinioWriter(s.ctx, s.client, s.bucket, path, opts...), nil
|
||||
}
|
||||
|
||||
// Create
|
||||
func (s *MinioStorage) Create(name string) (remote.File, error) {
|
||||
return s.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0)
|
||||
func (s *MinioStorage) Open(name string) (http.File, error) {
|
||||
obj, err := s.client.GetObject(s.ctx, s.bucket, name, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &minioFileWrapper{Object: obj, name: name}, nil
|
||||
}
|
||||
|
||||
// Open
|
||||
func (s *MinioStorage) Open(name string) (remote.File, error) {
|
||||
return s.OpenFile(name, os.O_RDONLY, 0)
|
||||
func (s *MinioStorage) Remove(path string) error {
|
||||
return s.client.RemoveObject(s.ctx, s.bucket, path, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
// OpenFile
|
||||
func (s *MinioStorage) OpenFile(name string, flag int, fileMode os.FileMode) (remote.File, error) {
|
||||
if flag&os.O_APPEND != 0 {
|
||||
return nil, errors.New("appending files will lead to trouble")
|
||||
func (s *MinioStorage) Stat(path string) (remote.FileInfo, error) {
|
||||
info, err := s.client.StatObject(s.ctx, s.bucket, path, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
name = strings.TrimPrefix(s.normSeparators(name), s.separator)
|
||||
file := NewMinioFile(s.ctx, s, flag, fileMode, name)
|
||||
var err error
|
||||
if flag&os.O_CREATE != 0 {
|
||||
_, err = file.WriteString("")
|
||||
}
|
||||
return file, err
|
||||
return newMinioFileInfo(info), nil
|
||||
}
|
||||
|
||||
// Remove
|
||||
func (s *MinioStorage) Remove(name string) error {
|
||||
name = strings.TrimPrefix(s.normSeparators(name), s.separator)
|
||||
return s.client.RemoveObject(s.ctx, s.bucket, name, minio.RemoveObjectOptions{GovernanceBypass: true})
|
||||
func (s *MinioStorage) Exists(path string) (bool, error) {
|
||||
// Сначала проверяем, является ли путь файлом
|
||||
if ok, err := s.IsFile(path); err == nil && ok {
|
||||
return true, nil
|
||||
}
|
||||
// Если не файл, то проверяем, является ли путь каталогом
|
||||
return s.IsDir(path)
|
||||
}
|
||||
|
||||
// RemoveAll
|
||||
func (s *MinioStorage) RemoveAll(path string) error {
|
||||
path = strings.TrimPrefix(s.normSeparators(path), s.separator)
|
||||
objectsCh := make(chan minio.ObjectInfo)
|
||||
go func() {
|
||||
defer close(objectsCh)
|
||||
opts := minio.ListObjectsOptions{Prefix: path, Recursive: true}
|
||||
for object := range s.client.ListObjects(s.ctx, s.bucket, opts) {
|
||||
if object.Err != nil {
|
||||
panic(object.Err)
|
||||
}
|
||||
objectsCh <- object
|
||||
}
|
||||
}()
|
||||
errorCh := s.client.RemoveObjects(s.ctx, s.bucket, objectsCh, minio.RemoveObjectsOptions{})
|
||||
for e := range errorCh {
|
||||
return errors.New("Failed to remove " + e.ObjectName + ", error: " + e.Err.Error())
|
||||
func (s *MinioStorage) IsDir(path string) (bool, error) {
|
||||
options := minio.ListObjectsOptions{
|
||||
Prefix: strings.TrimRight(path, "/") + "/",
|
||||
Recursive: false,
|
||||
MaxKeys: 1,
|
||||
}
|
||||
return nil
|
||||
objectChan := s.client.ListObjects(s.ctx, s.bucket, options)
|
||||
object, ok := <-objectChan
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
if object.Err != nil {
|
||||
return false, object.Err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Rename
|
||||
func (s *MinioStorage) Rename(oldName, newName string) error {
|
||||
if oldName == newName {
|
||||
return nil
|
||||
func (s *MinioStorage) IsFile(path string) (bool, error) {
|
||||
_, err := s.client.StatObject(s.ctx, s.bucket, path, minio.StatObjectOptions{})
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
oldName = strings.TrimPrefix(s.normSeparators(oldName), s.separator)
|
||||
newName = strings.TrimPrefix(s.normSeparators(newName), s.separator)
|
||||
src := minio.CopySrcOptions{
|
||||
Bucket: s.bucket,
|
||||
Object: oldName,
|
||||
if strings.Contains(err.Error(), "The specified key does not exist.") {
|
||||
return false, nil
|
||||
}
|
||||
dst := minio.CopyDestOptions{
|
||||
Bucket: s.bucket,
|
||||
Object: newName,
|
||||
}
|
||||
if _, err := s.client.CopyObject(s.ctx, dst, src); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Remove(oldName)
|
||||
}
|
||||
|
||||
// Stat
|
||||
func (s *MinioStorage) Stat(name string) (os.FileInfo, error) {
|
||||
name = strings.TrimPrefix(s.normSeparators(name), s.separator)
|
||||
file := NewMinioFile(s.ctx, s, os.O_RDWR, defaultFileMode, name)
|
||||
return file.Stat()
|
||||
return false, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package miniostorage
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"git.company.lan/gopkg/filestore/remote"
|
||||
)
|
||||
|
||||
func (s *MinioStorage) Uploader() remote.Uploader { return s }
|
||||
|
||||
func (s *MinioStorage) Upload(path string, reader io.Reader, opts ...remote.Option) error {
|
||||
file, err := s.Create(path, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := io.Copy(file, reader); err != nil {
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
return file.Close()
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package miniostorage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"git.company.lan/gopkg/filestore/remote"
|
||||
"github.com/minio/minio-go/v7"
|
||||
)
|
||||
|
||||
// Убеждаемся в том, что мы всегда реализуем интерфейс io.WriteCloser.
|
||||
var _ io.WriteCloser = (*minioWriter)(nil)
|
||||
|
||||
// minioWriter реализует интерфейс io.WriteCloser.
|
||||
type minioWriter struct {
|
||||
ctx context.Context
|
||||
client *minio.Client
|
||||
bucket string
|
||||
path string
|
||||
buffer *bytes.Buffer
|
||||
metadata remote.Metadata
|
||||
contentType string
|
||||
}
|
||||
|
||||
func newMinioWriter(ctx context.Context, client *minio.Client, bucket, path string, opts ...remote.Option) *minioWriter {
|
||||
o := &remote.Options{}
|
||||
for _, opt := range opts {
|
||||
opt(o)
|
||||
}
|
||||
|
||||
writer := &minioWriter{
|
||||
ctx: ctx,
|
||||
client: client,
|
||||
bucket: bucket,
|
||||
path: path,
|
||||
buffer: bytes.NewBuffer(nil),
|
||||
}
|
||||
|
||||
if o.ContentType != "" {
|
||||
writer.contentType = o.ContentType
|
||||
}
|
||||
if o.Metadata != nil {
|
||||
writer.metadata = o.Metadata
|
||||
}
|
||||
|
||||
return writer
|
||||
}
|
||||
|
||||
func (w *minioWriter) Write(p []byte) (n int, err error) {
|
||||
select {
|
||||
case <-w.ctx.Done():
|
||||
return 0, w.ctx.Err()
|
||||
default:
|
||||
return w.buffer.Write(p)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *minioWriter) Close() error {
|
||||
select {
|
||||
case <-w.ctx.Done():
|
||||
return w.ctx.Err()
|
||||
default:
|
||||
opts := minio.PutObjectOptions{}
|
||||
|
||||
if w.contentType != "" {
|
||||
opts.ContentType = w.contentType
|
||||
}
|
||||
|
||||
if w.metadata != nil {
|
||||
userMetadata := make(map[string]string)
|
||||
for k, v := range w.metadata {
|
||||
userMetadata[k] = fmt.Sprintf("%v", v)
|
||||
}
|
||||
opts.UserMetadata = userMetadata
|
||||
}
|
||||
|
||||
_, err := w.client.PutObject(
|
||||
w.ctx,
|
||||
w.bucket,
|
||||
w.path,
|
||||
bytes.NewReader(w.buffer.Bytes()),
|
||||
int64(w.buffer.Len()),
|
||||
opts,
|
||||
)
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package remote
|
||||
|
||||
type Option func(*Options)
|
||||
|
||||
type Options struct {
|
||||
Metadata Metadata
|
||||
ContentType string
|
||||
}
|
||||
|
||||
// WithMetadata устанавливает метаданные.
|
||||
func WithMetadata(metadata Metadata) Option {
|
||||
return func(o *Options) {
|
||||
o.Metadata = metadata
|
||||
}
|
||||
}
|
||||
|
||||
// WithContentType устанавливает тип файла.
|
||||
func WithContentType(contentType string) Option {
|
||||
return func(o *Options) {
|
||||
o.ContentType = contentType
|
||||
}
|
||||
}
|
||||
+34
-18
@@ -4,8 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"net/http"
|
||||
"sync"
|
||||
)
|
||||
|
||||
@@ -14,50 +13,67 @@ var (
|
||||
storages = make(map[string]Storage)
|
||||
)
|
||||
|
||||
type File interface {
|
||||
io.Closer
|
||||
io.Reader
|
||||
io.Seeker
|
||||
io.Writer
|
||||
Readdir(count int) ([]fs.FileInfo, error)
|
||||
Stat() (fs.FileInfo, error)
|
||||
type Uploader interface {
|
||||
Upload(path string, reader io.Reader, opts ...Option) error
|
||||
}
|
||||
|
||||
type Storage interface {
|
||||
NewStorage(ctx context.Context, connString string) (Storage, error)
|
||||
Create(name string) (File, error)
|
||||
Open(name string) (File, error)
|
||||
OpenFile(name string, flag int, fileMode os.FileMode) (File, error)
|
||||
Remove(name string) error
|
||||
RemoveAll(path string) error
|
||||
Rename(oldName, newName string) error
|
||||
Stat(name string) (os.FileInfo, error)
|
||||
|
||||
// Open реализует метод http.FileSystem.
|
||||
Open(name string) (http.File, error)
|
||||
|
||||
// Create создаёт файл и возвращает io.WriteCloser.
|
||||
Create(path string, opts ...Option) (io.WriteCloser, error)
|
||||
|
||||
// Remove удаляет файл.
|
||||
Remove(path string) error
|
||||
|
||||
// Stat получает информацию о файле/каталоге.
|
||||
Stat(path string) (FileInfo, error)
|
||||
|
||||
// Exists определяет, существует ли файл или каталог.
|
||||
Exists(path string) (bool, error)
|
||||
|
||||
// IsDir определяет, является ли путь каталогом.
|
||||
IsDir(path string) (bool, error)
|
||||
|
||||
// IsFile определяет, является ли путь файлом.
|
||||
IsFile(path string) (bool, error)
|
||||
|
||||
Uploader() Uploader
|
||||
}
|
||||
|
||||
// NewStorage returns a new remote storage instance.
|
||||
// NewStorage создаёт новый экземпляр удаленного хранилища.
|
||||
func NewStorage(ctx context.Context, connString string) (Storage, error) {
|
||||
scheme, err := schemeFromURL(connString)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
storagesMu.RLock()
|
||||
s, ok := storages[scheme]
|
||||
storagesMu.RUnlock()
|
||||
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown storage %v (forgotten import?)", scheme)
|
||||
}
|
||||
|
||||
return s.NewStorage(ctx, connString)
|
||||
}
|
||||
|
||||
// Register globally registers a storage
|
||||
// Register глобально регистрирует хранилище.
|
||||
func Register(name string, storage Storage) {
|
||||
storagesMu.Lock()
|
||||
defer storagesMu.Unlock()
|
||||
|
||||
if storage == nil {
|
||||
panic("Register storage is nil")
|
||||
}
|
||||
|
||||
if _, exists := storages[name]; exists {
|
||||
panic("Register called twice for storage " + name)
|
||||
}
|
||||
|
||||
storages[name] = storage
|
||||
}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package remote
|
||||
|
||||
import (
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FileInfo реализует интерфейс os.FileInfo.
|
||||
type FileInfo interface {
|
||||
Name() string
|
||||
Size() int64
|
||||
Mode() os.FileMode
|
||||
ModTime() time.Time
|
||||
IsDir() bool
|
||||
Sys() interface{}
|
||||
}
|
||||
|
||||
// Metadata метаданные файла
|
||||
type Metadata map[string]any
|
||||
+15
-17
@@ -20,8 +20,8 @@ const tmpfileName = "<temporary file>"
|
||||
|
||||
// LocalStorage описывает хранилище файлов.
|
||||
type LocalStorage struct {
|
||||
dir string
|
||||
permissions os.FileMode
|
||||
rootDir string
|
||||
perm os.FileMode
|
||||
|
||||
mutexes struct {
|
||||
sync.Mutex
|
||||
@@ -33,9 +33,9 @@ type LocalStorage struct {
|
||||
type LocalStorageOption func(*LocalStorage)
|
||||
|
||||
// WithPermissions
|
||||
func WithPermissions(permissions os.FileMode) LocalStorageOption {
|
||||
return func(storage *LocalStorage) {
|
||||
storage.permissions = permissions
|
||||
func WithPermissions(perm os.FileMode) LocalStorageOption {
|
||||
return func(s *LocalStorage) {
|
||||
s.perm = perm
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,19 +50,17 @@ type FileInfo struct {
|
||||
}
|
||||
|
||||
// NewLocalStorage открывает и возвращает хранилище файлов.
|
||||
func NewLocalStorage(dir string, opts ...LocalStorageOption) (*LocalStorage, error) {
|
||||
func NewLocalStorage(rootDir string, opts ...LocalStorageOption) (*LocalStorage, error) {
|
||||
s := &LocalStorage{}
|
||||
s.dir = dir
|
||||
s.permissions = 0700
|
||||
s.rootDir = rootDir
|
||||
s.perm = 0700
|
||||
|
||||
for _, opt := range opts {
|
||||
if opt != nil {
|
||||
opt(s)
|
||||
}
|
||||
opt(s)
|
||||
}
|
||||
|
||||
// Создаём каталог, если он ещё не создан
|
||||
if err := os.MkdirAll(s.dir, s.permissions); err != nil {
|
||||
if err := os.MkdirAll(s.rootDir, s.perm); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -72,7 +70,7 @@ func NewLocalStorage(dir string, opts ...LocalStorageOption) (*LocalStorage, err
|
||||
// Create сохраняет файл в хранилище. В качестве имени файла используется комбинация из двух хешей.
|
||||
func (s *LocalStorage) Create(r io.Reader) (*FileInfo, error) {
|
||||
// Создаём временный файл в корневом каталоге
|
||||
tmpfile, err := os.CreateTemp(s.dir, "~tmp")
|
||||
tmpfile, err := os.CreateTemp(s.rootDir, "~tmp")
|
||||
if err != nil {
|
||||
err.(*os.PathError).Path = tmpfileName // Подмениваем имя файла
|
||||
return nil, err
|
||||
@@ -127,7 +125,7 @@ func (s *LocalStorage) Create(r io.Reader) (*FileInfo, error) {
|
||||
}
|
||||
|
||||
// Если такого файла нет, то создаем для него каталог
|
||||
if err := os.MkdirAll(filepath.Dir(fi.Location), s.permissions); err != nil {
|
||||
if err := os.MkdirAll(filepath.Dir(fi.Location), s.perm); err != nil {
|
||||
err.(*os.PathError).Path = fi.Name
|
||||
return nil, err
|
||||
}
|
||||
@@ -212,7 +210,7 @@ func (s *LocalStorage) Remove(name string) error {
|
||||
func (s *LocalStorage) Clean(lifetime time.Duration) error {
|
||||
// Удаляем вообще все файлы, если время жизни не задано
|
||||
if lifetime <= 0 {
|
||||
files, err := filepath.Glob(filepath.Join(s.dir, "*"))
|
||||
files, err := filepath.Glob(filepath.Join(s.rootDir, "*"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -226,7 +224,7 @@ func (s *LocalStorage) Clean(lifetime time.Duration) error {
|
||||
// Вычисляем крайнюю дату валидности файлов
|
||||
valid := time.Now().Add(-lifetime)
|
||||
|
||||
err := filepath.Walk(s.dir,
|
||||
err := filepath.Walk(s.rootDir,
|
||||
func(filename string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -282,7 +280,7 @@ func (s *LocalStorage) GetFullName(name string) string {
|
||||
if len(name) < 27 {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(s.dir, name[:1], name[1:3], name[3:])
|
||||
return filepath.Join(s.rootDir, name[:1], name[1:3], name[3:])
|
||||
}
|
||||
|
||||
// IsExists проверяет: существует ли файл в хранилище?
|
||||
|
||||
Reference in New Issue
Block a user