store/attachment.go
Rama main · réplica de lectura del repositorio original.
package store
import (
"context"
"log/slog"
"os"
"path/filepath"
"github.com/pkg/errors"
"github.com/usememos/memos/internal/identifier"
storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/provider/storage"
)
type Attachment struct {
// ID is the system generated unique identifier for the attachment.
ID int32
// UID is the user defined unique identifier for the attachment.
UID string
// Standard fields
CreatorID int32
CreatedTs int64
UpdatedTs int64
// Domain specific fields
Filename string
Blob []byte
Type string
Size int64
StorageType storepb.AttachmentStorageType
Reference string
Payload *storepb.AttachmentPayload
// The related memo ID.
MemoID *int32
// Policy is present for transport-facing direct-link mutations. Drivers
// revalidate it in the same transaction as the attachment insert.
Policy *MemoWritePolicy
// Composed field
MemoUID *string
CreatorUsername *string
}
type FindAttachment struct {
GetBlob bool
ID *int32
UID *string
CreatorID *int32
Filename *string
FilenameSearch *string
MemoID *int32
MemoIDList []int32
HasRelatedMemo bool
Filters []string
// Access applies memo-local authorization to linked attachments
// before pagination. Unlinked attachments are visible only to the active
// matching creator. Nil is reserved for trusted internal callers.
Access *MemoAccessScope
Limit *int
Offset *int
SkipDefaultLimit bool
}
type UpdateAttachment struct {
ID int32
UID *string
UpdatedTs *int64
Filename *string
MemoID *int32
Payload *storepb.AttachmentPayload
// Content replaces the stored file. The caller has already written the new
// object; the previous one is the caller's to delete once this commits.
Content *AttachmentContent
// Policy is present for transport-facing updates. Drivers discover the
// current attachment binding and authorize any linked memo in the same
// transaction as the update.
Policy *MemoWritePolicy
}
// AttachmentContent is where an attachment's file is stored and what it is.
type AttachmentContent struct {
Type string
Size int64
StorageType storepb.AttachmentStorageType
// Reference is the local path of a LOCAL file; empty otherwise.
Reference string
// Blob holds the file of a DATABASE attachment; nil otherwise.
Blob []byte
}
type DeleteAttachment struct {
ID int32
MemoID *int32
}
const (
thumbnailCacheFolder = ".thumbnail_cache"
motionCacheFolder = ".motion_cache"
)
type deleteAttachmentStorageFailpointKey struct{}
type createAttachmentPolicyFailpointKey struct{}
type createAttachmentPostCommitFailpointKey struct{}
// ErrDeleteAttachmentStorageFailpoint is returned by the test-only attachment storage failpoint.
var ErrDeleteAttachmentStorageFailpoint = errors.New("delete attachment storage failpoint")
// ErrCreateAttachmentPostCommitFailpoint is returned after the test-only attachment create failpoint persists a row.
var ErrCreateAttachmentPostCommitFailpoint = errors.New("create attachment post-commit failpoint")
// WithDeleteAttachmentStorageFailpoint forces DeleteAttachmentStorage to return a failpoint error.
func WithDeleteAttachmentStorageFailpoint(ctx context.Context) context.Context {
return context.WithValue(ctx, deleteAttachmentStorageFailpointKey{}, true)
}
// WithCreateAttachmentPolicyFailpoint forces a policy-bearing attachment create to fail before its database insert.
func WithCreateAttachmentPolicyFailpoint(ctx context.Context) context.Context {
return context.WithValue(ctx, createAttachmentPolicyFailpointKey{}, true)
}
// WithCreateAttachmentPostCommitFailpoint forces CreateAttachment to return an error after its database insert succeeds.
func WithCreateAttachmentPostCommitFailpoint(ctx context.Context) context.Context {
return context.WithValue(ctx, createAttachmentPostCommitFailpointKey{}, true)
}
func (s *Store) CreateAttachment(ctx context.Context, create *Attachment) (*Attachment, error) {
if !identifier.UIDMatcher.MatchString(create.UID) {
return nil, errors.New("invalid uid")
}
if err := validateMemoWritePolicy(create.Policy); err != nil {
return nil, err
}
var (
attachment *Attachment
err error
)
if create.Policy != nil {
if create.MemoID == nil {
return nil, errors.New("attachment write policy requires a memo")
}
if create.CreatorID != create.Policy.ActorUserID {
return nil, ErrMemoPermissionDenied
}
if shouldFailCreateAttachmentPolicy(ctx) {
return nil, ErrMemoSpaceMembershipRequired
}
attachment, err = s.driver.CreateAttachment(ctx, create)
} else {
attachment, err = s.driver.CreateAttachment(ctx, create)
}
if err != nil {
return nil, err
}
if shouldFailCreateAttachmentPostCommit(ctx) {
return nil, ErrCreateAttachmentPostCommitFailpoint
}
return attachment, nil
}
func (s *Store) ListAttachments(ctx context.Context, find *FindAttachment) ([]*Attachment, error) {
// Set default limits to prevent loading too many attachments at once
shouldApplyDefaultLimit := find.Limit == nil && find.MemoID == nil && len(find.MemoIDList) == 0 && !find.SkipDefaultLimit
if shouldApplyDefaultLimit && find.GetBlob {
// When fetching blobs, we should be especially careful with limits
defaultLimit := 10
find.Limit = &defaultLimit
} else if shouldApplyDefaultLimit {
// Even without blobs, let's default to a reasonable limit
defaultLimit := 100
find.Limit = &defaultLimit
}
return s.driver.ListAttachments(ctx, find)
}
func (s *Store) GetAttachment(ctx context.Context, find *FindAttachment) (*Attachment, error) {
attachments, err := s.ListAttachments(ctx, find)
if err != nil {
return nil, err
}
if len(attachments) == 0 {
return nil, nil
}
return attachments[0], nil
}
func (s *Store) UpdateAttachment(ctx context.Context, update *UpdateAttachment) error {
if update.UID != nil && !identifier.UIDMatcher.MatchString(*update.UID) {
return errors.New("invalid uid")
}
if err := validateMemoWritePolicy(update.Policy); err != nil {
return err
}
if update.Policy != nil {
if update.MemoID != nil {
return errors.New("policy-bearing attachment updates cannot change memo binding")
}
return s.driver.UpdateAttachment(ctx, update)
}
return s.driver.UpdateAttachment(ctx, update)
}
func (s *Store) DeleteAttachment(ctx context.Context, delete *DeleteAttachment) error {
attachment, err := s.GetAttachment(ctx, &FindAttachment{ID: &delete.ID})
if err != nil {
return errors.Wrap(err, "failed to get attachment")
}
if attachment == nil {
return errors.New("attachment not found")
}
if err := s.DeleteAttachmentStorage(ctx, attachment); err != nil {
if attachment.StorageType == storepb.AttachmentStorageType_LOCAL {
return errors.Wrap(err, "failed to delete local file")
}
slog.Warn("Failed to delete attachment storage", slog.Any("err", err))
}
return s.driver.DeleteAttachment(ctx, delete)
}
func (s *Store) DeleteAttachments(ctx context.Context, attachments []*Attachment) error {
if len(attachments) == 0 {
return nil
}
deletes := make([]*DeleteAttachment, 0, len(attachments))
for _, attachment := range attachments {
if attachment == nil {
continue
}
deletes = append(deletes, &DeleteAttachment{ID: attachment.ID, MemoID: attachment.MemoID})
}
if len(deletes) == 0 {
return nil
}
if err := s.driver.DeleteAttachments(ctx, deletes); err != nil {
return err
}
return s.deleteAttachmentStorageSnapshots(ctx, attachments)
}
func (s *Store) deleteAttachmentStorageSnapshots(ctx context.Context, attachments []*Attachment) error {
instanceStorageSetting, instanceStorageSettingErr := s.getAttachmentStorageCleanupInstanceSetting(ctx, attachments)
for _, attachment := range attachments {
if attachment == nil {
continue
}
var err error
if instanceStorageSettingErr != nil && AttachmentNeedsInstanceStorageSetting(attachment) {
err = instanceStorageSettingErr
} else {
err = s.deleteAttachmentStorageImpl(ctx, attachment, instanceStorageSetting)
}
if err != nil {
if attachment.StorageType == storepb.AttachmentStorageType_LOCAL {
return errors.Wrap(err, "failed to delete local file")
}
slog.Warn("Failed to delete attachment storage", slog.Any("err", err))
}
}
return nil
}
func (s *Store) DeleteAttachmentStorage(ctx context.Context, attachment *Attachment) error {
return s.deleteAttachmentStorageImpl(ctx, attachment, nil)
}
// DeleteAttachmentStorageWithInstanceSetting deletes attachment storage using a preloaded instance storage setting.
func (s *Store) DeleteAttachmentStorageWithInstanceSetting(ctx context.Context, attachment *Attachment, instanceStorageSetting *storepb.InstanceStorageSetting) error {
return s.deleteAttachmentStorageImpl(ctx, attachment, instanceStorageSetting)
}
func (s *Store) deleteAttachmentStorageImpl(ctx context.Context, attachment *Attachment, instanceStorageSetting *storepb.InstanceStorageSetting) error {
if attachment == nil {
return nil
}
if shouldFailDeleteAttachmentStorage(ctx) {
return ErrDeleteAttachmentStorageFailpoint
}
if attachment.StorageType == storepb.AttachmentStorageType_LOCAL {
if err := func() error {
p := filepath.FromSlash(attachment.Reference)
if !filepath.IsAbs(p) {
p = filepath.Join(s.profile.Data, p)
}
err := os.Remove(p)
if err != nil && !os.IsNotExist(err) {
return errors.Wrap(err, "failed to delete local file")
}
return nil
}(); err != nil {
return err
}
} else if attachment.StorageType == storepb.AttachmentStorageType_S3 {
if err := func() error {
s3ObjectPayload := attachment.Payload.GetS3Object()
if s3ObjectPayload == nil {
return errors.Errorf("No s3 object found")
}
// Deletes resolve with the registry like reads do, so a legacy
// attachment picks up rotated credentials for its namespace instead
// of deleting with the stale embedded config.
if instanceStorageSetting == nil {
var err error
instanceStorageSetting, err = s.GetInstanceStorageSetting(ctx)
if err != nil {
if AttachmentNeedsInstanceStorageSetting(attachment) {
return errors.Wrap(err, "failed to get instance storage setting")
}
// The embedded config is sufficient on its own; keep the
// delete best-effort when the settings read fails.
instanceStorageSetting = nil
}
}
resolvedStorage, err := ResolveStorage(instanceStorageSetting, s3ObjectPayload.StorageId, s3ObjectPayload.S3Config)
if err != nil {
return errors.Wrap(err, "failed to resolve storage")
}
driver, err := s.StorageDriver(ctx, resolvedStorage)
if err != nil {
return errors.Wrap(err, "failed to create storage driver")
}
if err := driver.DeleteObject(ctx, s3ObjectPayload.Key); err != nil {
return errors.Wrap(err, "Failed to delete s3 object")
}
return nil
}(); err != nil {
return err
}
}
s.DeleteAttachmentDerivedCaches(attachment)
return nil
}
func (s *Store) getAttachmentStorageCleanupInstanceSetting(ctx context.Context, attachments []*Attachment) (*storepb.InstanceStorageSetting, error) {
for _, attachment := range attachments {
if AttachmentNeedsInstanceStorageSetting(attachment) {
instanceStorageSetting, err := s.GetInstanceStorageSetting(ctx)
if err != nil {
return nil, errors.Wrap(err, "failed to get instance storage setting")
}
return instanceStorageSetting, nil
}
}
return nil, nil
}
// ResolveAttachmentS3Driver resolves the storage driver referenced by an S3
// attachment payload, validating the payload and supplying the instance setting.
func (s *Store) ResolveAttachmentS3Driver(ctx context.Context, attachment *Attachment) (storage.Driver, *storepb.AttachmentPayload_S3Object, error) {
if attachment == nil {
return nil, nil, errors.New("attachment is missing")
}
if attachment.Payload == nil {
return nil, nil, errors.New("attachment payload is missing")
}
s3Object := attachment.Payload.GetS3Object()
if s3Object == nil {
return nil, nil, errors.New("S3 object payload is missing")
}
if s3Object.Key == "" {
return nil, nil, errors.New("S3 object key is missing")
}
instanceStorageSetting, err := s.GetInstanceStorageSetting(ctx)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to get instance storage setting")
}
resolvedStorage, err := ResolveStorage(instanceStorageSetting, s3Object.StorageId, s3Object.S3Config)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to resolve storage")
}
driver, err := s.StorageDriver(ctx, resolvedStorage)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to create storage driver")
}
return driver, s3Object, nil
}
// AttachmentNeedsInstanceStorageSetting reports whether cleanup should load
// the configured storage registry for an S3 attachment.
func AttachmentNeedsInstanceStorageSetting(attachment *Attachment) bool {
if attachment == nil || attachment.StorageType != storepb.AttachmentStorageType_S3 {
return false
}
return attachment.Payload.GetS3Object() != nil
}
// DeleteAttachmentDerivedCaches removes every derived asset the file server
// may have written for the attachment: the current thumbnail, its failure
// marker, the pre-v2 thumbnail name, and the extracted motion clip. The names
// must match server/fileserver.
func (s *Store) DeleteAttachmentDerivedCaches(attachment *Attachment) {
for _, cachePath := range []string{
filepath.Join(s.profile.Data, thumbnailCacheFolder, attachment.UID+".v2.jpeg"),
filepath.Join(s.profile.Data, thumbnailCacheFolder, attachment.UID+".v2.jpeg.failed"),
filepath.Join(s.profile.Data, thumbnailCacheFolder, attachment.UID+".jpeg"),
filepath.Join(s.profile.Data, motionCacheFolder, attachment.UID+".mp4"),
} {
if err := os.Remove(cachePath); err != nil && !os.IsNotExist(err) {
slog.Warn("Failed to delete derived attachment cache", slog.String("path", cachePath), slog.Any("err", err))
}
}
}
func shouldFailDeleteAttachmentStorage(ctx context.Context) bool {
failpoint, ok := ctx.Value(deleteAttachmentStorageFailpointKey{}).(bool)
return ok && failpoint
}
func shouldFailCreateAttachmentPolicy(ctx context.Context) bool {
failpoint, ok := ctx.Value(createAttachmentPolicyFailpointKey{}).(bool)
return ok && failpoint
}
func shouldFailCreateAttachmentPostCommit(ctx context.Context) bool {
failpoint, ok := ctx.Value(createAttachmentPostCommitFailpointKey{}).(bool)
return ok && failpoint
}
// GetAttachmentStorageUsage returns the total persisted attachment size for a
// creator in bytes. External links are not counted.
func (s *Store) GetAttachmentStorageUsage(ctx context.Context, creatorID int32) (int64, error) {
return s.driver.GetAttachmentStorageUsage(ctx, creatorID)
}