From ad22d950dde751acc828f7c37d8855396f1dc406 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 3 Feb 2025 13:18:52 +0100 Subject: [PATCH] Remove `bimg` thumbnailer (#3522) As it is most likely not used anyway. (It's not the default) --- go.mod | 1 - go.sum | 2 - mediaapi/README.md | 27 --- mediaapi/mediaapi.go | 3 +- mediaapi/thumbnailer/thumbnailer.go | 3 +- mediaapi/thumbnailer/thumbnailer_bimg.go | 241 ----------------------- mediaapi/thumbnailer/thumbnailer_nfnt.go | 11 +- 7 files changed, 8 insertions(+), 280 deletions(-) delete mode 100644 mediaapi/README.md delete mode 100644 mediaapi/thumbnailer/thumbnailer_bimg.go diff --git a/go.mod b/go.mod index 0aab23ea..b1fe33b7 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,6 @@ require ( golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b golang.org/x/sync v0.10.0 golang.org/x/term v0.28.0 - gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 gotest.tools/v3 v3.5.1 maunium.net/go/mautrix v0.15.1 diff --git a/go.sum b/go.sum index 353f1e84..0ff63744 100644 --- a/go.sum +++ b/go.sum @@ -488,8 +488,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/h2non/bimg.v1 v1.1.9 h1:wZIUbeOnwr37Ta4aofhIv8OI8v4ujpjXC9mXnAGpQjM= -gopkg.in/h2non/bimg.v1 v1.1.9/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= diff --git a/mediaapi/README.md b/mediaapi/README.md deleted file mode 100644 index baf5587f..00000000 --- a/mediaapi/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Media API - -This server is responsible for serving `/media` requests as per: - -http://matrix.org/docs/spec/client_server/r0.2.0.html#id43 - -## Scaling libraries - -### nfnt/resize (default) - -Thumbnailing uses https://github.com/nfnt/resize by default which is a pure golang image scaling library relying on image codecs from the standard library. It is ISC-licensed. - -It is multi-threaded and uses Lanczos3 so produces sharp images. Using Lanczos3 all the way makes it slower than some other approaches like bimg. (~845ms in total for pre-generating 32x32-crop, 96x96-crop, 320x240-scale, 640x480-scale and 800x600-scale from a given JPEG image on a given machine.) - -See the sample below for image quality with nfnt/resize: - -![](nfnt-96x96-crop.jpg) - -### bimg (uses libvips C library) - -Alternatively one can use `go build -tags bimg` to use bimg from https://github.com/h2non/bimg (MIT-licensed) which uses libvips from https://github.com/jcupitt/libvips (LGPL v2.1+ -licensed). libvips is a C library and must be installed/built separately. See the github page for details. Also note that libvips in turn has dependencies with a selection of FOSS licenses. - -bimg and libvips have significantly better performance than nfnt/resize but produce slightly less-sharp images. bimg uses a box filter for downscaling to within about 200% of the target scale and then uses Lanczos3 for the last bit. This is a much faster approach but comes at the expense of sharpness. (~295ms in total for pre-generating 32x32-crop, 96x96-crop, 320x240-scale, 640x480-scale and 800x600-scale from a given JPEG image on a given machine.) - -See the sample below for image quality with bimg: - -![](bimg-96x96-crop.jpg) diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index ac20c886..30700932 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -7,6 +7,8 @@ package mediaapi import ( + "github.com/sirupsen/logrus" + "github.com/element-hq/dendrite/internal/httputil" "github.com/element-hq/dendrite/internal/sqlutil" "github.com/element-hq/dendrite/mediaapi/routing" @@ -15,7 +17,6 @@ import ( userapi "github.com/element-hq/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/fclient" - "github.com/sirupsen/logrus" ) // AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component. diff --git a/mediaapi/thumbnailer/thumbnailer.go b/mediaapi/thumbnailer/thumbnailer.go index 88857f69..82d7f092 100644 --- a/mediaapi/thumbnailer/thumbnailer.go +++ b/mediaapi/thumbnailer/thumbnailer.go @@ -14,10 +14,11 @@ import ( "path/filepath" "sync" + log "github.com/sirupsen/logrus" + "github.com/element-hq/dendrite/mediaapi/storage" "github.com/element-hq/dendrite/mediaapi/types" "github.com/element-hq/dendrite/setup/config" - log "github.com/sirupsen/logrus" ) type thumbnailFitness struct { diff --git a/mediaapi/thumbnailer/thumbnailer_bimg.go b/mediaapi/thumbnailer/thumbnailer_bimg.go deleted file mode 100644 index 8bb8b2d2..00000000 --- a/mediaapi/thumbnailer/thumbnailer_bimg.go +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright 2024 New Vector Ltd. -// Copyright 2017 Vector Creations Ltd -// -// SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -// Please see LICENSE files in the repository root for full details. - -//go:build bimg -// +build bimg - -package thumbnailer - -import ( - "context" - "os" - "time" - - "github.com/element-hq/dendrite/mediaapi/storage" - "github.com/element-hq/dendrite/mediaapi/types" - "github.com/element-hq/dendrite/setup/config" - log "github.com/sirupsen/logrus" - "gopkg.in/h2non/bimg.v1" -) - -// GenerateThumbnails generates the configured thumbnail sizes for the source file -func GenerateThumbnails( - ctx context.Context, - src types.Path, - configs []config.ThumbnailSize, - mediaMetadata *types.MediaMetadata, - activeThumbnailGeneration *types.ActiveThumbnailGeneration, - maxThumbnailGenerators int, - db storage.Database, - logger *log.Entry, -) (busy bool, errorReturn error) { - buffer, err := bimg.Read(string(src)) - if err != nil { - logger.WithError(err).WithField("src", src).Error("Failed to read src file") - return false, err - } - img := bimg.NewImage(buffer) - for _, config := range configs { - // Note: createThumbnail does locking based on activeThumbnailGeneration - busy, err = createThumbnail( - ctx, src, img, types.ThumbnailSize(config), mediaMetadata, activeThumbnailGeneration, - maxThumbnailGenerators, db, logger, - ) - if err != nil { - logger.WithError(err).WithField("src", src).Error("Failed to generate thumbnails") - return false, err - } - if busy { - return true, nil - } - } - return false, nil -} - -// GenerateThumbnail generates the configured thumbnail size for the source file -func GenerateThumbnail( - ctx context.Context, - src types.Path, - config types.ThumbnailSize, - mediaMetadata *types.MediaMetadata, - activeThumbnailGeneration *types.ActiveThumbnailGeneration, - maxThumbnailGenerators int, - db storage.Database, - logger *log.Entry, -) (busy bool, errorReturn error) { - buffer, err := bimg.Read(string(src)) - if err != nil { - logger.WithError(err).WithFields(log.Fields{ - "src": src, - }).Error("Failed to read src file") - return false, err - } - img := bimg.NewImage(buffer) - // Note: createThumbnail does locking based on activeThumbnailGeneration - busy, err = createThumbnail( - ctx, src, img, config, mediaMetadata, activeThumbnailGeneration, - maxThumbnailGenerators, db, logger, - ) - if err != nil { - logger.WithError(err).WithFields(log.Fields{ - "src": src, - }).Error("Failed to generate thumbnails") - return false, err - } - if busy { - return true, nil - } - return false, nil -} - -// createThumbnail checks if the thumbnail exists, and if not, generates it -// Thumbnail generation is only done once for each non-existing thumbnail. -func createThumbnail( - ctx context.Context, - src types.Path, - img *bimg.Image, - config types.ThumbnailSize, - mediaMetadata *types.MediaMetadata, - activeThumbnailGeneration *types.ActiveThumbnailGeneration, - maxThumbnailGenerators int, - db storage.Database, - logger *log.Entry, -) (busy bool, errorReturn error) { - logger = logger.WithFields(log.Fields{ - "Width": config.Width, - "Height": config.Height, - "ResizeMethod": config.ResizeMethod, - }) - - // Check if request is larger than original - if isLargerThanOriginal(config, img) { - return false, nil - } - - dst := GetThumbnailPath(src, config) - - // Note: getActiveThumbnailGeneration uses mutexes and conditions from activeThumbnailGeneration - isActive, busy, err := getActiveThumbnailGeneration(dst, config, activeThumbnailGeneration, maxThumbnailGenerators, logger) - if err != nil { - return false, err - } - if busy { - return true, nil - } - - if isActive { - // Note: This is an active request that MUST broadcastGeneration to wake up waiting goroutines! - // Note: broadcastGeneration uses mutexes and conditions from activeThumbnailGeneration - defer func() { - // Note: errorReturn is the named return variable so we wrap this in a closure to re-evaluate the arguments at defer-time - if err := recover(); err != nil { - broadcastGeneration(dst, activeThumbnailGeneration, config, err.(error), logger) - panic(err) - } - broadcastGeneration(dst, activeThumbnailGeneration, config, errorReturn, logger) - }() - } - - exists, err := isThumbnailExists(ctx, dst, config, mediaMetadata, db, logger) - if err != nil || exists { - return false, err - } - - start := time.Now() - width, height, err := resize(dst, img, config.Width, config.Height, config.ResizeMethod == "crop", logger) - if err != nil { - return false, err - } - logger.WithFields(log.Fields{ - "ActualWidth": width, - "ActualHeight": height, - "processTime": time.Now().Sub(start), - }).Debugf("Generated thumbnail %q", dst) - - stat, err := os.Stat(string(dst)) - if err != nil { - return false, err - } - - thumbnailMetadata := &types.ThumbnailMetadata{ - MediaMetadata: &types.MediaMetadata{ - MediaID: mediaMetadata.MediaID, - Origin: mediaMetadata.Origin, - // Note: the code currently always creates a JPEG thumbnail - ContentType: types.ContentType("image/jpeg"), - FileSizeBytes: types.FileSizeBytes(stat.Size()), - }, - ThumbnailSize: types.ThumbnailSize{ - Width: config.Width, - Height: config.Height, - ResizeMethod: config.ResizeMethod, - }, - } - - err = db.StoreThumbnail(ctx, thumbnailMetadata) - if err != nil { - logger.WithError(err).WithFields(log.Fields{ - "ActualWidth": width, - "ActualHeight": height, - }).Error("Failed to store thumbnail metadata in database.") - return false, err - } - - return false, nil -} - -func isLargerThanOriginal(config types.ThumbnailSize, img *bimg.Image) bool { - imgSize, err := img.Size() - if err == nil && config.Width >= imgSize.Width && config.Height >= imgSize.Height { - return true - } - return false -} - -// resize scales an image to fit within the provided width and height -// If the source aspect ratio is different to the target dimensions, one edge will be smaller than requested -// If crop is set to true, the image will be scaled to fill the width and height with any excess being cropped off -func resize(dst types.Path, inImage *bimg.Image, w, h int, crop bool, logger *log.Entry) (int, int, error) { - inSize, err := inImage.Size() - if err != nil { - return -1, -1, err - } - - options := bimg.Options{ - Type: bimg.JPEG, - Quality: 85, - } - if crop { - options.Width = w - options.Height = h - options.Crop = true - } else { - inAR := float64(inSize.Width) / float64(inSize.Height) - outAR := float64(w) / float64(h) - - if inAR > outAR { - // input has wider AR than requested output so use requested width and calculate height to match input AR - options.Width = w - options.Height = int(float64(w) / inAR) - } else { - // input has narrower AR than requested output so use requested height and calculate width to match input AR - options.Width = int(float64(h) * inAR) - options.Height = h - } - } - - newImage, err := inImage.Process(options) - if err != nil { - return -1, -1, err - } - - if err = bimg.Write(string(dst), newImage); err != nil { - logger.WithError(err).Error("Failed to resize image") - return -1, -1, err - } - - return options.Width, options.Height, nil -} diff --git a/mediaapi/thumbnailer/thumbnailer_nfnt.go b/mediaapi/thumbnailer/thumbnailer_nfnt.go index bd1e820b..144e385d 100644 --- a/mediaapi/thumbnailer/thumbnailer_nfnt.go +++ b/mediaapi/thumbnailer/thumbnailer_nfnt.go @@ -4,9 +4,6 @@ // SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial // Please see LICENSE files in the repository root for full details. -//go:build !bimg -// +build !bimg - package thumbnailer import ( @@ -20,18 +17,18 @@ import ( // Imported for png codec _ "image/png" + "os" + "time" // Imported for webp codec _ "golang.org/x/image/webp" - "os" - "time" + "github.com/nfnt/resize" + log "github.com/sirupsen/logrus" "github.com/element-hq/dendrite/mediaapi/storage" "github.com/element-hq/dendrite/mediaapi/types" "github.com/element-hq/dendrite/setup/config" - "github.com/nfnt/resize" - log "github.com/sirupsen/logrus" ) // GenerateThumbnails generates the configured thumbnail sizes for the source file