From 8fa0fcecfa3c9e731f9c4424eb257ce09c3a0119 Mon Sep 17 00:00:00 2001 From: adnull Date: Tue, 21 Jan 2025 11:57:44 +0100 Subject: [PATCH] added federation api allow/deny ip list checking Signed-off-by: adnull --- go.mod | 2 + go.sum | 48 +++++++++++++++++- internal/netcontext.go | 76 ++++++++++++++++++++++++++++ mediaapi/routing/routing.go | 10 +++- mediaapi/routing/url_preview.go | 39 +++++++------- mediaapi/routing/url_preview_test.go | 70 ++++++++++++++++++++++--- setup/config/config_mediaapi.go | 4 +- 7 files changed, 219 insertions(+), 30 deletions(-) create mode 100644 internal/netcontext.go diff --git a/go.mod b/go.mod index 111004fe..da97d3d7 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/docker/go-connections v0.5.0 github.com/eyedeekay/goSam v0.32.54 github.com/eyedeekay/onramp v0.33.8 + github.com/foxcpp/go-mockdns v1.1.0 github.com/getsentry/sentry-go v0.14.0 github.com/gologme/log v1.3.0 github.com/google/go-cmp v0.6.0 @@ -108,6 +109,7 @@ require ( github.com/klauspost/compress v1.17.11 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/miekg/dns v1.1.57 // indirect github.com/minio/highwayhash v1.0.3 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/term v0.5.0 // indirect diff --git a/go.sum b/go.sum index 19381492..7729f234 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,8 @@ github.com/eyedeekay/sam3 v0.33.8/go.mod h1:ytbwLYLJlW6UA92Ffyc6oioWTKnGeeUMr9CL github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI= +github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= @@ -248,8 +250,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= -github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= +github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q= github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= @@ -364,6 +366,7 @@ github.com/yggdrasil-network/yggquic v0.0.0-20241212194307-0d495106021f h1:nqinj github.com/yggdrasil-network/yggquic v0.0.0-20241212194307-0d495106021f/go.mod h1:TVCKOUWiXR9cAqr3eDpKvXkVkTph38xwk0wjcvfrtKI= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= @@ -392,6 +395,10 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -412,6 +419,10 @@ golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b/go.mod h1:EiXZlVfUTaAyySF golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -420,11 +431,22 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -436,22 +458,40 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -464,6 +504,10 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/netcontext.go b/internal/netcontext.go new file mode 100644 index 00000000..1577eaa8 --- /dev/null +++ b/internal/netcontext.go @@ -0,0 +1,76 @@ +package internal + +import ( + "context" + "fmt" + "net" + "syscall" + "time" +) + +var ( + ErrDeniedAddress = fmt.Errorf("address is denied") +) + +func GetDialer(allowNetworks []string, denyNetworks []string, dialTimeout time.Duration) *net.Dialer { + if len(allowNetworks) == 0 && len(denyNetworks) == 0 { + return &net.Dialer{ + Timeout: dialTimeout, + } + } + + return &net.Dialer{ + Timeout: time.Second * 5, + ControlContext: allowDenyNetworksControl(allowNetworks, denyNetworks), + } +} + +// allowDenyNetworksControl is used to allow/deny access to certain networks +func allowDenyNetworksControl(allowNetworks, denyNetworks []string) func(_ context.Context, network string, address string, conn syscall.RawConn) error { + return func(_ context.Context, network string, address string, conn syscall.RawConn) error { + if network != "tcp4" && network != "tcp6" { + return fmt.Errorf("%s is not a safe network type", network) + } + + host, _, err := net.SplitHostPort(address) + if err != nil { + return fmt.Errorf("%s is not a valid host/port pair: %s", address, err) + } + + ipaddress := net.ParseIP(host) + if ipaddress == nil { + return fmt.Errorf("%s is not a valid IP address", host) + } + + if !isAllowed(ipaddress, allowNetworks, denyNetworks) { + return ErrDeniedAddress + } + + return nil // allow connection + } +} + +func isAllowed(ip net.IP, allowCIDRs []string, denyCIDRs []string) bool { + if inRange(ip, denyCIDRs) { + return false + } + if inRange(ip, allowCIDRs) { + return true + } + return false // "should never happen" +} + +func inRange(ip net.IP, CIDRs []string) bool { + for i := 0; i < len(CIDRs); i++ { + cidr := CIDRs[i] + _, network, err := net.ParseCIDR(cidr) + if err != nil { + return false + } + if network.Contains(ip) { + return true + } + } + + return false +} diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 43ed6204..950795f1 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -8,10 +8,13 @@ package routing import ( "encoding/json" + "net" "net/http" "strings" + "time" "github.com/element-hq/dendrite/federationapi/routing" + "github.com/element-hq/dendrite/internal" "github.com/element-hq/dendrite/internal/httputil" "github.com/element-hq/dendrite/mediaapi/storage" "github.com/element-hq/dendrite/mediaapi/types" @@ -89,7 +92,6 @@ func Setup( } // v1 url_preview endpoint requiring auth - downloadHandler := makeDownloadAPI("download_unauthed", &cfg.MediaAPI, rateLimits, db, client, federationClient, activeRemoteRequests, activeThumbnailGeneration, false) v3mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) v3mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) @@ -104,7 +106,11 @@ func Setup( v1mux.Handle("/download/{serverName}/{mediaId}", downloadHandlerAuthed).Methods(http.MethodGet, http.MethodOptions) v1mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandlerAuthed).Methods(http.MethodGet, http.MethodOptions) - urlPreviewHandler := httputil.MakeAuthAPI("preview_url", userAPI, makeUrlPreviewHandler(&cfg.MediaAPI, rateLimits, db, activeThumbnailGeneration)) + var dialer *net.Dialer + if cfg.FederationAPI.AllowNetworkCIDRs != nil || cfg.FederationAPI.DenyNetworkCIDRs != nil { + dialer = internal.GetDialer(cfg.FederationAPI.AllowNetworkCIDRs, cfg.FederationAPI.DenyNetworkCIDRs, time.Duration(cfg.MediaAPI.UrlPreviewTimeout)) + } + urlPreviewHandler := httputil.MakeAuthAPI("preview_url", userAPI, makeUrlPreviewHandler(&cfg.MediaAPI, dialer, rateLimits, db, activeThumbnailGeneration)) v1mux.Handle("/preview_url", urlPreviewHandler).Methods(http.MethodGet, http.MethodOptions) // That method is deprecated according to spec but still in use v3mux.Handle("/preview_url", urlPreviewHandler).Methods(http.MethodGet, http.MethodOptions) diff --git a/mediaapi/routing/url_preview.go b/mediaapi/routing/url_preview.go index 083fe161..4cbed66b 100644 --- a/mediaapi/routing/url_preview.go +++ b/mediaapi/routing/url_preview.go @@ -15,6 +15,7 @@ import ( "encoding/json" "fmt" "io" + "net" "net/http" "net/url" "os" @@ -47,11 +48,12 @@ var ( ErrorFileTooLarge = errors.New("file too large") ErrorTimeoutThumbnailGenerator = errors.New("timeout waiting for thumbnail generator") ErrNoMetadataFound = errors.New("no metadata found") - ErrorBlackListed = errors.New("url is blacklisted") + ErrorUrlDenied = errors.New("url is in the urls deny list") ) func makeUrlPreviewHandler( cfg *config.MediaAPI, + dialer *net.Dialer, rateLimits *httputil.RateLimits, db storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration, @@ -59,7 +61,7 @@ func makeUrlPreviewHandler( activeUrlPreviewRequests := &types.ActiveUrlPreviewRequests{Url: map[string]*types.UrlPreviewResult{}} urlPreviewCache := &types.UrlPreviewCache{Records: map[string]*types.UrlPreviewCacheRecord{}} - urlBlackList := createUrlBlackList(cfg) + urlDenyList := createUrlDenyList(cfg) go func() { for { @@ -95,10 +97,9 @@ func makeUrlPreviewHandler( return *r } - // Check if the url is in the blacklist - if checkURLBlacklisted(urlBlackList, pUrl) { - logger.Debug("The url is in the blacklist") - return util.ErrorResponse(ErrorBlackListed) + // Check if the url is in the deny list + if checkIsURLDenied(urlDenyList, pUrl) { + return util.ErrorResponse(ErrorUrlDenied) } urlParsed, perr := url.Parse(pUrl) @@ -173,7 +174,7 @@ func makeUrlPreviewHandler( defer activeUrlPreviewRequests.Unlock() }() - resp, err := downloadUrl(pUrl, time.Duration(cfg.UrlPreviewTimeout)*time.Second) + resp, err := downloadUrl(pUrl, dialer, time.Duration(cfg.UrlPreviewTimeout)*time.Second) if err != nil { activeUrlPreviewRequest.Error = err } else { @@ -189,7 +190,7 @@ func makeUrlPreviewHandler( result = getPreviewFromHTML(resp, urlParsed) if result.ImageUrl != "" { // In case of an image in the preview we download it - if imgReader, derr := downloadUrl(result.ImageUrl, time.Duration(cfg.UrlPreviewTimeout)*time.Second); derr == nil { + if imgReader, derr := downloadUrl(result.ImageUrl, dialer, time.Duration(cfg.UrlPreviewTimeout)*time.Second); derr == nil { mediaData, width, height, _ = downloadAndStoreImage("url_preview", req.Context(), imgReader, cfg, device, db, activeThumbnailGeneration, logger) } // We don't show the original image in the preview @@ -273,10 +274,14 @@ func checkActivePreviewResponse(activeUrlPreviewRequests *types.ActiveUrlPreview return util.JSONResponse{}, false } -func downloadUrl(url string, t time.Duration) (*http.Response, error) { +func downloadUrl(url string, dialer *net.Dialer, t time.Duration) (*http.Response, error) { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } + if dialer != nil { + tr.DialContext = dialer.DialContext + } + client := http.Client{Timeout: t, Transport: tr} resp, err := client.Get(url) if err != nil { @@ -668,17 +673,17 @@ func getMetaFieldsFromHTML(resp *http.Response) map[string]string { return ogValues } -func createUrlBlackList(cfg *config.MediaAPI) []*regexp.Regexp { - blackList := make([]*regexp.Regexp, len(cfg.UrlPreviewBlacklist)) - for i, pattern := range cfg.UrlPreviewBlacklist { - blackList[i] = regexp.MustCompile(pattern) +func createUrlDenyList(cfg *config.MediaAPI) []*regexp.Regexp { + denyList := make([]*regexp.Regexp, len(cfg.UrlPreviewDenylist)) + for i, pattern := range cfg.UrlPreviewDenylist { + denyList[i] = regexp.MustCompile(pattern) } - return blackList + return denyList } -func checkURLBlacklisted(blacklist []*regexp.Regexp, url string) bool { - // Check if the url is in the blacklist - for _, pattern := range blacklist { +func checkIsURLDenied(urldenylist []*regexp.Regexp, url string) bool { + // Check if the url is in the deny list + for _, pattern := range urldenylist { if pattern.MatchString(url) { return true } diff --git a/mediaapi/routing/url_preview_test.go b/mediaapi/routing/url_preview_test.go index 3665cb0a..331dae0c 100644 --- a/mediaapi/routing/url_preview_test.go +++ b/mediaapi/routing/url_preview_test.go @@ -10,6 +10,7 @@ import ( "context" "fmt" "io" + "net" "net/http" "net/http/httptest" "net/url" @@ -21,6 +22,7 @@ import ( "testing" "time" + "github.com/element-hq/dendrite/internal" "github.com/element-hq/dendrite/internal/httputil" "github.com/element-hq/dendrite/internal/sqlutil" "github.com/element-hq/dendrite/mediaapi/fileutils" @@ -28,6 +30,7 @@ import ( "github.com/element-hq/dendrite/mediaapi/types" "github.com/element-hq/dendrite/setup/config" userapi "github.com/element-hq/dendrite/userapi/api" + "github.com/foxcpp/go-mockdns" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" ) @@ -172,12 +175,12 @@ func Test_Blacklist(t *testing.T) { } cfg := &config.MediaAPI{ - UrlPreviewBlacklist: tests["entrys"].([]string), + UrlPreviewDenylist: tests["entrys"].([]string), } - blacklist := createUrlBlackList(cfg) + denylist := createUrlDenyList(cfg) for url, expected := range tests["tests"].(map[string]bool) { - value := checkURLBlacklisted(blacklist, url) + value := checkIsURLDenied(denylist, url) if value != expected { t.Errorf("Blacklist %v: expected=%v, got=%v", url, expected, value) } @@ -216,7 +219,7 @@ func Test_ActiveRequestWaiting(t *testing.T) { time.Sleep(time.Duration(1) * time.Second) successResultsLock.Lock() if successResults != 0 { - t.Error("Subroutines didn't wait") + t.Error("Subroutines haven't waited for the result") } successResultsLock.Unlock() activeRequests.Url["someurl"].Cond.Broadcast() @@ -299,11 +302,11 @@ func Test_UrlPreviewHandler(t *testing.T) { UserID: "user", } - handler := makeUrlPreviewHandler(cfg, rateLimits, db, activeThumbnailGeneration) + handler := makeUrlPreviewHandler(cfg, nil, rateLimits, db, activeThumbnailGeneration) // this handler is to test filecache - handler2 := makeUrlPreviewHandler(cfg, rateLimits, db, activeThumbnailGeneration) + handler2 := makeUrlPreviewHandler(cfg, nil, rateLimits, db, activeThumbnailGeneration) // this handler is to test image resize - handler3 := makeUrlPreviewHandler(cfg2, rateLimits, db2, activeThumbnailGeneration) + handler3 := makeUrlPreviewHandler(cfg2, nil, rateLimits, db2, activeThumbnailGeneration) responseBody := ` @@ -364,6 +367,38 @@ func Test_UrlPreviewHandler(t *testing.T) { assert.Less(t, result.JSON.(*types.UrlPreview).ImageHeight, srcHeight, "thumbnail height missmatch") assert.Less(t, result.JSON.(*types.UrlPreview).ImageWidth, srcWidth, "thumbnail width missmatch") + // Test denied addresses + + dns := SetupFakeResolver() + defer func(t *testing.T) { + t.Helper() + err := dns.Close() + assert.NoError(t, err) + }(t) + defer mockdns.UnpatchNet(net.DefaultResolver) + + // this handler is to test allow/deny nets + denyNets := []string{"192.168.1.1/24", "172.15.1.0/24"} + allowNets := []string{"127.0.0.1/24"} + dialer := internal.GetDialer(allowNets, denyNets, time.Duration(5*time.Second)) + handler4 := makeUrlPreviewHandler(cfg, dialer, rateLimits, db, activeThumbnailGeneration) + + serverUrlParsed, err := url.Parse(srv.URL) + assert.NoError(t, err) + tests := map[string]int{ + "http://deny1.example.com/test.png": 500, + "http://deny2.example.com/test.png": 500, + fmt.Sprintf("http://allow.example.com:%s/test.png", serverUrlParsed.Port()): 200, + } + for serverUrl, code := range tests { + ur4, _ := url.Parse("/?url=" + serverUrl) + result = handler4(&http.Request{ + Method: "GET", + URL: ur4, + }, &device) + assert.Equal(t, result.Code, code, "Deny: Response code mismatch: %s", result.JSON) + } + srv.Close() // Test in-memory cache @@ -379,3 +414,24 @@ func Test_UrlPreviewHandler(t *testing.T) { assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found") } + +// SetupFakeResolver sets up Fake DNS server to resolve SRV records. +func SetupFakeResolver() *mockdns.Server { + + testZone := map[string]mockdns.Zone{ + "allow.example.com.": { + A: []string{"127.0.0.1"}, + }, + "deny1.example.com.": { + A: []string{"192.168.1.10"}, + }, + "deny2.example.com.": { + A: []string{"172.15.1.10"}, + }, + } + + srv, _ := mockdns.NewServer(testZone, true) + srv.PatchNet(net.DefaultResolver) + + return srv +} diff --git a/setup/config/config_mediaapi.go b/setup/config/config_mediaapi.go index c9538e2e..8f3119b5 100644 --- a/setup/config/config_mediaapi.go +++ b/setup/config/config_mediaapi.go @@ -31,8 +31,8 @@ type MediaAPI struct { // A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"` - // Black list of urls - UrlPreviewBlacklist []string `yaml:"url_preview_blacklist"` + // Deny list of urls + UrlPreviewDenylist []string `yaml:"url_preview_denylist"` // The time in seconds to cache URL previews for UrlPreviewCacheTime int `yaml:"url_preview_cache_time"`