mirror of
https://github.com/element-hq/dendrite.git
synced 2025-09-13 12:52:24 +03:00
Initial Store & Forward Implementation (#2917)
This adds store & forward relays into dendrite for p2p. A few things have changed: - new relay api serves new http endpoints for s&f federation - updated outbound federation queueing which will attempt to forward using s&f if appropriate - database entries to track s&f relays for other nodes
This commit is contained in:
devonh
GitHub
parent
48fa869fa3
commit
5b73592f5a
77 changed files with 7646 additions and 1373 deletions
|
|
@ -29,7 +29,7 @@ import (
|
|||
fedapi "github.com/matrix-org/dendrite/federationapi/api"
|
||||
"github.com/matrix-org/dendrite/federationapi/statistics"
|
||||
"github.com/matrix-org/dendrite/federationapi/storage"
|
||||
"github.com/matrix-org/dendrite/federationapi/storage/shared"
|
||||
"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
|
||||
"github.com/matrix-org/dendrite/roomserver/api"
|
||||
"github.com/matrix-org/dendrite/setup/process"
|
||||
)
|
||||
|
|
@ -70,7 +70,7 @@ type destinationQueue struct {
|
|||
// Send event adds the event to the pending queue for the destination.
|
||||
// If the queue is empty then it starts a background goroutine to
|
||||
// start sending events to that destination.
|
||||
func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, receipt *shared.Receipt) {
|
||||
func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, dbReceipt *receipt.Receipt) {
|
||||
if event == nil {
|
||||
logrus.Errorf("attempt to send nil PDU with destination %q", oq.destination)
|
||||
return
|
||||
|
|
@ -84,8 +84,8 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re
|
|||
oq.pendingMutex.Lock()
|
||||
if len(oq.pendingPDUs) < maxPDUsInMemory {
|
||||
oq.pendingPDUs = append(oq.pendingPDUs, &queuedPDU{
|
||||
pdu: event,
|
||||
receipt: receipt,
|
||||
pdu: event,
|
||||
dbReceipt: dbReceipt,
|
||||
})
|
||||
} else {
|
||||
oq.overflowed.Store(true)
|
||||
|
|
@ -101,7 +101,7 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re
|
|||
// sendEDU adds the EDU event to the pending queue for the destination.
|
||||
// If the queue is empty then it starts a background goroutine to
|
||||
// start sending events to that destination.
|
||||
func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *shared.Receipt) {
|
||||
func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, dbReceipt *receipt.Receipt) {
|
||||
if event == nil {
|
||||
logrus.Errorf("attempt to send nil EDU with destination %q", oq.destination)
|
||||
return
|
||||
|
|
@ -115,8 +115,8 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share
|
|||
oq.pendingMutex.Lock()
|
||||
if len(oq.pendingEDUs) < maxEDUsInMemory {
|
||||
oq.pendingEDUs = append(oq.pendingEDUs, &queuedEDU{
|
||||
edu: event,
|
||||
receipt: receipt,
|
||||
edu: event,
|
||||
dbReceipt: dbReceipt,
|
||||
})
|
||||
} else {
|
||||
oq.overflowed.Store(true)
|
||||
|
|
@ -210,10 +210,10 @@ func (oq *destinationQueue) getPendingFromDatabase() {
|
|||
gotPDUs := map[string]struct{}{}
|
||||
gotEDUs := map[string]struct{}{}
|
||||
for _, pdu := range oq.pendingPDUs {
|
||||
gotPDUs[pdu.receipt.String()] = struct{}{}
|
||||
gotPDUs[pdu.dbReceipt.String()] = struct{}{}
|
||||
}
|
||||
for _, edu := range oq.pendingEDUs {
|
||||
gotEDUs[edu.receipt.String()] = struct{}{}
|
||||
gotEDUs[edu.dbReceipt.String()] = struct{}{}
|
||||
}
|
||||
|
||||
overflowed := false
|
||||
|
|
@ -371,7 +371,7 @@ func (oq *destinationQueue) backgroundSend() {
|
|||
|
||||
// If we have pending PDUs or EDUs then construct a transaction.
|
||||
// Try sending the next transaction and see what happens.
|
||||
terr := oq.nextTransaction(toSendPDUs, toSendEDUs)
|
||||
terr, sendMethod := oq.nextTransaction(toSendPDUs, toSendEDUs)
|
||||
if terr != nil {
|
||||
// We failed to send the transaction. Mark it as a failure.
|
||||
_, blacklisted := oq.statistics.Failure()
|
||||
|
|
@ -388,18 +388,19 @@ func (oq *destinationQueue) backgroundSend() {
|
|||
return
|
||||
}
|
||||
} else {
|
||||
oq.handleTransactionSuccess(pduCount, eduCount)
|
||||
oq.handleTransactionSuccess(pduCount, eduCount, sendMethod)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// nextTransaction creates a new transaction from the pending event
|
||||
// queue and sends it.
|
||||
// Returns an error if the transaction wasn't sent.
|
||||
// Returns an error if the transaction wasn't sent. And whether the success
|
||||
// was to a relay server or not.
|
||||
func (oq *destinationQueue) nextTransaction(
|
||||
pdus []*queuedPDU,
|
||||
edus []*queuedEDU,
|
||||
) error {
|
||||
) (err error, sendMethod statistics.SendMethod) {
|
||||
// Create the transaction.
|
||||
t, pduReceipts, eduReceipts := oq.createTransaction(pdus, edus)
|
||||
logrus.WithField("server_name", oq.destination).Debugf("Sending transaction %q containing %d PDUs, %d EDUs", t.TransactionID, len(t.PDUs), len(t.EDUs))
|
||||
|
|
@ -407,7 +408,37 @@ func (oq *destinationQueue) nextTransaction(
|
|||
// Try to send the transaction to the destination server.
|
||||
ctx, cancel := context.WithTimeout(oq.process.Context(), time.Minute*5)
|
||||
defer cancel()
|
||||
_, err := oq.client.SendTransaction(ctx, t)
|
||||
|
||||
relayServers := oq.statistics.KnownRelayServers()
|
||||
if oq.statistics.AssumedOffline() && len(relayServers) > 0 {
|
||||
sendMethod = statistics.SendViaRelay
|
||||
relaySuccess := false
|
||||
logrus.Infof("Sending to relay servers: %v", relayServers)
|
||||
// TODO : how to pass through actual userID here?!?!?!?!
|
||||
userID, userErr := gomatrixserverlib.NewUserID("@user:"+string(oq.destination), false)
|
||||
if userErr != nil {
|
||||
return userErr, sendMethod
|
||||
}
|
||||
|
||||
// Attempt sending to each known relay server.
|
||||
for _, relayServer := range relayServers {
|
||||
_, relayErr := oq.client.P2PSendTransactionToRelay(ctx, *userID, t, relayServer)
|
||||
if relayErr != nil {
|
||||
err = relayErr
|
||||
} else {
|
||||
// If sending to one of the relay servers succeeds, consider the send successful.
|
||||
relaySuccess = true
|
||||
}
|
||||
}
|
||||
|
||||
// Clear the error if sending to any of the relay servers succeeded.
|
||||
if relaySuccess {
|
||||
err = nil
|
||||
}
|
||||
} else {
|
||||
sendMethod = statistics.SendDirect
|
||||
_, err = oq.client.SendTransaction(ctx, t)
|
||||
}
|
||||
switch errResponse := err.(type) {
|
||||
case nil:
|
||||
// Clean up the transaction in the database.
|
||||
|
|
@ -427,7 +458,7 @@ func (oq *destinationQueue) nextTransaction(
|
|||
oq.transactionIDMutex.Lock()
|
||||
oq.transactionID = ""
|
||||
oq.transactionIDMutex.Unlock()
|
||||
return nil
|
||||
return nil, sendMethod
|
||||
case gomatrix.HTTPError:
|
||||
// Report that we failed to send the transaction and we
|
||||
// will retry again, subject to backoff.
|
||||
|
|
@ -437,13 +468,13 @@ func (oq *destinationQueue) nextTransaction(
|
|||
// to a 400-ish error
|
||||
code := errResponse.Code
|
||||
logrus.Debug("Transaction failed with HTTP", code)
|
||||
return err
|
||||
return err, sendMethod
|
||||
default:
|
||||
logrus.WithFields(logrus.Fields{
|
||||
"destination": oq.destination,
|
||||
logrus.ErrorKey: err,
|
||||
}).Debugf("Failed to send transaction %q", t.TransactionID)
|
||||
return err
|
||||
return err, sendMethod
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -453,7 +484,7 @@ func (oq *destinationQueue) nextTransaction(
|
|||
func (oq *destinationQueue) createTransaction(
|
||||
pdus []*queuedPDU,
|
||||
edus []*queuedEDU,
|
||||
) (gomatrixserverlib.Transaction, []*shared.Receipt, []*shared.Receipt) {
|
||||
) (gomatrixserverlib.Transaction, []*receipt.Receipt, []*receipt.Receipt) {
|
||||
// If there's no projected transaction ID then generate one. If
|
||||
// the transaction succeeds then we'll set it back to "" so that
|
||||
// we generate a new one next time. If it fails, we'll preserve
|
||||
|
|
@ -474,8 +505,8 @@ func (oq *destinationQueue) createTransaction(
|
|||
t.OriginServerTS = gomatrixserverlib.AsTimestamp(time.Now())
|
||||
t.TransactionID = oq.transactionID
|
||||
|
||||
var pduReceipts []*shared.Receipt
|
||||
var eduReceipts []*shared.Receipt
|
||||
var pduReceipts []*receipt.Receipt
|
||||
var eduReceipts []*receipt.Receipt
|
||||
|
||||
// Go through PDUs that we retrieved from the database, if any,
|
||||
// and add them into the transaction.
|
||||
|
|
@ -487,7 +518,7 @@ func (oq *destinationQueue) createTransaction(
|
|||
// Append the JSON of the event, since this is a json.RawMessage type in the
|
||||
// gomatrixserverlib.Transaction struct
|
||||
t.PDUs = append(t.PDUs, pdu.pdu.JSON())
|
||||
pduReceipts = append(pduReceipts, pdu.receipt)
|
||||
pduReceipts = append(pduReceipts, pdu.dbReceipt)
|
||||
}
|
||||
|
||||
// Do the same for pending EDUS in the queue.
|
||||
|
|
@ -497,7 +528,7 @@ func (oq *destinationQueue) createTransaction(
|
|||
continue
|
||||
}
|
||||
t.EDUs = append(t.EDUs, *edu.edu)
|
||||
eduReceipts = append(eduReceipts, edu.receipt)
|
||||
eduReceipts = append(eduReceipts, edu.dbReceipt)
|
||||
}
|
||||
|
||||
return t, pduReceipts, eduReceipts
|
||||
|
|
@ -530,10 +561,11 @@ func (oq *destinationQueue) blacklistDestination() {
|
|||
|
||||
// handleTransactionSuccess updates the cached event queues as well as the success and
|
||||
// backoff information for this server.
|
||||
func (oq *destinationQueue) handleTransactionSuccess(pduCount int, eduCount int) {
|
||||
func (oq *destinationQueue) handleTransactionSuccess(pduCount int, eduCount int, sendMethod statistics.SendMethod) {
|
||||
// If we successfully sent the transaction then clear out
|
||||
// the pending events and EDUs, and wipe our transaction ID.
|
||||
oq.statistics.Success()
|
||||
|
||||
oq.statistics.Success(sendMethod)
|
||||
oq.pendingMutex.Lock()
|
||||
defer oq.pendingMutex.Unlock()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue