Automatically upgrade databases on startup (#1529)

* Support auto-upgrading accounts DB

* Auto-upgrade device DB deltas

* Support up/downgrading from cmd/goose

* Linting

* Create tables then do migrations then prepare statements

To avoid failing due to some things not existing

* Linting

userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.go

@@ -0,0 +1,39 @@
+package deltas
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/pressly/goose"
+)
+
+func LoadFromGoose() {
+	goose.AddMigration(UpLastSeenTSIP, DownLastSeenTSIP)
+}
+
+func LoadLastSeenTSIP(m *sqlutil.Migrations) {
+	m.AddMigration(UpLastSeenTSIP, DownLastSeenTSIP)
+}
+
+func UpLastSeenTSIP(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS last_seen_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)*1000;
+ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS ip TEXT;
+ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS user_agent TEXT;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
+
+func DownLastSeenTSIP(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+	ALTER TABLE device_devices DROP COLUMN last_seen_ts;
+	ALTER TABLE device_devices DROP COLUMN ip;
+	ALTER TABLE device_devices DROP COLUMN user_agent;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	return nil
+}

userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql

@@ -1,13 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS last_seen_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)*1000;
-ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS ip TEXT;
-ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS user_agent TEXT;
--- +goose StatementEnd
-
--- +goose Down
--- +goose StatementBegin
-ALTER TABLE device_devices DROP COLUMN last_seen_ts;
-ALTER TABLE device_devices DROP COLUMN ip;
-ALTER TABLE device_devices DROP COLUMN user_agent;
--- +goose StatementEnd

userapi/storage/devices/postgres/devices_table.go

@@ -111,11 +111,12 @@ type devicesStatements struct {
 	serverName                   gomatrixserverlib.ServerName
 }
 
+func (s *devicesStatements) execSchema(db *sql.DB) error {
+	_, err := db.Exec(devicesSchema)
+	return err
+}
+
 func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) {
-	_, err = db.Exec(devicesSchema)
-	if err != nil {
-		return
-	}
 	if s.insertDeviceStmt, err = db.Prepare(insertDeviceSQL); err != nil {
 		return
 	}

userapi/storage/devices/postgres/storage.go

@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/devices/postgres/deltas"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -42,9 +43,22 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver
 		return nil, err
 	}
 	d := devicesStatements{}
+
+	// Create tables before executing migrations so we don't fail if the table is missing,
+	// and THEN prepare statements so we don't fail due to referencing new columns
+	if err = d.execSchema(db); err != nil {
+		return nil, err
+	}
+	m := sqlutil.NewMigrations()
+	deltas.LoadLastSeenTSIP(m)
+	if err = m.RunDeltas(db, dbProperties); err != nil {
+		return nil, err
+	}
+
 	if err = d.prepare(db, serverName); err != nil {
 		return nil, err
 	}
+
 	return &Database{db, d}, nil
 }