From 98c90b23af05376889e2683e24d23ffb02afe317 Mon Sep 17 00:00:00 2001 From: Timo K Date: Sun, 12 Oct 2025 19:22:58 +0200 Subject: [PATCH 01/10] dependencies Signed-off-by: Timo K --- package-lock.json | 7 +++++++ package.json | 2 ++ vite.config.js | 13 ++++++++++--- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index d032ace2..6c69c5d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@atlaskit/pragmatic-drag-and-drop": "1.1.6", "@atlaskit/pragmatic-drag-and-drop-auto-scroll": "1.3.0", "@atlaskit/pragmatic-drag-and-drop-hitbox": "1.0.3", + "@element-hq/element-call-embedded": "0.16.0", "@fontsource/inter": "4.5.14", "@tanstack/react-query": "5.24.1", "@tanstack/react-query-devtools": "5.24.1", @@ -44,6 +45,7 @@ "linkify-react": "4.1.3", "linkifyjs": "4.1.3", "matrix-js-sdk": "38.2.0", + "matrix-widget-api": "1.13.1", "millify": "6.1.0", "pdfjs-dist": "4.2.67", "prismjs": "1.30.0", @@ -1649,6 +1651,11 @@ "node": ">=6.9.0" } }, + "node_modules/@element-hq/element-call-embedded": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/@element-hq/element-call-embedded/-/element-call-embedded-0.16.0.tgz", + "integrity": "sha512-cTJwW9cmQHrTUvcJm0xv9uSTMYiToDEaw5AuxXQS9jVjHQT8B3W3DWtKYAzq1PRRH13JZkcwXbHjdFpkxzhzCQ==" + }, "node_modules/@emotion/hash": { "version": "0.9.2", "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.2.tgz", diff --git a/package.json b/package.json index 577e129d..505df13f 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "@atlaskit/pragmatic-drag-and-drop": "1.1.6", "@atlaskit/pragmatic-drag-and-drop-auto-scroll": "1.3.0", "@atlaskit/pragmatic-drag-and-drop-hitbox": "1.0.3", + "@element-hq/element-call-embedded": "0.16.0", "@fontsource/inter": "4.5.14", "@tanstack/react-query": "5.24.1", "@tanstack/react-query-devtools": "5.24.1", @@ -55,6 +56,7 @@ "linkify-react": "4.1.3", "linkifyjs": "4.1.3", "matrix-js-sdk": "38.2.0", + "matrix-widget-api": "1.13.1", "millify": "6.1.0", "pdfjs-dist": "4.2.67", "prismjs": "1.30.0", diff --git a/vite.config.js b/vite.config.js index dfa02fc4..444aa0e0 100644 --- a/vite.config.js +++ b/vite.config.js @@ -38,6 +38,10 @@ const copyFiles = { src: 'public/locales', dest: 'public/', }, + { + src: 'node_modules/@element-hq/element-call-embedded/dist/*', + dest: './widgets/element-call/', + }, ], }; @@ -47,7 +51,10 @@ function serverMatrixSdkCryptoWasm(wasmFilePath) { configureServer(server) { server.middlewares.use((req, res, next) => { if (req.url === wasmFilePath) { - const resolvedPath = path.join(path.resolve(), "/node_modules/@matrix-org/matrix-sdk-crypto-wasm/pkg/matrix_sdk_crypto_wasm_bg.wasm"); + const resolvedPath = path.join( + path.resolve(), + '/node_modules/@matrix-org/matrix-sdk-crypto-wasm/pkg/matrix_sdk_crypto_wasm_bg.wasm' + ); if (fs.existsSync(resolvedPath)) { res.setHeader('Content-Type', 'application/wasm'); @@ -102,8 +109,8 @@ export default defineConfig({ }, devOptions: { enabled: true, - type: 'module' - } + type: 'module', + }, }), ], optimizeDeps: { From bbe53d6d6ff05beb00c9afeb749da2d0a7006a64 Mon Sep 17 00:00:00 2001 From: Timo K Date: Sun, 12 Oct 2025 19:23:48 +0200 Subject: [PATCH 02/10] copy over utils form EW Signed-off-by: Timo K --- src/app/components/element-call/utils.ts | 157 +++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 src/app/components/element-call/utils.ts diff --git a/src/app/components/element-call/utils.ts b/src/app/components/element-call/utils.ts new file mode 100644 index 00000000..a867a49f --- /dev/null +++ b/src/app/components/element-call/utils.ts @@ -0,0 +1,157 @@ +import { logger } from 'matrix-js-sdk/lib/logger'; + +import { EventType } from 'matrix-js-sdk'; +import { EventDirection, MatrixCapabilities, WidgetEventCapability } from 'matrix-widget-api'; +import EventEmitter from 'events'; +import { useEffect } from 'react'; + +function getCredentials() { + const accessToken = localStorage.getItem('mx_access_token'); + const userId = localStorage.getItem('mx_user_id'); + const deviceId = localStorage.getItem('mx_device_id'); + return [accessToken, userId, deviceId]; +} + +export async function downloadFromUrlToFile(url: string, filename?: string): Promise { + const [accessToken] = getCredentials(); + try { + const response = await fetch(url, { + body: 'blob', + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }); + const file = new File([await response.blob()], filename || 'file', { + type: response.type, + }); + return file; + } catch (error) { + logger.error('Error downloading file', error); + throw error; + } +} +export type Diff = { added: T[]; removed: T[] }; + +/** + * Performs a diff on two arrays. The result is what is different with the + * first array (`added` in the returned object means objects in B that aren't + * in A). Shallow comparisons are used to perform the diff. + * @param a The first array. Must be defined. + * @param b The second array. Must be defined. + * @returns The diff between the arrays. + */ +export function arrayDiff(a: T[], b: T[]): Diff { + return { + added: b.filter((i) => !a.includes(i)), + removed: a.filter((i) => !b.includes(i)), + }; +} + +export default function iterableDiff( + a: Iterable, + b: Iterable +): { added: Iterable; removed: Iterable } { + return arrayDiff(Array.from(a), Array.from(b)); +} + +/** + * Clones an array as fast as possible, retaining references of the array's values. + * @param a The array to clone. Must be defined. + * @returns A copy of the array. + */ +export function arrayFastClone(a: T[]): T[] { + return a.slice(0, a.length); +} + +export function elementCallCapabilities( + inRoomId: string, + clientUserId: string, + clientDeviceId?: string +): Set { + const allowedCapabilities = new Set(); + + // This is a trusted Element Call widget that we control + const addCapability = (type: string, state: boolean, dir: EventDirection, stateKey?: string) => + allowedCapabilities.add( + state + ? WidgetEventCapability.forStateEvent(dir, type, stateKey).raw + : WidgetEventCapability.forRoomEvent(dir, type).raw + ); + const addToDeviceCapability = (eventType: string, dir: EventDirection) => + allowedCapabilities.add(WidgetEventCapability.forToDeviceEvent(dir, eventType).raw); + const recvState = (eventType: string, stateKey?: string) => + addCapability(eventType, true, EventDirection.Receive, stateKey); + const sendState = (eventType: string, stateKey?: string) => + addCapability(eventType, true, EventDirection.Send, stateKey); + const sendRecvToDevice = (eventType: string) => { + addToDeviceCapability(eventType, EventDirection.Receive); + addToDeviceCapability(eventType, EventDirection.Send); + }; + const recvRoom = (eventType: string) => addCapability(eventType, false, EventDirection.Receive); + const sendRoom = (eventType: string) => addCapability(eventType, false, EventDirection.Send); + const sendRecvRoom = (eventType: string) => { + recvRoom(eventType); + sendRoom(eventType); + }; + + allowedCapabilities.add(MatrixCapabilities.AlwaysOnScreen); + allowedCapabilities.add(MatrixCapabilities.MSC3846TurnServers); + allowedCapabilities.add(`org.matrix.msc2762.timeline:${inRoomId}`); + allowedCapabilities.add(MatrixCapabilities.MSC4157SendDelayedEvent); + allowedCapabilities.add(MatrixCapabilities.MSC4157UpdateDelayedEvent); + recvState(EventType.RoomMember); + recvState('org.matrix.msc3401.call'); + recvState(EventType.RoomEncryption); + recvState(EventType.RoomName); + // For the legacy membership type + sendState('org.matrix.msc3401.call.member', clientUserId); + // For the session membership type compliant with MSC4143 + sendState('org.matrix.msc3401.call.member', `_${clientUserId}_${clientDeviceId}_m.call`); + sendState('org.matrix.msc3401.call.member', `${clientUserId}_${clientDeviceId}_m.call`); + sendState('org.matrix.msc3401.call.member', `_${clientUserId}_${clientDeviceId}`); + sendState('org.matrix.msc3401.call.member', `${clientUserId}_${clientDeviceId}`); + recvState('org.matrix.msc3401.call.member'); + // for determining auth rules specific to the room version + recvState(EventType.RoomCreate); + + sendRoom('org.matrix.msc4075.rtc.notification'); + sendRecvRoom('io.element.call.encryption_keys'); + sendRecvRoom('org.matrix.rageshake_request'); + sendRecvRoom(EventType.Reaction); + sendRecvRoom(EventType.RoomRedaction); + sendRecvRoom('io.element.call.reaction'); + sendRecvRoom('org.matrix.msc4310.rtc.decline'); + + sendRecvToDevice(EventType.CallInvite); + sendRecvToDevice(EventType.CallCandidates); + sendRecvToDevice(EventType.CallAnswer); + sendRecvToDevice(EventType.CallHangup); + sendRecvToDevice(EventType.CallReject); + sendRecvToDevice(EventType.CallSelectAnswer); + sendRecvToDevice(EventType.CallNegotiate); + sendRecvToDevice(EventType.CallSDPStreamMetadataChanged); + sendRecvToDevice(EventType.CallSDPStreamMetadataChangedPrefix); + sendRecvToDevice(EventType.CallReplaces); + sendRecvToDevice(EventType.CallEncryptionKeysPrefix); + + return allowedCapabilities; +} + +// Shortcut for registering a listener on an EventTarget +// Copied from element-web +export function useEventEmitter( + emitter: EventEmitter | null | undefined, + eventType: string, + listener: (event: T) => void +): void { + useEffect((): (() => void) => { + if (emitter) { + emitter.on(eventType, listener); + return (): void => { + emitter.off(eventType, listener); + }; + } + // eslint-disable-next-line @typescript-eslint/no-empty-function + return () => {}; + }, [emitter, eventType, listener]); +} From 8df27ac688423ccba2291b6bf653fb0a6779bb0c Mon Sep 17 00:00:00 2001 From: Timo K Date: Sun, 12 Oct 2025 19:24:02 +0200 Subject: [PATCH 03/10] copy over widgetDriver from EW Signed-off-by: Timo K --- .../element-call/CallWidgetDriver.ts | 497 ++++++++++++++++++ 1 file changed, 497 insertions(+) create mode 100644 src/app/components/element-call/CallWidgetDriver.ts diff --git a/src/app/components/element-call/CallWidgetDriver.ts b/src/app/components/element-call/CallWidgetDriver.ts new file mode 100644 index 00000000..0ed4977d --- /dev/null +++ b/src/app/components/element-call/CallWidgetDriver.ts @@ -0,0 +1,497 @@ +/* eslint-disable no-plusplus */ +/* eslint-disable no-dupe-class-members */ +/* eslint-disable lines-between-class-members */ +/* eslint-disable class-methods-use-this */ +import { + type Capability, + type IOpenIDCredentials, + type IOpenIDUpdate, + type ISendDelayedEventDetails, + type ISendEventDetails, + type ITurnServer, + type IReadEventRelationsResult, + type IRoomEvent, + OpenIDRequestState, + type SimpleObservable, + WidgetDriver, + type IWidgetApiErrorResponseDataDetails, + type ISearchUserDirectoryResult, + type IGetMediaConfigResult, + type UpdateDelayedEventAction, +} from 'matrix-widget-api'; +import { + ClientEvent, + type ITurnServer as IClientTurnServer, + EventType, + type IContent, + MatrixError, + type MatrixEvent, + Direction, + type SendDelayedEventResponse, + type StateEvents, + type TimelineEvents, + MatrixClient, + getHttpUriForMxc, +} from 'matrix-js-sdk'; +import { logger } from 'matrix-js-sdk/lib/logger'; +import iterableDiff, { downloadFromUrlToFile } from './utils'; + +// TODO: Purge this from the universe + +const normalizeTurnServer = ({ urls, username, credential }: IClientTurnServer): ITurnServer => ({ + uris: urls, + username, + password: credential, +}); + +export default class CallWidgetDriver extends WidgetDriver { + // TODO: Refactor widgetKind into the Widget class + public constructor( + private client: MatrixClient, + private allowedCapabilities = new Set(), + private inRoomId: string + ) { + super(); + } + + public async validateCapabilities(requested: Set): Promise> { + // Check to see if any capabilities aren't automatically accepted (such as sticker pickers + // allowing stickers to be sent). If there are excess capabilities to be approved, the user + // will be prompted to accept them. + const missing = new Set(iterableDiff(requested, this.allowedCapabilities).removed); // "removed" is "in A (requested) but not in B (allowed)" + if (missing.size > 0) logger.error('missing widget capabilities', missing); + return this.allowedCapabilities; + } + + public async sendEvent( + eventType: K, + content: StateEvents[K], + stateKey: string | null, + targetRoomId: string | null + ): Promise; + public async sendEvent( + eventType: K, + content: TimelineEvents[K], + stateKey: null, + targetRoomId: string | null + ): Promise; + public async sendEvent( + eventType: string, + content: IContent, + stateKey: string | null = null, + targetRoomId: string | null = null + ): Promise { + const { client } = this; + const roomId = targetRoomId || this.inRoomId; + + if (!client || !roomId) throw new Error('Not in a room or not attached to a client'); + + let r: { event_id: string } | null; + if (stateKey !== null) { + // state event + r = await client.sendStateEvent( + roomId, + eventType as keyof StateEvents, + content as StateEvents[keyof StateEvents], + stateKey + ); + } else if (eventType === EventType.RoomRedaction) { + // special case: extract the `redacts` property and call redact + r = await client.redactEvent(roomId, content.redacts); + } else { + // message event + r = await client.sendEvent( + roomId, + eventType as keyof TimelineEvents, + content as TimelineEvents[keyof TimelineEvents] + ); + } + + return { roomId, eventId: r.event_id }; + } + + /** + * @experimental Part of MSC4140 & MSC4157 + * @see {@link WidgetDriver#sendDelayedEvent} + */ + public async sendDelayedEvent( + delay: number | null, + parentDelayId: string | null, + eventType: K, + content: StateEvents[K], + stateKey: string | null, + targetRoomId: string | null + ): Promise; + /** + * @experimental Part of MSC4140 & MSC4157 + */ + public async sendDelayedEvent( + delay: number | null, + parentDelayId: string | null, + eventType: K, + content: TimelineEvents[K], + stateKey: null, + targetRoomId: string | null + ): Promise; + public async sendDelayedEvent( + delay: number | null, + parentDelayId: string | null, + eventType: string, + content: IContent, + stateKey: string | null = null, + targetRoomId: string | null = null + ): Promise { + const { client } = this; + const roomId = targetRoomId || this.inRoomId; + + if (!client || !roomId) throw new Error('Not in a room or not attached to a client'); + + let delayOpts; + if (delay !== null) { + delayOpts = { + delay, + ...(parentDelayId !== null && { parent_delay_id: parentDelayId }), + }; + } else if (parentDelayId !== null) { + delayOpts = { + parent_delay_id: parentDelayId, + }; + } else { + throw new Error('Must provide at least one of delay or parentDelayId'); + } + + let r: SendDelayedEventResponse | null; + if (stateKey !== null) { + // state event + r = await client._unstable_sendDelayedStateEvent( + roomId, + delayOpts, + eventType as keyof StateEvents, + content as StateEvents[keyof StateEvents], + stateKey + ); + } else { + // message event + r = await client._unstable_sendDelayedEvent( + roomId, + delayOpts, + null, + eventType as keyof TimelineEvents, + content as TimelineEvents[keyof TimelineEvents] + ); + } + + return { + roomId, + delayId: r.delay_id, + }; + } + + /** + * @experimental Part of MSC4140 & MSC4157 + */ + public async updateDelayedEvent( + delayId: string, + action: UpdateDelayedEventAction + ): Promise { + const { client } = this; + + if (!client) throw new Error('Not in a room or not attached to a client'); + + await client._unstable_updateDelayedEvent(delayId, action); + } + + /** + * Implements {@link WidgetDriver#sendToDevice} + */ + public async sendToDevice( + eventType: string, + encrypted: boolean, + contentMap: { [userId: string]: { [deviceId: string]: object } } + ): Promise { + const { client } = this; + + if (encrypted) { + const crypto = client.getCrypto(); + if (!crypto) throw new Error('E2EE not enabled'); + + // attempt to re-batch these up into a single request + const invertedContentMap: { [content: string]: { userId: string; deviceId: string }[] } = {}; + Object.keys(contentMap).forEach((userId) => { + const userContentMap = contentMap[userId]; + Object.keys(userContentMap).forEach((deviceId) => { + const content = userContentMap[deviceId]; + const stringifiedContent = JSON.stringify(content); + invertedContentMap[stringifiedContent] = invertedContentMap[stringifiedContent] || []; + invertedContentMap[stringifiedContent].push({ userId, deviceId }); + }); + }); + + await Promise.all( + Object.entries(invertedContentMap).map(async ([stringifiedContent, recipients]) => { + const batch = await crypto.encryptToDeviceMessages( + eventType, + recipients, + JSON.parse(stringifiedContent) + ); + + await client.queueToDevice(batch); + }) + ); + } else { + await client.queueToDevice({ + eventType, + batch: Object.entries(contentMap).flatMap(([userId, userContentMap]) => + Object.entries(userContentMap).map(([deviceId, content]) => ({ + userId, + deviceId, + payload: content, + })) + ), + }); + } + } + + /** + * Reads all events of the given type, and optionally `msgtype` (if applicable/defined), + * the user has access to. The widget API will have already verified that the widget is + * capable of receiving the events. Less events than the limit are allowed to be returned, + * but not more. + * @param roomId The ID of the room to look within. + * @param eventType The event type to be read. + * @param msgtype The msgtype of the events to be read, if applicable/defined. + * @param stateKey The state key of the events to be read, if applicable/defined. + * @param limit The maximum number of events to retrieve. Will be zero to denote "as many as + * possible". + * @param since When null, retrieves the number of events specified by the "limit" parameter. + * Otherwise, the event ID at which only subsequent events will be returned, as many as specified + * in "limit". + * @returns {Promise} Resolves to the room events, or an empty array. + */ + public async readRoomTimeline( + roomId: string, + eventType: string, + msgtype: string | undefined, + stateKey: string | undefined, + limit: number, + since: string | undefined + ): Promise { + // relatively arbitrary + const timelineLimit = + limit > 0 ? Math.min(limit, Number.MAX_SAFE_INTEGER) : Number.MAX_SAFE_INTEGER; + + const room = this.client.getRoom(roomId); + if (room === null) return []; + const results: MatrixEvent[] = []; + const events = room.getLiveTimeline().getEvents(); // timelines are most recent last + for (let i = events.length - 1; i >= 0; i--) { + const ev = events[i]; + if (results.length >= timelineLimit) break; + if (since !== undefined && ev.getId() === since) break; + + if ( + ev.getType() === eventType && + !ev.isState() && + (eventType !== EventType.RoomMessage || !msgtype || msgtype === ev.getContent().msgtype) && + (ev.getStateKey() === undefined || stateKey === undefined || ev.getStateKey() === stateKey) + ) { + results.push(ev); + } + } + + return results.map((e) => e.getEffectiveEvent() as IRoomEvent); + } + + /** + * Reads the current values of all matching room state entries. + * @param roomId The ID of the room. + * @param eventType The event type of the entries to be read. + * @param stateKey The state key of the entry to be read. If undefined, + * all room state entries with a matching event type should be returned. + * @returns {Promise} Resolves to the events representing the + * current values of the room state entries. + */ + public async readRoomState( + roomId: string, + eventType: string, + stateKey: string | undefined + ): Promise { + const room = this.client.getRoom(roomId); + if (room === null) return []; + const state = room.getLiveTimeline().getState(Direction.Forward); + if (state === undefined) return []; + + if (stateKey === undefined) { + return state.getStateEvents(eventType).map((e) => e.getEffectiveEvent() as IRoomEvent); + } + const event = state.getStateEvents(eventType, stateKey); + return event === null ? [] : [event.getEffectiveEvent() as IRoomEvent]; + } + + public async askOpenID(observer: SimpleObservable): Promise { + // TODO: Fully functional widget driver a user prompt is required here, see element web + const getToken = (): Promise => this.client.getOpenIdToken(); + return observer.update({ state: OpenIDRequestState.Allowed, token: await getToken() }); + } + + public async navigate(uri: string): Promise { + // navigateToPermalink(uri); + // TODO: Dummy code until we figured out navigateToPermalink implementation + if (uri) return Promise.resolve(); + return Promise.reject(); + } + + public async *getTurnServers(): AsyncGenerator { + const { client } = this; + if (!client.pollingTurnServers || !client.getTurnServers().length) return; + + let setTurnServer: (server: ITurnServer) => void; + let setError: (error: Error) => void; + + const onTurnServers = ([server]: IClientTurnServer[]): void => + setTurnServer(normalizeTurnServer(server)); + const onTurnServersError = (error: Error, fatal: boolean): void => { + if (fatal) setError(error); + }; + + client.on(ClientEvent.TurnServers, onTurnServers); + client.on(ClientEvent.TurnServersError, onTurnServersError); + + try { + const initialTurnServer = client.getTurnServers()[0]; + yield normalizeTurnServer(initialTurnServer); + + const waitForTurnServer = (): Promise => + new Promise((resolve, reject) => { + setTurnServer = resolve; + setError = reject; + }); + + // Repeatedly listen for new TURN servers until an error occurs or + // the caller stops this generator + while (true) { + // eslint-disable-next-line no-await-in-loop + yield await waitForTurnServer(); + } + } finally { + // The loop was broken - clean up + client.off(ClientEvent.TurnServers, onTurnServers); + client.off(ClientEvent.TurnServersError, onTurnServersError); + } + } + + public async readEventRelations( + eventId: string, + roomId?: string, + relationType?: string, + eventType?: string, + from?: string, + to?: string, + limit?: number, + direction?: 'f' | 'b' + ): Promise { + const { client } = this; + const dir = direction as Direction; + const rId = roomId || this.inRoomId; + + if (typeof rId !== 'string') { + throw new Error('Error while reading the current room'); + } + + const { events, nextBatch, prevBatch } = await client.relations( + rId, + eventId, + relationType ?? null, + eventType ?? null, + { + from, + to, + limit, + dir, + } + ); + + return { + chunk: events.map((e) => e.getEffectiveEvent() as IRoomEvent), + nextBatch: nextBatch ?? undefined, + prevBatch: prevBatch ?? undefined, + }; + } + + public async searchUserDirectory( + searchTerm: string, + limit?: number + ): Promise { + const { client } = this; + + const { limited, results } = await client.searchUserDirectory({ term: searchTerm, limit }); + + return { + limited, + results: results.map((r) => ({ + userId: r.user_id, + displayName: r.display_name, + avatarUrl: r.avatar_url, + })), + }; + } + + public async getMediaConfig(): Promise { + const { client } = this; + + return client.getMediaConfig(); + } + + public async uploadFile(file: XMLHttpRequestBodyInit): Promise<{ contentUri: string }> { + const { client } = this; + + const uploadResult = await client.uploadContent(file); + + return { contentUri: uploadResult.content_uri }; + } + + /** + * Download a file from the media repository on the homeserver. + * + * @param contentUri - the MXC URI of the file to download + * @returns an object with: file - response contents as Blob + */ + public async downloadFile(contentUri: string): Promise<{ file: XMLHttpRequestBodyInit }> { + const httpUrl = getHttpUriForMxc( + this.client.baseUrl, + contentUri, + undefined, + undefined, + undefined, + false, + undefined, + true + ); + const file = await downloadFromUrlToFile(httpUrl); + return { file }; + } + + /** + * Gets the IDs of all joined or invited rooms currently known to the + * client. + * @returns The room IDs. + */ + public getKnownRooms(): string[] { + return this.client.getVisibleRooms(false).map((r) => r.roomId); + } + + /** + * Expresses a {@link MatrixError} as a JSON payload + * for use by Widget API error responses. + * @param error The error to handle. + * @returns The error expressed as a JSON payload, + * or undefined if it is not a {@link MatrixError}. + */ + public processError(error: unknown): IWidgetApiErrorResponseDataDetails | undefined { + return error instanceof MatrixError + ? { + matrix_api_error: error.asWidgetApiErrorData(), + } + : undefined; + } +} From e18769f9c6137004401084a11e8a95cf19b1b22d Mon Sep 17 00:00:00 2001 From: Timo K Date: Sun, 12 Oct 2025 19:29:30 +0200 Subject: [PATCH 04/10] copy over StopGapWidget (ElementCallMessaging class) Signed-off-by: Timo K --- .../components/element-call/ElementCall.ts | 290 ++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 src/app/components/element-call/ElementCall.ts diff --git a/src/app/components/element-call/ElementCall.ts b/src/app/components/element-call/ElementCall.ts new file mode 100644 index 00000000..fb44d333 --- /dev/null +++ b/src/app/components/element-call/ElementCall.ts @@ -0,0 +1,290 @@ +import { + MatrixEvent, + MatrixClient, + MatrixEventEvent, + ClientEvent, + RoomStateEvent, + KnownMembership, + Room, +} from 'matrix-js-sdk'; + +import { ClientWidgetApi, type IRoomEvent, Widget } from 'matrix-widget-api'; +import { EventEmitter } from 'events'; +import { logger } from 'matrix-js-sdk/src/logger'; + +import { arrayFastClone, elementCallCapabilities } from './utils'; +import CallWidgetDriver from './CallWidgetDriver'; + +export enum ElementCallIntent { + StartCall = 'start_call', + JoinExisting = 'join_existing', + StartCallDM = 'start_call_dm', + JoinExistingDM = 'join_existing_dm', +} + +function createCallWidget(room: Room, client: MatrixClient, intent: string): Widget { + const perParticipantE2EE = room?.hasEncryptionStateEvent() ?? false; + + const baseUrl = new URL(window.location.href).origin; + const url = new URL('./widgets/element-call/index.html#', baseUrl); // this strips hash fragment from baseUrl + const widgetId = 'io-element-call-widget-id'; + // Splice together the Element Call URL for this call + const paramsHash = new URLSearchParams({ + perParticipantE2EE: perParticipantE2EE ? 'true' : 'false', + intent, + userId: client.getSafeUserId(), + deviceId: client.getDeviceId() ?? '', + roomId: room.roomId, + baseUrl: client.baseUrl, + lang: 'en-EN', + theme: 'light', + }); + const paramsSearch = new URLSearchParams({ + widgetId, + parentUrl: window.location.href.split('#', 2)[0], + }); + + url.search = paramsSearch.toString(); + const replacedUrl = paramsHash.toString().replace(/%24/g, '$'); + url.hash = `#?${replacedUrl}`; + + return new Widget({ + id: widgetId, + creatorUserId: client.getSafeUserId(), + name: 'Element Call', + type: 'm.call', + url: url.toString(), + waitForIframeLoad: false, + data: {}, + }); +} + +export default class ElementCall extends EventEmitter { + private messaging: ClientWidgetApi | null = null; + + public widget: Widget; + + private readUpToMap: { [roomId: string]: string } = {}; // room ID to event ID + + // Holds events that should be fed to the widget once they finish decrypting + private eventsToFeed = new WeakSet(); + + public constructor( + private client: MatrixClient, + private room: Room, + isDirect: boolean, + callOngoing: boolean + ) { + super(); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const intent = new Map([ + ['start_call_dm', ElementCallIntent.StartCallDM], + ['join_existing_dm', ElementCallIntent.JoinExistingDM], + ['start_call', ElementCallIntent.StartCall], + ['join_existing', ElementCallIntent.JoinExisting], + ]).get((callOngoing ? 'join_existing' : 'start_call') + (isDirect ? '_dm' : ''))!; + + this.widget = createCallWidget(this.room, this.client, intent); + } + + public get widgetApi(): ClientWidgetApi | null { + return this.messaging; + } + + /** + * The URL to use in the iframe + */ + public get embedUrl(): string { + return this.widget.templateUrl; + } + + /** + * This starts the messaging for the widget if it is not in the state `started` yet. + * @param iframe the iframe the widget should use + */ + public startMessaging(iframe: HTMLIFrameElement) { + if (this.messaging) return; + + const userId = this.client.getSafeUserId(); + const deviceId = this.client.getDeviceId() ?? undefined; + const driver = new CallWidgetDriver( + this.client, + elementCallCapabilities(this.room.roomId, userId, deviceId), + this.room.roomId + ); + + this.messaging = new ClientWidgetApi(this.widget, iframe, driver); + this.messaging.on('preparing', () => this.emit('preparing')); + this.messaging.on('error:preparing', (err: unknown) => this.emit('error:preparing', err)); + this.messaging.once('ready', () => { + this.emit('ready'); + }); + this.messaging.on('capabilitiesNotified', () => this.emit('capabilitiesNotified')); + + // Room widgets get locked to the room they were added in + this.messaging.setViewedRoomId(this.room.roomId); + + // Populate the map of "read up to" events for this widget with the current event in every room. + // This is a bit inefficient, but should be okay. We do this for all rooms in case the widget + // requests timeline capabilities in other rooms down the road. It's just easier to manage here. + this.client.getRooms().forEach((room) => { + // Timelines are most recent last + const events = room.getLiveTimeline()?.getEvents() || []; + const roomEvent = events[events.length - 1]; + if (!roomEvent) return; // force later code to think the room is fresh + this.readUpToMap[room.roomId] = roomEvent.getId()!; + }); + + // Attach listeners for feeding events - the underlying widget classes handle permissions for us + this.client.on(ClientEvent.Event, this.onEvent); + this.client.on(MatrixEventEvent.Decrypted, this.onEventDecrypted); + this.client.on(RoomStateEvent.Events, this.onStateUpdate); + this.client.on(ClientEvent.ToDeviceEvent, this.onToDeviceEvent); + } + + /** + * Stops the widget messaging for if it is started. Skips stopping if it is an active + * widget. + * @param opts + */ + public stopMessaging(): void { + if (this.messaging) { + this.messaging.stop(); + } + this.messaging = null; + + this.client.off(ClientEvent.Event, this.onEvent); + this.client.off(MatrixEventEvent.Decrypted, this.onEventDecrypted); + this.client.off(RoomStateEvent.Events, this.onStateUpdate); + this.client.off(ClientEvent.ToDeviceEvent, this.onToDeviceEvent); + + // Clear internal state + this.readUpToMap = {}; + this.eventsToFeed = new WeakSet(); + } + + private onEvent = (ev: MatrixEvent): void => { + this.client.decryptEventIfNeeded(ev); + this.feedEvent(ev); + }; + + private onEventDecrypted = (ev: MatrixEvent): void => { + this.feedEvent(ev); + }; + + private onStateUpdate = (ev: MatrixEvent): void => { + if (this.messaging === null) return; + const raw = ev.getEffectiveEvent(); + this.messaging.feedStateUpdate(raw as IRoomEvent).catch((e) => { + logger.error('Error sending state update to widget: ', e); + }); + }; + + private onToDeviceEvent = async (ev: MatrixEvent): Promise => { + await this.client.decryptEventIfNeeded(ev); + if (ev.isDecryptionFailure()) return; + await this.messaging?.feedToDevice(ev.getEffectiveEvent() as IRoomEvent, ev.isEncrypted()); + }; + + /** + * Determines whether the event has a relation to an unknown parent. + */ + private relatesToUnknown(ev: MatrixEvent): boolean { + // Replies to unknown events don't count + if (!ev.relationEventId || ev.replyEventId) return false; + const room = this.client.getRoom(ev.getRoomId()); + return room === null || !room.findEventById(ev.relationEventId); + } + + /** + * Advances the "read up to" marker for a room to a certain event. No-ops if + * the event is before the marker. + * @returns Whether the "read up to" marker was advanced. + */ + private advanceReadUpToMarker(ev: MatrixEvent): boolean { + const evId = ev.getId(); + if (evId === undefined) return false; + const roomId = ev.getRoomId(); + if (roomId === undefined) return false; + const room = this.client.getRoom(roomId); + if (room === null) return false; + + const upToEventId = this.readUpToMap[ev.getRoomId()!]; + if (!upToEventId) { + // There's no marker yet; start it at this event + this.readUpToMap[roomId] = evId; + return true; + } + + // Small optimization for exact match (skip the search) + if (upToEventId === evId) return false; + + // Timelines are most recent last, so reverse the order and limit ourselves to 100 events + // to avoid overusing the CPU. + const timeline = room.getLiveTimeline(); + const events = arrayFastClone(timeline.getEvents()).reverse().slice(0, 100); + function isRelevantTimelineEvent(timelineEvent: MatrixEvent): boolean { + return timelineEvent.getId() === upToEventId || timelineEvent.getId() === ev.getId(); + } + const possibleMarkerEv = events.find(isRelevantTimelineEvent); + if (possibleMarkerEv?.getId() === upToEventId) { + // The event must be somewhere before the "read up to" marker + return false; + } + if (possibleMarkerEv?.getId() === ev.getId()) { + // The event is after the marker; advance it + this.readUpToMap[roomId] = evId; + return true; + } + + // We can't say for sure whether the widget has seen the event; let's + // just assume that it has + return false; + } + + /** + * Determines whether the event comes from a room that we've been invited to + * (in which case we likely don't have the full timeline). + */ + private isFromInvite(ev: MatrixEvent): boolean { + const room = this.client.getRoom(ev.getRoomId()); + return room?.getMyMembership() === KnownMembership.Invite; + } + + private feedEvent(ev: MatrixEvent): void { + if (this.messaging === null) return; + if ( + // If we had decided earlier to feed this event to the widget, but + // it just wasn't ready, give it another try + this.eventsToFeed.delete(ev) || + // Skip marker timeline check for events with relations to unknown parent because these + // events are not added to the timeline here and will be ignored otherwise: + // https://github.com/matrix-org/matrix-js-sdk/blob/d3dfcd924201d71b434af3d77343b5229b6ed75e/src/models/room.ts#L2207-L2213 + this.relatesToUnknown(ev) || + // Skip marker timeline check for rooms where membership is + // 'invite', otherwise the membership event from the invitation room + // will advance the marker and new state events will not be + // forwarded to the widget. + this.isFromInvite(ev) || + // Check whether this event would be before or after our "read up to" marker. If it's + // before, or we can't decide, then we assume the widget will have already seen the event. + // If the event is after, or we don't have a marker for the room, + // then the marker will advance and we'll send it through. + // This approach of "read up to" prevents widgets receiving decryption spam from startup or + // receiving ancient events from backfill and such. + this.advanceReadUpToMarker(ev) + ) { + // If the event is still being decrypted, remember that we want to + // feed it to the widget (even if not strictly in the order given by + // the timeline) and get back to it later + if (ev.isBeingDecrypted() || ev.isDecryptionFailure()) { + this.eventsToFeed.add(ev); + } else { + const raw = ev.getEffectiveEvent(); + this.messaging.feedEvent(raw as IRoomEvent).catch((e) => { + logger.error('Error sending event to widget: ', e); + }); + } + } + } +} From 65085e84f775b8deb4a602029ba3058945d24a70 Mon Sep 17 00:00:00 2001 From: Timo K Date: Sun, 12 Oct 2025 19:31:32 +0200 Subject: [PATCH 05/10] Call View template Signed-off-by: Timo K --- src/app/components/element-call/CallView.tsx | 72 ++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/app/components/element-call/CallView.tsx diff --git a/src/app/components/element-call/CallView.tsx b/src/app/components/element-call/CallView.tsx new file mode 100644 index 00000000..2f1ecc33 --- /dev/null +++ b/src/app/components/element-call/CallView.tsx @@ -0,0 +1,72 @@ +import React, { useEffect, useRef, useState } from 'react'; +import { ClientWidgetApi } from 'matrix-widget-api'; +import { Button, Text } from 'folds'; +import ElementCall from './ElementCall'; +import { useMatrixClient } from '../../hooks/useMatrixClient'; +import { useEventEmitter } from './utils'; +import { useIsDirectRoom, useRoom } from '../../hooks/useRoom'; +import { useCallOngoing } from '../../hooks/useCallOngoing'; + +export enum CallWidgetActions { + // All of these actions are currently specific to Jitsi and Element Call + JoinCall = 'io.element.join', + HangupCall = 'im.vector.hangup', + Close = 'io.element.close', +} +export function action(type: CallWidgetActions): string { + return `action:${type}`; +} + +const iframeFeatures = + 'microphone; camera; encrypted-media; autoplay; display-capture; clipboard-write; ' + + 'clipboard-read;'; +const sandboxFlags = + 'allow-forms allow-popups allow-popups-to-escape-sandbox ' + + 'allow-same-origin allow-scripts allow-presentation allow-downloads'; +const iframeStyles = { flex: '1 1', border: 'none' }; +const containerStyle = (hidden: boolean): React.CSSProperties => ({ + display: 'flex', + flexDirection: 'column', + position: 'relative', + height: '100%', + width: '100%', + ...(hidden + ? { + overflow: 'hidden', + width: 0, + height: 0, + } + : {}), +}); +const closeButtonStyle: React.CSSProperties = { + position: 'absolute', + top: 8, + right: 8, + zIndex: 100, + maxWidth: 'fit-content', +}; + +enum State { + Preparing = 'preparing', + Lobby = 'lobby', + Joined = 'joined', + HungUp = 'hung_up', + CanClose = 'can_close', +} + +/** + * Shows a call for this room. Rendering this component will + * automatically create a call widget and join the call in the room. + * @returns + */ +export interface IRoomCallViewProps { + onClose?: () => void; + onJoin?: () => void; + onHangup?: (errorMessage?: string) => void; +} + +export function CallView({ + onJoin = undefined, + onClose = undefined, + onHangup = undefined, +}: IRoomCallViewProps): JSX.Element {} From 4985de841c9fbf14b3d1a1dc38c1b818f9e906b9 Mon Sep 17 00:00:00 2001 From: Timo K Date: Sun, 12 Oct 2025 19:32:11 +0200 Subject: [PATCH 06/10] call view impl Signed-off-by: Timo K --- src/app/components/element-call/CallView.tsx | 81 +++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/src/app/components/element-call/CallView.tsx b/src/app/components/element-call/CallView.tsx index 2f1ecc33..7aff6f49 100644 --- a/src/app/components/element-call/CallView.tsx +++ b/src/app/components/element-call/CallView.tsx @@ -69,4 +69,83 @@ export function CallView({ onJoin = undefined, onClose = undefined, onHangup = undefined, -}: IRoomCallViewProps): JSX.Element {} +}: IRoomCallViewProps): JSX.Element { + // Construct required variables + const room = useRoom(); + const client = useMatrixClient(); + const iframe = useRef(null); + + // Model state + const [elementCall, setElementCall] = useState(); + const [widgetApi, setWidgetApi] = useState(null); + const [state, setState] = useState(State.Preparing); + + // Initialization parameters + const isDirect = useIsDirectRoom(); + const callOngoing = useCallOngoing(room); + const initialCallOngoing = React.useRef(callOngoing); + const initialIsDirect = React.useRef(isDirect); + useEffect(() => { + if (client && room && !elementCall) { + const e = new ElementCall(client, room, initialIsDirect.current, initialCallOngoing.current); + setElementCall(e); + } + }, [client, room, setElementCall, elementCall]); + + // Start the messaging over the widget api. + useEffect(() => { + if (iframe.current && elementCall) { + elementCall.startMessaging(iframe.current); + } + return () => { + elementCall?.stopMessaging(); + }; + }, [iframe, elementCall]); + + // Widget api ready + useEventEmitter(elementCall, 'ready', () => { + setWidgetApi(elementCall?.widgetApi ?? null); + setState(State.Lobby); + }); + + // Use widget api to listen for hangup/join/close actions + useEventEmitter(widgetApi, action(CallWidgetActions.HangupCall), () => { + setState(State.HungUp); + onHangup?.(); + }); + useEventEmitter(widgetApi, action(CallWidgetActions.JoinCall), () => { + setState(State.Joined); + onJoin?.(); + }); + useEventEmitter(widgetApi, action(CallWidgetActions.Close), () => { + setState(State.CanClose); + onClose?.(); + }); + + // render component + return ( +
+ {/* Exit button for lobby state */} + {state === State.Lobby && ( + + )} +