From c53d14d91ead361ebc51b7134974d6ccd3a6ee72 Mon Sep 17 00:00:00 2001 From: Gigiaj Date: Sat, 12 Apr 2025 03:38:58 -0500 Subject: [PATCH 001/242] add groupcall state event --- src/types/matrix/room.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/types/matrix/room.ts b/src/types/matrix/room.ts index 65dc35f4..d41f73c7 100644 --- a/src/types/matrix/room.ts +++ b/src/types/matrix/room.ts @@ -30,6 +30,8 @@ export enum StateEvent { RoomGuestAccess = 'm.room.guest_access', RoomServerAcl = 'm.room.server_acl', RoomTombstone = 'm.room.tombstone', + GroupCallPrefix = "org.matrix.msc3401.call", + GroupCallMemberPrefix = "org.matrix.msc3401.call.member", SpaceChild = 'm.space.child', SpaceParent = 'm.space.parent', From 7e74fb146233c3a0ea9cd1b80392e461ba35da31 Mon Sep 17 00:00:00 2001 From: Gigiaj Date: Sat, 12 Apr 2025 03:39:21 -0500 Subject: [PATCH 002/242] docker nginx embedded element-call --- docker-nginx.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-nginx.conf b/docker-nginx.conf index a2dbeba0..aea395ae 100644 --- a/docker-nginx.conf +++ b/docker-nginx.conf @@ -14,6 +14,8 @@ server { rewrite ^/public/(.*)$ /public/$1 break; rewrite ^/assets/(.*)$ /assets/$1 break; + rewrite ^/element-call/dist/(.*)$ /element-call/dist/$1 break; + rewrite ^(.+)$ /index.html break; } } From 389fc17e454993da45b9950ec79f9f15d4c0ad76 Mon Sep 17 00:00:00 2001 From: Gigiaj Date: Sat, 12 Apr 2025 03:39:48 -0500 Subject: [PATCH 003/242] add embedded-element-call and react-sdk-module-api --- package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package.json b/package.json index b489dc2c..2ea322ee 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "@atlaskit/pragmatic-drag-and-drop-auto-scroll": "1.3.0", "@atlaskit/pragmatic-drag-and-drop-hitbox": "1.0.3", "@fontsource/inter": "4.5.14", + "@matrix-org/react-sdk-module-api": "^2.5.0", "@tanstack/react-query": "5.24.1", "@tanstack/react-query-devtools": "5.24.1", "@tanstack/react-virtual": "3.2.0", @@ -39,6 +40,7 @@ "dateformat": "5.0.3", "dayjs": "1.11.10", "domhandler": "5.0.3", + "@element-hq/element-call-embedded": "^0.9.0", "emojibase": "15.3.1", "emojibase-data": "15.3.2", "file-saver": "2.0.5", From ac0ac4ff352a17da66c620b988a744d370cd9f1e Mon Sep 17 00:00:00 2001 From: Gigiaj Date: Sat, 12 Apr 2025 03:39:59 -0500 Subject: [PATCH 004/242] add small widgetdriver --- src/app/features/room/SmallWidgetDriver.ts | 158 +++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 src/app/features/room/SmallWidgetDriver.ts diff --git a/src/app/features/room/SmallWidgetDriver.ts b/src/app/features/room/SmallWidgetDriver.ts new file mode 100644 index 00000000..0dc57065 --- /dev/null +++ b/src/app/features/room/SmallWidgetDriver.ts @@ -0,0 +1,158 @@ +import { + type Capability, + EventDirection, + type IOpenIDCredentials, + type IOpenIDUpdate, + type ISendDelayedEventDetails, + type ISendEventDetails, + type ITurnServer, + type IReadEventRelationsResult, + type IRoomEvent, + MatrixCapabilities, + OpenIDRequestState, + type SimpleObservable, + type Widget, + WidgetDriver, + WidgetEventCapability, + WidgetKind, + type IWidgetApiErrorResponseDataDetails, + type ISearchUserDirectoryResult, + type IGetMediaConfigResult, + type UpdateDelayedEventAction, +} from "matrix-widget-api"; +import { + ClientEvent, + type ITurnServer as IClientTurnServer, + EventType, + type IContent, + MatrixError, + type MatrixEvent, + Direction, + THREAD_RELATION_TYPE, + type SendDelayedEventResponse, + type StateEvents, + type TimelineEvents, + MatrixClient, +} from "matrix-js-sdk"; +import { + type ApprovalOpts, + type CapabilitiesOpts, + WidgetLifecycle, +} from "@matrix-org/react-sdk-module-api/lib/lifecycles/WidgetLifecycle"; + + +export class SmallWidgetDriver extends WidgetDriver { + private allowedCapabilities: Set; + private readonly mxClient: MatrixClient; // Store the client instance + + public constructor( + mx: MatrixClient, + allowedCapabilities: Capability[], + private forWidget: Widget, + private forWidgetKind: WidgetKind, + virtual: boolean, + private inRoomId?: string, + ) { + super(); + this.mxClient = mx; // Store the passed instance + + this.allowedCapabilities = new Set([ + ...allowedCapabilities, + MatrixCapabilities.Screenshots, + ]); + + // This is a trusted Element Call widget that we control + this.allowedCapabilities.add(MatrixCapabilities.AlwaysOnScreen); + this.allowedCapabilities.add(MatrixCapabilities.MSC3846TurnServers); + + // Capability to access the room timeline (MSC2762) + // Ensure inRoomId is correctly passed during SmallWidgetDriver instantiation + if (inRoomId) { + this.allowedCapabilities.add(`org.matrix.msc2762.timeline:${inRoomId}`); + } else { + console.warn("inRoomId is undefined, cannot add timeline capability."); + } + this.allowedCapabilities.add(MatrixCapabilities.MSC4157SendDelayedEvent); + this.allowedCapabilities.add(MatrixCapabilities.MSC4157UpdateDelayedEvent); + + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent(EventDirection.Receive, EventType.RoomMember).raw, + ); + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent(EventDirection.Receive, "org.matrix.msc3401.call").raw, + ); + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent(EventDirection.Receive, EventType.RoomEncryption).raw, + ); + const clientUserId = this.mxClient.getUserId(); + // For the legacy membership type + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent(EventDirection.Send, "org.matrix.msc3401.call.member", clientUserId ?? undefined) + .raw, + ); + const clientDeviceId = this.mxClient.getDeviceId(); + if (clientDeviceId !== null) { + // For the session membership type compliant with MSC4143 + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent( + EventDirection.Send, + "org.matrix.msc3401.call.member", + `_${clientUserId}_${clientDeviceId}`, + ).raw, + ); + // Version with no leading underscore, for room versions whose auth rules allow it + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent( + EventDirection.Send, + "org.matrix.msc3401.call.member", + `${clientUserId}_${clientDeviceId}`, + ).raw, + ); + } + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent(EventDirection.Receive, "org.matrix.msc3401.call.member").raw, + ); + // for determining auth rules specific to the room version + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent(EventDirection.Receive, EventType.RoomCreate).raw, + ); + + const sendRecvRoomEvents = [ + "io.element.call.encryption_keys", + "org.matrix.rageshake_request", + EventType.Reaction, + EventType.RoomRedaction, + "io.element.call.reaction", + ]; + for (const eventType of sendRecvRoomEvents) { + this.allowedCapabilities.add(WidgetEventCapability.forRoomEvent(EventDirection.Send, eventType).raw); + this.allowedCapabilities.add(WidgetEventCapability.forRoomEvent(EventDirection.Receive, eventType).raw); + } + + const sendRecvToDevice = [ + EventType.CallInvite, + EventType.CallCandidates, + EventType.CallAnswer, + EventType.CallHangup, + EventType.CallReject, + EventType.CallSelectAnswer, + EventType.CallNegotiate, + EventType.CallSDPStreamMetadataChanged, + EventType.CallSDPStreamMetadataChangedPrefix, + EventType.CallReplaces, + EventType.CallEncryptionKeysPrefix, + ]; + for (const eventType of sendRecvToDevice) { + this.allowedCapabilities.add( + WidgetEventCapability.forToDeviceEvent(EventDirection.Send, eventType).raw, + ); + this.allowedCapabilities.add( + WidgetEventCapability.forToDeviceEvent(EventDirection.Receive, eventType).raw, + ); + } + + // To always allow OIDC requests for element call, the widgetPermissionStore is used: + + + } +} \ No newline at end of file From aa65dd57ba1e1b4d57cdb20e845b730a1d08f724 Mon Sep 17 00:00:00 2001 From: Gigiaj Date: Sat, 12 Apr 2025 03:40:13 -0500 Subject: [PATCH 005/242] add scaffolding for widget-based call --- src/app/features/room/RoomView.tsx | 145 ++++++++++++++++++++++++++--- 1 file changed, 131 insertions(+), 14 deletions(-) diff --git a/src/app/features/room/RoomView.tsx b/src/app/features/room/RoomView.tsx index b6eebdf2..43912c87 100644 --- a/src/app/features/room/RoomView.tsx +++ b/src/app/features/room/RoomView.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useRef } from 'react'; +import React, { useCallback, useRef, useEffect } from 'react'; // Added useEffect import { Box, Text, config } from 'folds'; import { EventType, Room } from 'matrix-js-sdk'; import { ReactEditor } from 'slate-react'; @@ -23,6 +23,9 @@ import { settingsAtom } from '../../state/settings'; import { useSetting } from '../../state/hooks/settings'; import { useAccessibleTagColors, usePowerLevelTags } from '../../hooks/usePowerLevelTags'; import { useTheme } from '../../hooks/useTheme'; +import { logger } from 'matrix-js-sdk/lib/logger'; +import { ClientWidgetApi, Widget, WidgetKind } from 'matrix-widget-api'; +import { SmallWidgetDriver } from './SmallWidgetDriver'; const FN_KEYS_REGEX = /^F\d+$/; const shouldFocusMessageField = (evt: KeyboardEvent): boolean => { @@ -57,9 +60,38 @@ const shouldFocusMessageField = (evt: KeyboardEvent): boolean => { return true; }; +// Keep this function to generate the URL +const getWidgetUrl = (mx, roomId) => { + const baseUrl = window.location.href; + const params = new URLSearchParams({ + embed: "true", // We're embedding EC within another application + widgetId: "test", + // Template variables are used, so that this can be configured using the data. + preload: "$preload", // We want it to load in the background. + // skipLobby: "true", // Skip the lobby in case we show a lobby component of our own. + returnToLobby: "$returnToLobby", // Returns to the lobby (instead of blank screen) when the call ends. (For video rooms) + perParticipantE2EE: "$perParticipantE2EE", + hideHeader: "true", // Hide the header since our room header is enough + userId: mx.getUserId()!, + deviceId: mx.getDeviceId()!, + roomId: roomId, + baseUrl: window.location.href, + parentUrl: baseUrl, + // lang: getCurrentLanguage().replace("_", "-"), + // fontScale: (FontWatcher.getRootFontSize() / FontWatcher.getBrowserDefaultFontSize()).toString(), + theme: "$org.matrix.msc2873.client_theme", +}); +const replacedUrl = params.toString().replace(/%24/g, "$"); +const url= 'https://elementcall.example.quest' + `#?${replacedUrl}`; +logger.error(url); + return url; +} + + export function RoomView({ room, eventId }: { room: Room; eventId?: string }) { const roomInputRef = useRef(null); const roomViewRef = useRef(null); + const iframeRef = useRef(null); // Ref for the iframe const [hideActivity] = useSetting(settingsAtom, 'hideActivity'); @@ -80,31 +112,117 @@ export function RoomView({ room, eventId }: { room: Room; eventId?: string }) { const theme = useTheme(); const accessibleTagColors = useAccessibleTagColors(theme.kind, powerLevelTags); + const isCall = room.isCallRoom(); // Determine if it's a call room + useKeyDown( window, useCallback( (evt) => { if (editableActiveElement()) return; - if ( - document.body.lastElementChild?.className !== 'ReactModalPortal' || - navigation.isRawModalVisible - ) { - return; + // Check modal visibility more robustly if needed + if (document.querySelector('.ReactModalPortal > *')) { // Simple check if any modal portal has content + if (navigation.isRawModalVisible) return; // Skip if raw modal is explicitly visible + // Add other modal checks if necessary } + if (shouldFocusMessageField(evt) || isKeyHotkey('mod+v', evt)) { - ReactEditor.focus(editor); + // Only focus editor if not in a call view where editor isn't present + if (!isCall && editor) { + ReactEditor.focus(editor); + } } }, - [editor] + [editor, isCall] // Add isCall dependency ) ); + // Effect to setup the widget API when the iframe is mounted for a call room + useEffect(() => { + let widgetApi: ClientWidgetApi | null = null; + let driver: SmallWidgetDriver | null = null; + + // Only run setup if it's a call room and the iframe ref is available + if (isCall && iframeRef.current) { + const iframe = iframeRef.current; + const url = getWidgetUrl(mx, roomId); + + // Update iframe src if necessary (though it's set in JSX, this ensures it if URL changes) + if (iframe.src !== url) { + iframe.src = url; + } + + logger.info(`Setting up widget API for room ${roomId}`); + + const widget = new Widget({ + id: 'test-call-widget', // Match ID used in URL params + creatorUserId: mx.getUserId()!, + type: 'm.custom', // Or appropriate widget type e.g., m.video + url: url, + roomId: roomId, // Pass roomId if needed by Widget constructor + waitForIframeLoad: false, + // Add other necessary Widget properties + }); + + // Ensure driver is correctly instantiated with necessary parameters + // The second argument `[]` might need adjustment based on SmallWidgetDriver's needs (e.g., allowed capabilities) + driver = new SmallWidgetDriver(mx, [], widget, WidgetKind.Room, true, roomId); + + widgetApi = new ClientWidgetApi(widget, iframe, driver); + // widgetApi.start(); // Start communication if required by your setup + + // Return a cleanup function + return () => { + logger.info(`Cleaning up widget API for room ${roomId}`); + // Implement proper cleanup for ClientWidgetApi and SmallWidgetDriver + // This might involve calling stop methods, removing listeners, etc. + // Example: widgetApi?.stop(); + // Example: driver?.stop(); + widgetApi = null; + driver = null; + // Clear iframe src to stop loading/activity + if (iframeRef.current) { + iframeRef.current.src = 'about:blank'; + } + }; + } + + // If it's not a call room or the iframe isn't ready, ensure no setup runs/is cleaned up + return undefined; + + }, [isCall, mx, roomId]); // Dependencies: run effect if call status, client, or room ID changes + + + // Render Call View + if (isCall) { + const url = getWidgetUrl(mx, roomId); + return ( + // Attach roomViewRef here if is the main container you want to reference + + + {/* Embed the iframe directly. Ensure parent has definite height or use flex grow */} + {/* Use Box with grow */} +