diff --git a/config.json b/config.json index de6015a1..b307bb1d 100644 --- a/config.json +++ b/config.json @@ -9,6 +9,7 @@ "xmr.se" ], "allowCustomHomeservers": true, + "elementCallUrl": null, "featuredCommunities": { "openAsDefault": false, diff --git a/docker-nginx.conf b/docker-nginx.conf index a2dbeba0..aea395ae 100644 --- a/docker-nginx.conf +++ b/docker-nginx.conf @@ -14,6 +14,8 @@ server { rewrite ^/public/(.*)$ /public/$1 break; rewrite ^/assets/(.*)$ /assets/$1 break; + rewrite ^/element-call/dist/(.*)$ /element-call/dist/$1 break; + rewrite ^(.+)$ /index.html break; } } diff --git a/package-lock.json b/package-lock.json index 3fc29010..968eebcb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@atlaskit/pragmatic-drag-and-drop-auto-scroll": "1.3.0", "@atlaskit/pragmatic-drag-and-drop-hitbox": "1.0.3", "@fontsource/inter": "4.5.14", + "@matrix-org/react-sdk-module-api": "2.5.0", "@tanstack/react-query": "5.24.1", "@tanstack/react-query-devtools": "5.24.1", "@tanstack/react-virtual": "3.2.0", @@ -47,6 +48,7 @@ "linkify-react": "4.1.3", "linkifyjs": "4.1.3", "matrix-js-sdk": "37.5.0", + "matrix-widget-api": "1.11.0", "millify": "6.1.0", "pdfjs-dist": "4.2.67", "prismjs": "1.30.0", @@ -72,6 +74,7 @@ "ua-parser-js": "1.0.35" }, "devDependencies": { + "@element-hq/element-call-embedded": "0.12.2", "@esbuild-plugins/node-globals-polyfill": "0.2.3", "@rollup/plugin-inject": "5.0.3", "@rollup/plugin-wasm": "6.1.1", @@ -1657,6 +1660,12 @@ "node": ">=6.9.0" } }, + "node_modules/@element-hq/element-call-embedded": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/@element-hq/element-call-embedded/-/element-call-embedded-0.12.2.tgz", + "integrity": "sha512-2u5/bOARcjc5TFq4929x1R0tvsNbeVA58FBtiW05GlIJCapxzPSOeeGhbqEcJ1TW3/hLGpiKMcw0QwRBQVNzQA==", + "dev": true + }, "node_modules/@emotion/hash": { "version": "0.9.2", "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.2.tgz", @@ -2278,6 +2287,18 @@ "integrity": "sha512-S7lOrndAK9/8qOtaTq/WhttJC/o4GAzdfK0MUPpo8ApzsJEC0QjtwrkC3KBXdFP1cD1MXi/mlKR7aaoVMKgs6Q==", "license": "Apache-2.0" }, + "node_modules/@matrix-org/react-sdk-module-api": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@matrix-org/react-sdk-module-api/-/react-sdk-module-api-2.5.0.tgz", + "integrity": "sha512-l/SmiO47gPIRd6YJJGj+B6qbxyypJF6SEsfYr7j9rSW6E85ZYCqf+TpMM2LmfwZRADyKfCVkaJbbBZYpoD02VA==", + "license": "Apache-2.0", + "dependencies": { + "@babel/runtime": "^7.17.9" + }, + "peerDependencies": { + "react": "^18" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -8826,9 +8847,9 @@ } }, "node_modules/matrix-widget-api": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/matrix-widget-api/-/matrix-widget-api-1.13.1.tgz", - "integrity": "sha512-mkOHUVzaN018TCbObfGOSaMW2GoUxOfcxNNlTVx5/HeMk3OSQPQM0C9oEME5Liiv/dBUoSrEB64V8wF7e/gb1w==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/matrix-widget-api/-/matrix-widget-api-1.11.0.tgz", + "integrity": "sha512-ED/9hrJqDWVLeED0g1uJnYRhINh3ZTquwurdM+Hc8wLVJIQ8G/r7A7z74NC+8bBIHQ1Jo7i1Uq5CoJp/TzFYrA==", "license": "Apache-2.0", "dependencies": { "@types/events": "^3.0.0", diff --git a/package.json b/package.json index 81d0e20a..8eea2b37 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "scripts": { "start": "vite", "build": "vite build", + "preview": "vite preview", "lint": "yarn check:eslint && yarn check:prettier", "check:eslint": "eslint src/*", "check:prettier": "prettier --check .", @@ -24,6 +25,7 @@ "@atlaskit/pragmatic-drag-and-drop-auto-scroll": "1.3.0", "@atlaskit/pragmatic-drag-and-drop-hitbox": "1.0.3", "@fontsource/inter": "4.5.14", + "@matrix-org/react-sdk-module-api": "2.5.0", "@tanstack/react-query": "5.24.1", "@tanstack/react-query-devtools": "5.24.1", "@tanstack/react-virtual": "3.2.0", @@ -57,6 +59,7 @@ "jotai": "2.6.0", "linkify-react": "4.1.3", "linkifyjs": "4.1.3", + "matrix-widget-api": "1.11.0", "matrix-js-sdk": "37.5.0", "millify": "6.1.0", "pdfjs-dist": "4.2.67", @@ -83,6 +86,7 @@ "ua-parser-js": "1.0.35" }, "devDependencies": { + "@element-hq/element-call-embedded": "0.12.2", "@esbuild-plugins/node-globals-polyfill": "0.2.3", "@rollup/plugin-inject": "5.0.3", "@rollup/plugin-wasm": "6.1.1", @@ -115,4 +119,5 @@ "vite-plugin-static-copy": "1.0.4", "vite-plugin-top-level-await": "1.4.4" } + } diff --git a/src/app/components/room-avatar/RoomAvatar.tsx b/src/app/components/room-avatar/RoomAvatar.tsx index 23f3998d..60de8346 100644 --- a/src/app/components/room-avatar/RoomAvatar.tsx +++ b/src/app/components/room-avatar/RoomAvatar.tsx @@ -46,10 +46,11 @@ export const RoomIcon = forwardRef< Omit, 'src'> & { joinRule: JoinRule; space?: boolean; + call?: boolean; } ->(({ joinRule, space, ...props }, ref) => ( +>(({ joinRule, space, call, ...props }, ref) => ( diff --git a/src/app/features/call/CallView.tsx b/src/app/features/call/CallView.tsx new file mode 100644 index 00000000..516a7909 --- /dev/null +++ b/src/app/features/call/CallView.tsx @@ -0,0 +1,155 @@ +/* eslint-disable no-nested-ternary */ +import { Room } from 'matrix-js-sdk'; +import React, { useContext, useMemo, useCallback, useEffect, useRef } from 'react'; +import { Box } from 'folds'; +import { useCallState } from '../../pages/client/call/CallProvider'; +import { + PrimaryRefContext, + BackupRefContext, +} from '../../pages/client/call/PersistentCallContainer'; +import { ScreenSize, useScreenSizeContext } from '../../hooks/useScreenSize'; + +function debounce any>(func: F, waitFor: number) { + let timeoutId: ReturnType | null = null; + return (...args: Parameters): void => { + if (timeoutId) { + clearTimeout(timeoutId); + } + timeoutId = setTimeout(() => func(...args), waitFor); + }; +} + +type OriginalStyles = { + position?: string; + top?: string; + left?: string; + width?: string; + height?: string; + zIndex?: string; + display?: string; + visibility?: string; + pointerEvents?: string; + border?: string; +}; + +export function CallView({ room }: { room: Room }) { + const primaryIframeRef = useContext(PrimaryRefContext); + const backupIframeRef = useContext(BackupRefContext); + const iframeHostRef = useRef(null); + + const originalIframeStylesRef = useRef(null); + const { activeCallRoomId, isPrimaryIframe, isChatOpen } = useCallState(); + const isViewingActiveCall = useMemo( + () => activeCallRoomId !== null && activeCallRoomId === room.roomId, + [activeCallRoomId, room.roomId] + ); + + const screenSize = useScreenSizeContext(); + const isMobile = screenSize === ScreenSize.Mobile; + const activeIframeDisplayRef = isPrimaryIframe + ? isViewingActiveCall + ? primaryIframeRef + : backupIframeRef + : isViewingActiveCall + ? backupIframeRef + : primaryIframeRef; + + const applyFixedPositioningToIframe = useCallback(() => { + const iframeElement = activeIframeDisplayRef?.current; + const hostElement = iframeHostRef?.current; + + if (iframeElement && hostElement) { + if (!originalIframeStylesRef.current) { + const computed = window.getComputedStyle(iframeElement); + originalIframeStylesRef.current = { + position: iframeElement.style.position || computed.position, + top: iframeElement.style.top || computed.top, + left: iframeElement.style.left || computed.left, + width: iframeElement.style.width || computed.width, + height: iframeElement.style.height || computed.height, + zIndex: iframeElement.style.zIndex || computed.zIndex, + display: iframeElement.style.display || computed.display, + visibility: iframeElement.style.visibility || computed.visibility, + pointerEvents: iframeElement.style.pointerEvents || computed.pointerEvents, + border: iframeElement.style.border || computed.border, + }; + } + + const hostRect = hostElement.getBoundingClientRect(); + + iframeElement.style.position = 'fixed'; + iframeElement.style.top = `${hostRect.top}px`; + iframeElement.style.left = `${hostRect.left}px`; + iframeElement.style.width = `${hostRect.width}px`; + iframeElement.style.height = `${hostRect.height}px`; + iframeElement.style.border = 'none'; + iframeElement.style.zIndex = '1000'; + iframeElement.style.display = room.isCallRoom() ? 'block' : 'none'; + iframeElement.style.visibility = 'visible'; + iframeElement.style.pointerEvents = 'auto'; + } + }, [activeIframeDisplayRef, room]); + + const debouncedApplyFixedPositioning = useCallback(debounce(applyFixedPositioningToIframe, 50), [ + applyFixedPositioningToIframe, + primaryIframeRef, + backupIframeRef, + ]); + useEffect(() => { + const iframeElement = activeIframeDisplayRef?.current; + const hostElement = iframeHostRef?.current; + + if (room.isCallRoom() || (isViewingActiveCall && iframeElement && hostElement)) { + applyFixedPositioningToIframe(); + + const resizeObserver = new ResizeObserver(debouncedApplyFixedPositioning); + resizeObserver.observe(hostElement); + window.addEventListener('scroll', debouncedApplyFixedPositioning, true); + + return () => { + resizeObserver.disconnect(); + window.removeEventListener('scroll', debouncedApplyFixedPositioning, true); + + if (iframeElement && originalIframeStylesRef.current) { + const originalStyles = originalIframeStylesRef.current; + (Object.keys(originalStyles) as Array).forEach((key) => { + if (key in iframeElement.style) { + iframeElement.style[key as any] = originalStyles[key] || ''; + } + }); + } + originalIframeStylesRef.current = null; + }; + } + }, [ + activeIframeDisplayRef, + applyFixedPositioningToIframe, + debouncedApplyFixedPositioning, + isPrimaryIframe, + isViewingActiveCall, + room, + ]); + + const isCallViewVisible = room.isCallRoom(); + + return ( + +
+ + ); +} diff --git a/src/app/features/call/CinnyWidget.ts b/src/app/features/call/CinnyWidget.ts new file mode 100644 index 00000000..3894af87 --- /dev/null +++ b/src/app/features/call/CinnyWidget.ts @@ -0,0 +1,9 @@ +import { Widget } from 'matrix-widget-api'; +import { IApp } from './SmallWidget'; + +// Wrapper class for the widget definition +export class CinnyWidget extends Widget { + public constructor(private rawDefinition: IApp) { + super(rawDefinition); + } +} diff --git a/src/app/features/call/SmallWidget.ts b/src/app/features/call/SmallWidget.ts new file mode 100644 index 00000000..2b24e978 --- /dev/null +++ b/src/app/features/call/SmallWidget.ts @@ -0,0 +1,410 @@ +/* + * Copyright 2024 New Vector Ltd. + * Copyright 2020-2023 The Matrix.org Foundation C.I.C. + * + * SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial + * Please see LICENSE files in the repository root for full details. + */ + +import EventEmitter from 'events'; +import { + ClientEvent, + Direction, + IEvent, + KnownMembership, + MatrixClient, + MatrixEvent, + MatrixEventEvent, +} from 'matrix-js-sdk'; +import { + ClientWidgetApi, + IRoomEvent, + IStickyActionRequest, + IWidget, + IWidgetData, + MatrixCapabilities, + WidgetApiFromWidgetAction, + WidgetKind, +} from 'matrix-widget-api'; +import { logger } from 'matrix-js-sdk/lib/logger'; +import { CinnyWidget } from './CinnyWidget'; +import { SmallWidgetDriver } from './SmallWidgetDriver'; + +/** + * Generates the URL for the Element Call widget. + * @param mx - The MatrixClient instance. + * @param roomId - The ID of the room. + * @returns The generated URL object. + */ +export const getWidgetUrl = ( + mx: MatrixClient, + roomId: string, + elementCallUrl: string, + widgetId: string, + setParams: any +): URL => { + const baseUrl = window.location.origin; + const url = elementCallUrl + ? new URL(`${elementCallUrl}/room`) + : new URL('/public/element-call/index.html#', baseUrl); + + const params = new URLSearchParams({ + embed: 'true', + widgetId, + appPrompt: 'false', + preload: 'false', + skipLobby: setParams.skipLobby ?? 'true', + returnToLobby: setParams.returnToLobby ?? 'true', + perParticipantE2EE: setParams.perParticipantE2EE ?? 'true', + hideHeader: 'true', + userId: mx.getUserId()!, + deviceId: mx.getDeviceId()!, + roomId, + baseUrl: mx.baseUrl!, + parentUrl: window.location.origin, + }); + + const replacedParams = params.toString().replace(/%24/g, '$'); + url.search = `?${replacedParams}`; + + logger.info('Generated Element Call Widget URL:', url.toString()); + return url; +}; + +export interface IApp extends IWidget { + client: MatrixClient; + roomId: string; + eventId?: string; + avatar_url?: string; + 'io.element.managed_hybrid'?: boolean; +} + +export class SmallWidget extends EventEmitter { + private client: MatrixClient; + + private messaging: ClientWidgetApi | null = null; + + private mockWidget: CinnyWidget; + + public roomId?: string; + + public url?: string; + + public iframe: HTMLElement | null; + + private type: string; // Type of the widget (e.g., 'm.call') + + private readUpToMap: { [roomId: string]: string } = {}; // room ID to event ID + + private readonly eventsToFeed = new WeakSet(); + + private stickyPromise?: () => Promise; + + constructor(private iapp: IApp) { + super(); + this.client = iapp.client; + this.roomId = iapp.roomId; + this.url = iapp.url; + this.type = iapp.type; + this.mockWidget = new CinnyWidget(iapp); + } + + /** + * Initializes the widget messaging API. + * @param iframe - The HTMLIFrameElement to bind to. + * @returns The initialized ClientWidgetApi instance. + */ + startMessaging(iframe: HTMLIFrameElement): ClientWidgetApi { + // Ensure the driver is correctly instantiated + // The capabilities array might need adjustment based on required permissions + const driver = new SmallWidgetDriver( + this.client, + [], + this.mockWidget, + WidgetKind.Room, + true, + this.roomId + ); + this.iframe = iframe; + this.messaging = new ClientWidgetApi(this.mockWidget, iframe, driver); + + // Emit events during the widget lifecycle + this.messaging.on('preparing', () => this.emit('preparing')); + this.messaging.on('error:preparing', (err: unknown) => this.emit('error:preparing', err)); + this.messaging.once('ready', () => this.emit('ready')); + // this.messaging.on("capabilitiesNotified", () => this.emit("capabilitiesNotified")); // Uncomment if needed + + // Populate the map of "read up to" events for this widget with the current event in every room. + // This is a bit inefficient, but should be okay. We do this for all rooms in case the widget + // requests timeline capabilities in other rooms down the road. It's just easier to manage here. + for (const room of this.client.getRooms()) { + // Timelines are most recent last + const events = room.getLiveTimeline()?.getEvents() || []; + const roomEvent = events[events.length - 1]; + if (!roomEvent) continue; // force later code to think the room is fresh + this.readUpToMap[room.roomId] = roomEvent.getId()!; + } + + this.messaging.on('action:org.matrix.msc2876.read_events', (ev: CustomEvent) => { + const room = this.client.getRoom(this.roomId); + const events: Partial[] = []; + const { type } = ev.detail.data; + + ev.preventDefault(); + if (room === null) { + return this.messaging?.transport.reply(ev.detail, { events }); + } + const state = room.getLiveTimeline().getState(Direction.Forward); + if (state === undefined) { + return this.messaging?.transport.reply(ev.detail, { events }); + } + + const stateEvents = state.events?.get(type); + + for (const [key, eventObject] of stateEvents?.entries() ?? []) { + events.push(eventObject.event); + } + + return this.messaging?.transport.reply(ev.detail, { events }); + }); + + /* + this.messaging?.on('action:content_loaded', () => { + this.messaging?.transport?.send('io.element.join', { + audioInput: 'true', + videoInput: 'true', + }); + }); + */ + + this.client.on(ClientEvent.Event, this.onEvent); + this.client.on(MatrixEventEvent.Decrypted, this.onEventDecrypted); + //this.client.on(RoomStateEvent.Events, this.onStateUpdate); + this.client.on(ClientEvent.ToDeviceEvent, this.onToDeviceEvent); + //this.client.on(RoomStateEvent.Events, this.onReadEvent); + // this.messaging.setViewedRoomId(this.roomId ?? null); + this.messaging.on( + `action:${WidgetApiFromWidgetAction.UpdateAlwaysOnScreen}`, + async (ev: CustomEvent) => { + if (this.messaging?.hasCapability(MatrixCapabilities.AlwaysOnScreen)) { + ev.preventDefault(); + if (ev.detail.data.value) { + // If the widget wants to become sticky we wait for the stickyPromise to resolve + if (this.stickyPromise) await this.stickyPromise(); + this.messaging.transport.reply(ev.detail, {}); + } + // Stop being persistent can be done instantly + //MAKE PERSISTENT HERE + // Send the ack after the widget actually has become sticky. + } + } + ); + + logger.info(`Widget messaging started for widgetId: ${this.mockWidget.id}`); + return this.messaging; + } + + private onEvent = (ev: MatrixEvent): void => { + this.client.decryptEventIfNeeded(ev); + this.feedEvent(ev); + }; + + private onEventDecrypted = (ev: MatrixEvent): void => { + this.feedEvent(ev); + }; + + private onReadEvent = (ev: MatrixEvent): void => { + this.feedEvent(ev); + }; + + private onToDeviceEvent = async (ev: MatrixEvent): Promise => { + await this.client.decryptEventIfNeeded(ev); + if (ev.isDecryptionFailure()) return; + await this.messaging?.feedToDevice(ev.getEffectiveEvent() as IRoomEvent, ev.isEncrypted()); + }; + + /** + * Determines whether the event comes from a room that we've been invited to + * (in which case we likely don't have the full timeline). + */ + private isFromInvite(ev: MatrixEvent): boolean { + const room = this.client.getRoom(ev.getRoomId()); + return room?.getMyMembership() === KnownMembership.Invite; + } + + /** + * Determines whether the event has a relation to an unknown parent. + */ + private relatesToUnknown(ev: MatrixEvent): boolean { + // Replies to unknown events don't count + if (!ev.relationEventId || ev.replyEventId) return false; + const room = this.client.getRoom(ev.getRoomId()); + return room === null || !room.findEventById(ev.relationEventId); + } + + // eslint-disable-next-line class-methods-use-this + private arrayFastClone(a: T[]): T[] { + return a.slice(0, a.length); + } + + private advanceReadUpToMarker(ev: MatrixEvent): boolean { + const evId = ev.getId(); + if (evId === undefined) return false; + const roomId = ev.getRoomId(); + if (roomId === undefined) return false; + const room = this.client.getRoom(roomId); + if (room === null) return false; + + const upToEventId = this.readUpToMap[ev.getRoomId()!]; + if (!upToEventId) { + // There's no marker yet; start it at this event + this.readUpToMap[roomId] = evId; + return true; + } + + // Small optimization for exact match (skip the search) + if (upToEventId === evId) return false; + + // Timelines are most recent last, so reverse the order and limit ourselves to 100 events + // to avoid overusing the CPU. + const timeline = room.getLiveTimeline(); + const events = this.arrayFastClone(timeline.getEvents()).reverse().slice(0, 100); + + for (const timelineEvent of events) { + if (timelineEvent.getId() === upToEventId) { + // The event must be somewhere before the "read up to" marker + return false; + } + if (timelineEvent.getId() === ev.getId()) { + // The event is after the marker; advance it + this.readUpToMap[roomId] = evId; + return true; + } + } + + // We can't say for sure whether the widget has seen the event; let's + // just assume that it has + return false; + } + + private feedEvent(ev: MatrixEvent): void { + if (this.messaging === null) return; + + if ( + // If we had decided earlier to feed this event to the widget, but + // it just wasn't ready, give it another try + this.eventsToFeed.delete(ev) || + // Skip marker timeline check for events with relations to unknown parent because these + // events are not added to the timeline here and will be ignored otherwise: + // https://github.com/matrix-org/matrix-js-sdk/blob/d3dfcd924201d71b434af3d77343b5229b6ed75e/src/models/room.ts#L2207-L2213 + this.relatesToUnknown(ev) || + // Skip marker timeline check for rooms where membership is + // 'invite', otherwise the membership event from the invitation room + // will advance the marker and new state events will not be + // forwarded to the widget. + this.isFromInvite(ev) || + // Check whether this event would be before or after our "read up to" marker. If it's + // before, or we can't decide, then we assume the widget will have already seen the event. + // If the event is after, or we don't have a marker for the room, then the marker will advance and we'll + // send it through. + // This approach of "read up to" prevents widgets receiving decryption spam from startup or + // receiving ancient events from backfill and such. + this.advanceReadUpToMarker(ev) + ) { + // If the event is still being decrypted, remember that we want to + // feed it to the widget (even if not strictly in the order given by + // the timeline) and get back to it later + if (ev.isBeingDecrypted() || ev.isDecryptionFailure()) { + this.eventsToFeed.add(ev); + } else { + const raw = ev.getEffectiveEvent(); + this.messaging.feedEvent(raw as IRoomEvent, this.roomId ?? '').catch((e) => { + logger.error('Error sending event to widget: ', e); + }); + } + } + } + + /** + * Stops the widget messaging and cleans up resources. + */ + stopMessaging() { + if (this.messaging) { + this.messaging.stop(); // Example if a stop method exists + this.messaging.removeAllListeners(); // Remove listeners attached by SmallWidget + logger.info(`Widget messaging stopped for widgetId: ${this.mockWidget.id}`); + this.messaging = null; + } + } +} + +/** + * Creates the data object for the widget. + * @param client - The MatrixClient instance. + * @param roomId - The ID of the room. + * @param currentData - Existing widget data. + * @param overwriteData - Data to merge or overwrite. + * @returns The final widget data object. + */ +export const getWidgetData = ( + client: MatrixClient, + roomId: string, + currentData: object, + overwriteData: object +): IWidgetData => { + // Example: Determine E2EE based on room state if needed + const perParticipantE2EE = true; // Default or based on logic + // const roomEncryption = client.getRoom(roomId)?.currentState.getStateEvents(EventType.RoomEncryption, ""); + // if (roomEncryption) perParticipantE2EE = true; // Simplified example + + return { + ...currentData, + ...overwriteData, + perParticipantE2EE, + }; +}; + +/** + * Creates a virtual widget definition (IApp). + * @param client - MatrixClient instance. + * @param id - Widget ID. + * @param creatorUserId - User ID of the creator. + * @param name - Widget display name. + * @param type - Widget type (e.g., 'm.call'). + * @param url - Widget URL. + * @param waitForIframeLoad - Whether to wait for iframe load signal. + * @param data - Widget data. + * @param roomId - Room ID. + * @returns The IApp widget definition. + */ +export const createVirtualWidget = ( + client: MatrixClient, + id: string, + creatorUserId: string, + name: string, + type: string, + url: URL, + waitForIframeLoad: boolean, + data: IWidgetData, + roomId: string +): IApp => ({ + client, + id, + creatorUserId, + name, + type, + url: url.toString(), // Store URL as string in the definition + waitForIframeLoad, + data, + roomId, + // Add other required fields from IWidget if necessary + sender: creatorUserId, // Example: Assuming sender is the creator + content: { + // Example content structure + type, + url: url.toString(), + name, + data, + creatorUserId, + }, +}); diff --git a/src/app/features/call/SmallWidgetDriver.ts b/src/app/features/call/SmallWidgetDriver.ts new file mode 100644 index 00000000..6764bb40 --- /dev/null +++ b/src/app/features/call/SmallWidgetDriver.ts @@ -0,0 +1,551 @@ +/* eslint-disable no-return-await */ +/* eslint-disable no-param-reassign */ +/* eslint-disable no-continue */ +/* eslint-disable class-methods-use-this */ +/* eslint-disable no-dupe-class-members */ +/* + * Copyright 2024 New Vector Ltd. + * Copyright 2020-2023 The Matrix.org Foundation C.I.C. + * + * SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial + * Please see LICENSE files in the repository root for full details. + */ +import { + type Capability, + EventDirection, + type ISendDelayedEventDetails, + type ISendEventDetails, + type IReadEventRelationsResult, + type IRoomEvent, + MatrixCapabilities, + type Widget, + WidgetDriver, + WidgetEventCapability, + WidgetKind, + type IWidgetApiErrorResponseDataDetails, + type ISearchUserDirectoryResult, + type IGetMediaConfigResult, + type UpdateDelayedEventAction, + OpenIDRequestState, + SimpleObservable, + IOpenIDUpdate, +} from 'matrix-widget-api'; +import { + EventType, + type IContent, + MatrixError, + type MatrixEvent, + Direction, + type SendDelayedEventResponse, + type StateEvents, + type TimelineEvents, + MatrixClient, +} from 'matrix-js-sdk'; + +export class SmallWidgetDriver extends WidgetDriver { + private allowedCapabilities: Set; + + private readonly mxClient: MatrixClient; // Store the client instance + + public constructor( + mx: MatrixClient, + allowedCapabilities: Capability[], + private forWidget: Widget, + private forWidgetKind: WidgetKind, + virtual: boolean, // Assuming 'virtual' might be needed later, kept for consistency + private inRoomId?: string + ) { + super(); + this.mxClient = mx; // Store the passed instance + + this.allowedCapabilities = new Set([ + ...allowedCapabilities, + MatrixCapabilities.Screenshots, + // Add other base capabilities as needed, e.g., ElementWidgetCapabilities.RequiresClient + ]); + + // --- Capabilities specific to Element Call (or similar trusted widgets) --- + // This is a trusted Element Call widget that we control (adjust if not Element Call) + this.allowedCapabilities.add(MatrixCapabilities.AlwaysOnScreen); + this.allowedCapabilities.add(MatrixCapabilities.MSC3846TurnServers); + this.allowedCapabilities.add(MatrixCapabilities.MSC4157SendDelayedEvent); + this.allowedCapabilities.add(MatrixCapabilities.MSC4157UpdateDelayedEvent); + // Capability to access the room timeline (MSC2762) + this.allowedCapabilities.add(`org.matrix.msc2762.timeline:${inRoomId}`); + // Capability to read room state (MSC2762) + this.allowedCapabilities.add(`org.matrix.msc2762.state:${inRoomId}`); + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent(EventDirection.Receive, EventType.RoomMember).raw + ); + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent(EventDirection.Receive, 'org.matrix.msc3401.call').raw + ); + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent(EventDirection.Receive, EventType.RoomEncryption).raw + ); + const clientUserId = this.mxClient.getSafeUserId(); + // For the legacy membership type + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent( + EventDirection.Send, + 'org.matrix.msc3401.call.member', + clientUserId + ).raw + ); + const clientDeviceId = this.mxClient.getDeviceId(); + if (clientDeviceId !== null) { + // For the session membership type compliant with MSC4143 + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent( + EventDirection.Send, + 'org.matrix.msc3401.call.member', + `_${clientUserId}_${clientDeviceId}` + ).raw + ); + // Version with no leading underscore, for room versions whose auth rules allow it + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent( + EventDirection.Send, + 'org.matrix.msc3401.call.member', + `${clientUserId}_${clientDeviceId}` + ).raw + ); + } + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent(EventDirection.Receive, 'org.matrix.msc3401.call.member') + .raw + ); + // for determining auth rules specific to the room version + this.allowedCapabilities.add( + WidgetEventCapability.forStateEvent(EventDirection.Receive, EventType.RoomCreate).raw + ); + + const sendRecvRoomEvents = [ + 'io.element.call.encryption_keys', + 'org.matrix.rageshake_request', + EventType.Reaction, + EventType.RoomRedaction, + 'io.element.call.reaction', + ]; + // eslint-disable-next-line no-restricted-syntax + for (const eventType of sendRecvRoomEvents) { + this.allowedCapabilities.add( + WidgetEventCapability.forRoomEvent(EventDirection.Send, eventType).raw + ); + this.allowedCapabilities.add( + WidgetEventCapability.forRoomEvent(EventDirection.Receive, eventType).raw + ); + } + + const sendRecvToDevice = [ + EventType.CallInvite, + EventType.CallCandidates, + EventType.CallAnswer, + EventType.CallHangup, + EventType.CallReject, + EventType.CallSelectAnswer, + EventType.CallNegotiate, + EventType.CallSDPStreamMetadataChanged, + EventType.CallSDPStreamMetadataChangedPrefix, + EventType.CallReplaces, + EventType.CallEncryptionKeysPrefix, + ]; + // eslint-disable-next-line no-restricted-syntax + for (const eventType of sendRecvToDevice) { + this.allowedCapabilities.add( + WidgetEventCapability.forToDeviceEvent(EventDirection.Send, eventType).raw + ); + this.allowedCapabilities.add( + WidgetEventCapability.forToDeviceEvent(EventDirection.Receive, eventType).raw + ); + } + } + + public async validateCapabilities(requested: Set): Promise> { + // Stubbed under the assumption voice calls will be valid thru element-call + return requested; + } + + public async sendEvent( + eventType: K, + content: StateEvents[K], + stateKey: string | null, + targetRoomId: string | null + ): Promise; + + public async sendEvent( + eventType: K, + content: TimelineEvents[K], + stateKey: null, + targetRoomId: string | null + ): Promise; + + public async sendEvent( + eventType: string, + content: IContent, + stateKey: string | null = null, + targetRoomId: string | null = null + ): Promise { + const client = this.mxClient; + const roomId = targetRoomId || this.inRoomId; + + if (!client || !roomId) throw new Error('Not in a room or not attached to a client'); + + let r: { event_id: string } | null; + if (stateKey !== null) { + // state event + r = await client.sendStateEvent( + roomId, + eventType as keyof StateEvents, + content as StateEvents[keyof StateEvents], + stateKey + ); + } else if (eventType === EventType.RoomRedaction) { + // special case: extract the `redacts` property and call redact + r = await client.redactEvent(roomId, content.redacts); + } else { + // message event + r = await client.sendEvent( + roomId, + eventType as keyof TimelineEvents, + content as TimelineEvents[keyof TimelineEvents] + ); + } + + return { roomId, eventId: r.event_id }; + } + + /** + * @experimental Part of MSC4140 & MSC4157 + * @see {@link WidgetDriver#sendDelayedEvent} + */ + public async sendDelayedEvent( + delay: number | null, + parentDelayId: string | null, + eventType: K, + content: StateEvents[K], + stateKey: string | null, + targetRoomId: string | null + ): Promise; + + /** + * @experimental Part of MSC4140 & MSC4157 + */ + public async sendDelayedEvent( + delay: number | null, + parentDelayId: string | null, + eventType: K, + content: TimelineEvents[K], + stateKey: null, + targetRoomId: string | null + ): Promise; + + public async sendDelayedEvent( + delay: number | null, + parentDelayId: string | null, + eventType: string, + content: IContent, + stateKey: string | null = null, + targetRoomId: string | null = null + ): Promise { + const client = this.mxClient; + const roomId = targetRoomId || this.inRoomId; + + if (!client || !roomId) throw new Error('Not in a room or not attached to a client'); + + let delayOpts; + if (delay !== null) { + delayOpts = { + delay, + ...(parentDelayId !== null && { parent_delay_id: parentDelayId }), + }; + } else if (parentDelayId !== null) { + delayOpts = { + parent_delay_id: parentDelayId, + }; + } else { + throw new Error('Must provide at least one of delay or parentDelayId'); + } + + let r: SendDelayedEventResponse | null; + if (stateKey !== null) { + // state event + r = await client._unstable_sendDelayedStateEvent( + roomId, + delayOpts, + eventType as keyof StateEvents, + content as StateEvents[keyof StateEvents], + stateKey + ); + } else { + // message event + r = await client._unstable_sendDelayedEvent( + roomId, + delayOpts, + null, + eventType as keyof TimelineEvents, + content as TimelineEvents[keyof TimelineEvents] + ); + } + + return { + roomId, + delayId: r.delay_id, + }; + } + + /** + * @experimental Part of MSC4140 & MSC4157 + */ + public async updateDelayedEvent( + delayId: string, + action: UpdateDelayedEventAction + ): Promise { + const client = this.mxClient; + + if (!client) throw new Error('Not in a room or not attached to a client'); + + await client._unstable_updateDelayedEvent(delayId, action); + } + + /** + * Implements {@link WidgetDriver#sendToDevice} + */ + public async sendToDevice( + eventType: string, + encrypted: boolean, + contentMap: { [userId: string]: { [deviceId: string]: object } } + ): Promise { + const client = this.mxClient; + + if (encrypted) { + const crypto = client.getCrypto(); + if (!crypto) throw new Error('E2EE not enabled'); + + // attempt to re-batch these up into a single request + const invertedContentMap: { [content: string]: { userId: string; deviceId: string }[] } = {}; + + // eslint-disable-next-line no-restricted-syntax + for (const userId of Object.keys(contentMap)) { + const userContentMap = contentMap[userId]; + // eslint-disable-next-line no-restricted-syntax + for (const deviceId of Object.keys(userContentMap)) { + const content = userContentMap[deviceId]; + const stringifiedContent = JSON.stringify(content); + invertedContentMap[stringifiedContent] = invertedContentMap[stringifiedContent] || []; + invertedContentMap[stringifiedContent].push({ userId, deviceId }); + } + } + + await Promise.all( + Object.entries(invertedContentMap).map(async ([stringifiedContent, recipients]) => { + const batch = await crypto.encryptToDeviceMessages( + eventType, + recipients, + JSON.parse(stringifiedContent) + ); + + await client.queueToDevice(batch); + }) + ); + } else { + await client.queueToDevice({ + eventType, + batch: Object.entries(contentMap).flatMap(([userId, userContentMap]) => + Object.entries(userContentMap).map(([deviceId, content]) => ({ + userId, + deviceId, + payload: content, + })) + ), + }); + } + } + + /** + * Reads all events of the given type, and optionally `msgtype` (if applicable/defined), + * the user has access to. The widget API will have already verified that the widget is + * capable of receiving the events. Less events than the limit are allowed to be returned, + * but not more. + * @param roomId The ID of the room to look within. + * @param eventType The event type to be read. + * @param msgtype The msgtype of the events to be read, if applicable/defined. + * @param stateKey The state key of the events to be read, if applicable/defined. + * @param limit The maximum number of events to retrieve. Will be zero to denote "as many as + * possible". + * @param since When null, retrieves the number of events specified by the "limit" parameter. + * Otherwise, the event ID at which only subsequent events will be returned, as many as specified + * in "limit". + * @returns {Promise} Resolves to the room events, or an empty array. + */ + public async readRoomTimeline( + roomId: string, + eventType: string, + msgtype: string | undefined, + stateKey: string | undefined, + limit: number, + since: string | undefined + ): Promise { + limit = limit > 0 ? Math.min(limit, Number.MAX_SAFE_INTEGER) : Number.MAX_SAFE_INTEGER; // relatively arbitrary + + const room = this.mxClient.getRoom(roomId); + if (room === null) return []; + const results: MatrixEvent[] = []; + const events = room.getLiveTimeline().getEvents(); // timelines are most recent last + for (let i = events.length - 1; i >= 0; i--) { + const ev = events[i]; + if (results.length >= limit) break; + if (since !== undefined && ev.getId() === since) break; + + if (ev.getType() !== eventType || ev.isState()) continue; + if (eventType === EventType.RoomMessage && msgtype && msgtype !== ev.getContent().msgtype) + continue; + if (ev.getStateKey() !== undefined && stateKey !== undefined && ev.getStateKey() !== stateKey) + continue; + results.push(ev); + } + + return results.map((e) => e.getEffectiveEvent() as IRoomEvent); + } + + public async askOpenID(observer: SimpleObservable): Promise { + return observer.update({ + state: OpenIDRequestState.Allowed, + token: await this.mxClient.getOpenIdToken(), + }); + } + + /** + * Reads the current values of all matching room state entries. + * @param roomId The ID of the room. + * @param eventType The event type of the entries to be read. + * @param stateKey The state key of the entry to be read. If undefined, + * all room state entries with a matching event type should be returned. + * @returns {Promise} Resolves to the events representing the + * current values of the room state entries. + */ + public async readRoomState( + roomId: string, + eventType: string, + stateKey: string | undefined + ): Promise { + const room = this.mxClient.getRoom(roomId); + if (room === null) return []; + const state = room.getLiveTimeline().getState(Direction.Forward); + if (state === undefined) return []; + + if (stateKey === undefined) + return state.getStateEvents(eventType).map((e) => e.getEffectiveEvent() as IRoomEvent); + const event = state.getStateEvents(eventType, stateKey); + return event === null ? [] : [event.getEffectiveEvent() as IRoomEvent]; + } + + /* + public async navigate(uri: string): Promise { + navigateToPermalink(uri); + } + */ + + public async readEventRelations( + eventId: string, + roomId?: string, + relationType?: string, + eventType?: string, + from?: string, + to?: string, + limit?: number, + direction?: 'f' | 'b' + ): Promise { + const client = this.mxClient; + const dir = direction as Direction; + roomId = roomId ?? this.inRoomId ?? undefined; + + if (typeof roomId !== 'string') { + throw new Error('Error while reading the current room'); + } + + const { events, nextBatch, prevBatch } = await client.relations( + roomId, + eventId, + relationType ?? null, + eventType ?? null, + { from, to, limit, dir } + ); + + return { + chunk: events.map((e) => e.getEffectiveEvent() as IRoomEvent), + nextBatch: nextBatch ?? undefined, + prevBatch: prevBatch ?? undefined, + }; + } + + public async searchUserDirectory( + searchTerm: string, + limit?: number + ): Promise { + const client = this.mxClient; + + const { limited, results } = await client.searchUserDirectory({ term: searchTerm, limit }); + + return { + limited, + results: results.map((r) => ({ + userId: r.user_id, + displayName: r.display_name, + avatarUrl: r.avatar_url, + })), + }; + } + + public async getMediaConfig(): Promise { + const client = this.mxClient; + + return await client.getMediaConfig(); + } + + public async uploadFile(file: XMLHttpRequestBodyInit): Promise<{ contentUri: string }> { + const client = this.mxClient; + + const uploadResult = await client.uploadContent(file); + + return { contentUri: uploadResult.content_uri }; + } + + /** + * Download a file from the media repository on the homeserver. + * + * @param contentUri - the MXC URI of the file to download + * @returns an object with: file - response contents as Blob + */ + /* + public async downloadFile(contentUri: string): Promise<{ file: XMLHttpRequestBodyInit }> { + const client = this.mxClient; + const media = mediaFromMxc(contentUri, client); + const response = await media.downloadSource(); + const blob = await response.blob(); + return { file: blob }; + } + */ + + /** + * Gets the IDs of all joined or invited rooms currently known to the + * client. + * @returns The room IDs. + */ + public getKnownRooms(): string[] { + return this.mxClient.getVisibleRooms().map((r) => r.roomId); + } + + /** + * Expresses a {@link MatrixError} as a JSON payload + * for use by Widget API error responses. + * @param error The error to handle. + * @returns The error expressed as a JSON payload, + * or undefined if it is not a {@link MatrixError}. + */ + public processError(error: unknown): IWidgetApiErrorResponseDataDetails | undefined { + return error instanceof MatrixError + ? { matrix_api_error: error.asWidgetApiErrorData() } + : undefined; + } +} diff --git a/src/app/features/room-nav/RoomCallNavStatus.tsx b/src/app/features/room-nav/RoomCallNavStatus.tsx new file mode 100644 index 00000000..d3f23794 --- /dev/null +++ b/src/app/features/room-nav/RoomCallNavStatus.tsx @@ -0,0 +1,115 @@ +import { Box, Chip, Icon, IconButton, Icons, Text, Tooltip, TooltipProvider } from 'folds'; +import React from 'react'; +import { useMatrixClient } from '../../hooks/useMatrixClient'; +import { useCallState } from '../../pages/client/call/CallProvider'; +import { useRoomNavigate } from '../../hooks/useRoomNavigate'; + +export function CallNavStatus() { + const { + activeCallRoomId, + isAudioEnabled, + isVideoEnabled, + isCallActive, + toggleAudio, + toggleVideo, + hangUp, + } = useCallState(); + const mx = useMatrixClient(); + const { navigateRoom } = useRoomNavigate(); + const handleGoToCallRoom = () => { + if (activeCallRoomId) { + navigateRoom(activeCallRoomId); + } + }; + if (!isCallActive) { + return null; + } + + return ( + + + {/* Going to need better icons for this */} + + {!isAudioEnabled ? 'Unmute' : 'Mute'} + + } + > + {(triggerRef) => ( + + + + )} + + + {!isVideoEnabled ? 'Video On' : 'Video Off'} + + } + > + {(triggerRef) => ( + + + + )} + + + + Hang Up + + } + > + {(triggerRef) => ( + + + + )} + + + + + Go to Room + + } + > + {(triggerRef) => ( + + + {mx.getRoom(activeCallRoomId)?.name} + + + )} + + + + + ); +} diff --git a/src/app/features/room-nav/RoomNavItem.tsx b/src/app/features/room-nav/RoomNavItem.tsx index bdb81418..b666c48b 100644 --- a/src/app/features/room-nav/RoomNavItem.tsx +++ b/src/app/features/room-nav/RoomNavItem.tsx @@ -1,4 +1,4 @@ -import React, { MouseEventHandler, forwardRef, useState } from 'react'; +import React, { MouseEventHandler, forwardRef, useState, MouseEvent } from 'react'; import { Room } from 'matrix-js-sdk'; import { Avatar, @@ -16,10 +16,13 @@ import { RectCords, Badge, Spinner, + Tooltip, + TooltipProvider, } from 'folds'; import { useFocusWithin, useHover } from 'react-aria'; import FocusTrap from 'focus-trap-react'; -import { NavItem, NavItemContent, NavItemOptions, NavLink } from '../../components/nav'; +import { useParams } from 'react-router-dom'; +import { NavButton, NavItem, NavItemContent, NavItemOptions, NavLink } from '../../components/nav'; import { UnreadBadge, UnreadBadgeCenter } from '../../components/unread-badge'; import { RoomAvatar, RoomIcon } from '../../components/room-avatar'; import { getDirectRoomAvatarUrl, getRoomAvatarUrl } from '../../utils/room'; @@ -49,6 +52,11 @@ import { RoomNotificationMode, } from '../../hooks/useRoomsNotificationPreferences'; import { RoomNotificationModeSwitcher } from '../../components/RoomNotificationSwitcher'; +import { useCallState } from '../../pages/client/call/CallProvider'; +import { useCallMembers } from '../../hooks/useCallMemberships'; +import { useRoomNavigate } from '../../hooks/useRoomNavigate'; +import { ScreenSize, useScreenSizeContext } from '../../hooks/useScreenSize'; +import { RoomNavUser } from './RoomNavUser'; type RoomNavItemMenuProps = { room: Room; @@ -193,6 +201,7 @@ const RoomNavItemMenu = forwardRef( ); } ); +RoomNavItemMenu.displayName = 'RoomNavItemMenu'; type RoomNavItemProps = { room: Room; @@ -217,9 +226,24 @@ export function RoomNavItem({ const { focusWithinProps } = useFocusWithin({ onFocusWithinChange: setHover }); const [menuAnchor, setMenuAnchor] = useState(); const unread = useRoomUnread(room.roomId, roomToUnreadAtom); + const { + isCallActive, + activeCallRoomId, + setActiveCallRoomId, + setViewedCallRoomId, + isChatOpen, + toggleChat, + hangUp, + } = useCallState(); const typingMember = useRoomTypingMember(room.roomId).filter( (receipt) => receipt.userId !== mx.getUserId() ); + const isActiveCall = isCallActive && activeCallRoomId === room.roomId; + const callMemberships = useCallMembers(mx, room.roomId); + const { navigateRoom } = useRoomNavigate(); + const { roomIdOrAlias: viewedRoomId } = useParams(); + const screenSize = useScreenSizeContext(); + const isMobile = screenSize === ScreenSize.Mobile; const handleContextMenu: MouseEventHandler = (evt) => { evt.preventDefault(); @@ -235,109 +259,213 @@ export function RoomNavItem({ setMenuAnchor(evt.currentTarget.getBoundingClientRect()); }; + const handleNavItemClick: MouseEventHandler = (evt) => { + const target = evt.target as HTMLElement; + const chatButton = (evt.currentTarget as HTMLElement).querySelector( + '[data-testid="chat-button"]' + ); + if (chatButton && chatButton.contains(target)) { + return; + } + if (room.isCallRoom()) { + if (!isMobile) { + if (activeCallRoomId !== room.roomId) { + if (mx.getRoom(viewedRoomId)?.isCallRoom()) { + navigateRoom(room.roomId); + } + hangUp(room.roomId); + setActiveCallRoomId(room.roomId); + } else { + navigateRoom(room.roomId); + } + } else { + evt.stopPropagation(); + if (isChatOpen) toggleChat(); + setViewedCallRoomId(room.roomId); + navigateRoom(room.roomId); + } + } else { + navigateRoom(room.roomId); + } + }; + + const handleChatButtonClick = (evt: MouseEvent) => { + evt.stopPropagation(); + if (!isChatOpen) toggleChat(); + setViewedCallRoomId(room.roomId); + }; + const optionsVisible = hover || !!menuAnchor; + const ariaLabel = [ + room.name, + room.isCallRoom() + ? [ + 'Call Room', + isActiveCall && 'Currently in Call', + callMemberships.length && `${callMemberships.length} in Call`, + ] + : 'Text Room', + unread?.total && `${unread.total} Messages`, + ] + .flat() + .filter(Boolean) + .join(', '); return ( - - - - - - {showAvatar ? ( - ( - - {nameInitials(room.name)} - - )} - /> - ) : ( - + + + + + + {showAvatar ? ( + ( + + {nameInitials(room.name)} + + )} + /> + ) : ( + + )} + + + + {room.name} + + + {!optionsVisible && !unread && !selected && typingMember.length > 0 && ( + + + + )} + {!optionsVisible && unread && ( + + 0} count={unread.total} /> + + )} + {!optionsVisible && notificationMode !== RoomNotificationMode.Unset && ( + )} - - - - {room.name} - - {!optionsVisible && !unread && !selected && typingMember.length > 0 && ( - - - - )} - {!optionsVisible && unread && ( - - 0} count={unread.total} /> - - )} - {!optionsVisible && notificationMode !== RoomNotificationMode.Unset && ( - - )} - - - - {optionsVisible && ( - - setMenuAnchor(undefined), - clickOutsideDeactivates: true, - isKeyForward: (evt: KeyboardEvent) => evt.key === 'ArrowDown', - isKeyBackward: (evt: KeyboardEvent) => evt.key === 'ArrowUp', - escapeDeactivates: stopPropagation, - }} + + {optionsVisible && ( + + setMenuAnchor(undefined), + clickOutsideDeactivates: true, + isKeyForward: (evt: KeyboardEvent) => evt.key === 'ArrowDown', + isKeyBackward: (evt: KeyboardEvent) => evt.key === 'ArrowUp', + escapeDeactivates: stopPropagation, + }} + > + setMenuAnchor(undefined)} + notificationMode={notificationMode} + /> + + } > - setMenuAnchor(undefined)} - notificationMode={notificationMode} - /> - - } - > - - - - - + {room.isCallRoom() && ( + + Open Chat + + } + > + {(triggerRef) => ( + + + + + + )} + + )} + + + + + + )} + + + {room.isCallRoom() && ( + + {callMemberships.map((callMembership) => ( + + ))} + )} - + ); } diff --git a/src/app/features/room-nav/RoomNavUser.tsx b/src/app/features/room-nav/RoomNavUser.tsx new file mode 100644 index 00000000..55a95c15 --- /dev/null +++ b/src/app/features/room-nav/RoomNavUser.tsx @@ -0,0 +1,165 @@ +import { + Avatar, + Box, + config, + Icon, + IconButton, + Icons, + Text, + Tooltip, + TooltipProvider, +} from 'folds'; +import React, { useState } from 'react'; +import { Room } from 'matrix-js-sdk'; +import { useFocusWithin, useHover } from 'react-aria'; +import { CallMembership } from 'matrix-js-sdk/lib/matrixrtc/CallMembership'; +import { NavItem, NavItemContent, NavItemOptions } from '../../components/nav'; +import { UserAvatar } from '../../components/user-avatar'; +import { useMatrixClient } from '../../hooks/useMatrixClient'; +import { useCallState } from '../../pages/client/call/CallProvider'; +import { getMxIdLocalPart } from '../../utils/matrix'; +import { getMemberAvatarMxc, getMemberDisplayName } from '../../utils/room'; +import { useMediaAuthentication } from '../../hooks/useMediaAuthentication'; +import { openProfileViewer } from '../../../client/action/navigation'; + +type RoomNavUserProps = { + room: Room; + callMembership: CallMembership; +}; +export function RoomNavUser({ room, callMembership }: RoomNavUserProps) { + const mx = useMatrixClient(); + const useAuthentication = useMediaAuthentication(); + const [navUserExpanded, setNavUserExpanded] = useState(false); + const [hover, setHover] = useState(false); + const { hoverProps } = useHover({ onHoverChange: setHover }); + const { focusWithinProps } = useFocusWithin({ + onFocusWithinChange: (isFocused) => { + setHover(isFocused); + if (!isFocused) setNavUserExpanded(false); + }, + }); + const { isCallActive, activeCallRoomId } = useCallState(); + const isActiveCall = isCallActive && activeCallRoomId === room.roomId; + const userId = callMembership.sender ?? ''; + const avatarMxcUrl = getMemberAvatarMxc(room, userId); + const avatarUrl = avatarMxcUrl + ? mx.mxcUrlToHttp(avatarMxcUrl, 32, 32, 'crop', undefined, false, useAuthentication) + : undefined; + const getName = getMemberDisplayName(room, userId) ?? getMxIdLocalPart(userId); + const isCallParticipant = isActiveCall && userId !== mx.getUserId(); + + const handleNavUserClick = () => { + if (isCallParticipant) { + setNavUserExpanded((prev) => !prev); + } + }; + + const handleClickUser = () => { + openProfileViewer(userId, room.roomId); + }; + + // PLACEHOLDER + const [userMuted, setUserMuted] = useState(false); + const handleToggleMute = () => { + setUserMuted(!userMuted); + }; + + const optionsVisible = (hover || userMuted || navUserExpanded) && isCallParticipant && false; // Disable until individual volume control and mute have been added + const ariaLabel = isCallParticipant + ? `Call Participant: ${getName}${userMuted ? ', Muted' : ''}` + : getName; + + return ( + + + + + + } + /> + + + {getName} + + + {navUserExpanded && ( + + {/* Slider here, when implemented into folds */} + ---- THIS IS A SLIDER --- + + )} + + + {optionsVisible && ( + + + {userMuted ? 'Unmute' : 'Mute'} + + } + > + {(triggerRef) => ( + + + + )} + + {navUserExpanded && ( + + View Profile + + } + > + {(triggerRef) => ( + + + + )} + + )} + + )} + + ); +} diff --git a/src/app/features/room/Room.tsx b/src/app/features/room/Room.tsx index ffc21857..24f378b4 100644 --- a/src/app/features/room/Room.tsx +++ b/src/app/features/room/Room.tsx @@ -1,3 +1,4 @@ +/* eslint-disable no-nested-ternary */ import React, { useCallback } from 'react'; import { Box, Line } from 'folds'; import { useParams } from 'react-router-dom'; @@ -13,6 +14,9 @@ import { useKeyDown } from '../../hooks/useKeyDown'; import { markAsRead } from '../../../client/action/notifications'; import { useMatrixClient } from '../../hooks/useMatrixClient'; import { useRoomMembers } from '../../hooks/useRoomMembers'; +import { CallView } from '../call/CallView'; +import { useCallState } from '../../pages/client/call/CallProvider'; +import { RoomViewHeader } from './RoomViewHeader'; export function Room() { const { eventId } = useParams(); @@ -21,9 +25,10 @@ export function Room() { const [isDrawer] = useSetting(settingsAtom, 'isPeopleDrawer'); const [hideActivity] = useSetting(settingsAtom, 'hideActivity'); + const { isChatOpen } = useCallState(); const screenSize = useScreenSizeContext(); const powerLevels = usePowerLevels(room); - const members = useRoomMembers(mx, room.roomId); + const members = useRoomMembers(mx, room?.roomId); useKeyDown( window, @@ -39,14 +44,48 @@ export function Room() { return ( - - - {screenSize === ScreenSize.Desktop && isDrawer && ( - <> - - - - )} + + {room.isCallRoom() && } + + + {(!room.isCallRoom() || isChatOpen) && ( + + + + + + )} + {screenSize === ScreenSize.Desktop && !room.isCallRoom() && isDrawer && ( + <> + + + + )} + ); diff --git a/src/app/features/room/RoomView.tsx b/src/app/features/room/RoomView.tsx index b6eebdf2..25d8fbd8 100644 --- a/src/app/features/room/RoomView.tsx +++ b/src/app/features/room/RoomView.tsx @@ -1,5 +1,5 @@ import React, { useCallback, useRef } from 'react'; -import { Box, Text, config } from 'folds'; +import { Box, Text, config } from 'folds'; // Assuming 'folds' is a UI library import { EventType, Room } from 'matrix-js-sdk'; import { ReactEditor } from 'slate-react'; import { isKeyHotkey } from 'is-hotkey'; @@ -25,16 +25,20 @@ import { useAccessibleTagColors, usePowerLevelTags } from '../../hooks/usePowerL import { useTheme } from '../../hooks/useTheme'; const FN_KEYS_REGEX = /^F\d+$/; + +/** + * Determines if a keyboard event should trigger focusing the message input field. + * @param evt - The KeyboardEvent. + * @returns True if the input should be focused, false otherwise. + */ const shouldFocusMessageField = (evt: KeyboardEvent): boolean => { const { code } = evt; if (evt.metaKey || evt.altKey || evt.ctrlKey) { return false; } - // do not focus on F keys if (FN_KEYS_REGEX.test(code)) return false; - // do not focus on numlock/scroll lock if ( code.startsWith('OS') || code.startsWith('Meta') || @@ -53,21 +57,16 @@ const shouldFocusMessageField = (evt: KeyboardEvent): boolean => { ) { return false; } - return true; }; export function RoomView({ room, eventId }: { room: Room; eventId?: string }) { const roomInputRef = useRef(null); const roomViewRef = useRef(null); - const [hideActivity] = useSetting(settingsAtom, 'hideActivity'); - const { roomId } = room; const editor = useEditor(); - const mx = useMatrixClient(); - const tombstoneEvent = useStateEvent(room, StateEvent.RoomTombstone); const powerLevels = usePowerLevelsContext(); const { getPowerLevel, canSendEvent } = usePowerLevelsAPI(powerLevels); @@ -75,7 +74,6 @@ export function RoomView({ room, eventId }: { room: Room; eventId?: string }) { const canMessage = myUserId ? canSendEvent(EventType.RoomMessage, getPowerLevel(myUserId)) : false; - const [powerLevelTags, getPowerLevelTag] = usePowerLevelTags(room, powerLevels); const theme = useTheme(); const accessibleTagColors = useAccessibleTagColors(theme.kind, powerLevelTags); @@ -85,14 +83,14 @@ export function RoomView({ room, eventId }: { room: Room; eventId?: string }) { useCallback( (evt) => { if (editableActiveElement()) return; - if ( - document.body.lastElementChild?.className !== 'ReactModalPortal' || - navigation.isRawModalVisible - ) { + if (document.querySelector('.ReactModalPortal > *') || navigation.isRawModalVisible) { return; } + if (shouldFocusMessageField(evt) || isKeyHotkey('mod+v', evt)) { - ReactEditor.focus(editor); + if (editor) { + ReactEditor.focus(editor); + } } }, [editor] @@ -101,8 +99,8 @@ export function RoomView({ room, eventId }: { room: Room; eventId?: string }) { return ( - - + {!room.isCallRoom() && } +
+ {' '} {tombstoneEvent ? ( ) : ( <> - {canMessage && ( + {canMessage ? ( - )} - {!canMessage && ( + ) : ( (); const mDirects = useAtomValue(mDirectAtom); + const { isChatOpen, toggleChat } = useCallState(); const pinnedEvents = useRoomPinnedEvents(room); const encryptionEvent = useStateEvent(room, StateEvent.RoomEncryption); const ecryptedRoom = !!encryptionEvent; @@ -232,6 +234,24 @@ export function RoomViewHeader() { const setPeopleDrawer = useSetSetting(settingsAtom, 'isPeopleDrawer'); + // I assume there is a global state so I don't have to run this check every time but for now we'll stub this in + const isDirectMessage = () => { + const mDirectsEvent = mx.getAccountData('m.direct'); + if (mDirectsEvent?.event?.content === undefined) { + return false; + } + const { roomId } = room; + return ( + Object.values(mDirectsEvent?.event?.content).filter((e) => { + if (e.indexOf(roomId) === 0) return true; + }).length !== 0 + ); + }; + + const handleCall: MouseEventHandler = (evt) => { + setMenuAnchor(evt.currentTarget.getBoundingClientRect()); + }; + const handleSearchClick = () => { const searchParams: _SearchPathSearchParams = { rooms: room.roomId, @@ -324,8 +344,28 @@ export function RoomViewHeader() { )} + - {!ecryptedRoom && ( + {false && isDirectMessage() && ( + + Start a Call + + } + > + {(triggerRef) => ( + + + + )} + + )} + + {!ecryptedRoom && (!room.isCallRoom() || isChatOpen) && ( )} - - Pinned Messages - - } - > - {(triggerRef) => ( - - {pinnedEvents.length > 0 && ( - - - {pinnedEvents.length} - - - )} - - - )} - - setPinMenuAnchor(undefined), - clickOutsideDeactivates: true, - isKeyForward: (evt: KeyboardEvent) => evt.key === 'ArrowDown', - isKeyBackward: (evt: KeyboardEvent) => evt.key === 'ArrowUp', - escapeDeactivates: stopPropagation, - }} - > - setPinMenuAnchor(undefined)} /> - - } - /> - {screenSize === ScreenSize.Desktop && ( + {(!room.isCallRoom() || isChatOpen) && ( + + Pinned Messages + + } + > + {(triggerRef) => ( + + {pinnedEvents.length > 0 && ( + + + {pinnedEvents.length} + + + )} + + + )} + + )} + {(!room.isCallRoom() || isChatOpen) && ( + setPinMenuAnchor(undefined), + clickOutsideDeactivates: true, + isKeyForward: (evt: KeyboardEvent) => evt.key === 'ArrowDown', + isKeyBackward: (evt: KeyboardEvent) => evt.key === 'ArrowUp', + escapeDeactivates: stopPropagation, + }} + > + setPinMenuAnchor(undefined)} /> + + } + /> + )} + + {!room.isCallRoom() && screenSize === ScreenSize.Desktop && ( )} + + {room.isCallRoom() && !isDirectMessage() && ( + + Chat + + } + > + {(triggerRef) => ( + + + + )} + + )} + { + const [memberships, setMemberships] = useState([]); + const room = mx.getRoom(roomId); + const mxr = mx.matrixRTC.getRoomSession(room); + useEffect(() => { + const updateMemberships = () => { + if (!room?.isCallRoom()) return; + setMemberships(MatrixRTCSession.callMembershipsForRoom(room)); + }; + + updateMemberships(); + + mxr.on(MatrixRTCSessionEvent.MembershipsChanged, updateMemberships); + return () => { + mxr.removeListener(MatrixRTCSessionEvent.MembershipsChanged, updateMemberships); + }; + }, [mx, mxr, room, roomId]); + + return memberships; +}; diff --git a/src/app/hooks/useClientConfig.ts b/src/app/hooks/useClientConfig.ts index e5fc6cc6..70ba26bb 100644 --- a/src/app/hooks/useClientConfig.ts +++ b/src/app/hooks/useClientConfig.ts @@ -9,6 +9,7 @@ export type ClientConfig = { defaultHomeserver?: number; homeserverList?: string[]; allowCustomHomeservers?: boolean; + elementCallUrl?: string; featuredCommunities?: { openAsDefault?: boolean; diff --git a/src/app/hooks/useStateEvents.ts b/src/app/hooks/useStateEvents.ts index dd085693..6d1c80bc 100644 --- a/src/app/hooks/useStateEvents.ts +++ b/src/app/hooks/useStateEvents.ts @@ -1,28 +1,35 @@ import { useCallback, useMemo } from 'react'; -import { Room } from 'matrix-js-sdk'; +import { MatrixEvent, Room } from 'matrix-js-sdk'; import { StateEvent } from '../../types/matrix/room'; +import { useMatrixClient } from './useMatrixClient'; import { useForceUpdate } from './useForceUpdate'; import { useStateEventCallback } from './useStateEventCallback'; -import { getStateEvents } from '../utils/room'; -export const useStateEvents = (room: Room, eventType: StateEvent) => { +export const useStateEvents = (rooms: Room[], eventType: StateEvent): number => { + const mx = useMatrixClient(); + const [updateCount, forceUpdate] = useForceUpdate(); - useStateEventCallback( - room.client, - useCallback( - (event) => { - if (event.getRoomId() === room.roomId && event.getType() === eventType) { - forceUpdate(); + const relevantRoomIds = useMemo(() => { + const ids = new Set(); + if (rooms && Array.isArray(rooms)) { + rooms.forEach((room) => { + if (room?.roomId) { + ids.add(room.roomId); } - }, - [room, eventType, forceUpdate] - ) - ); - - return useMemo( - () => getStateEvents(room, eventType), - // eslint-disable-next-line react-hooks/exhaustive-deps - [room, eventType, updateCount] + }); + } + return ids; + }, [rooms]); + const handleEventCallback = useCallback( + (event: MatrixEvent) => { + const eventRoomId = event.getRoomId(); + if (eventRoomId && event.getType() === eventType && relevantRoomIds.has(eventRoomId)) { + forceUpdate(); + } + }, + [eventType, relevantRoomIds, forceUpdate] ); + useStateEventCallback(mx, handleEventCallback); + return updateCount; }; diff --git a/src/app/organisms/create-room/CreateRoom.jsx b/src/app/organisms/create-room/CreateRoom.jsx index 04b2faeb..ce204d08 100644 --- a/src/app/organisms/create-room/CreateRoom.jsx +++ b/src/app/organisms/create-room/CreateRoom.jsx @@ -43,6 +43,7 @@ function CreateRoomContent({ isSpace, parentId, onRequestClose }) { const [isValidAddress, setIsValidAddress] = useState(null); const [addressValue, setAddressValue] = useState(undefined); const [roleIndex, setRoleIndex] = useState(0); + const [roomType, setRoomType] = useState(0); const addressRef = useRef(null); @@ -75,6 +76,7 @@ function CreateRoomContent({ isSpace, parentId, onRequestClose }) { joinRule, alias: roomAlias, isEncrypted: isSpace || joinRule === 'public' ? false : isEncrypted, + roomType, powerLevel, isSpace, parentId, @@ -218,6 +220,19 @@ function CreateRoomContent({ isSpace, parentId, onRequestClose }) { Selecting Admin sets 100 power level whereas Founder sets 101. } /> + {!isSpace && ( + + } + content={Select the type of room.} + /> + )}
diff --git a/src/app/pages/Router.tsx b/src/app/pages/Router.tsx index 89743693..1af0648e 100644 --- a/src/app/pages/Router.tsx +++ b/src/app/pages/Router.tsx @@ -61,6 +61,8 @@ import { AutoRestoreBackupOnVerification } from '../components/BackupRestore'; import { RoomSettingsRenderer } from '../features/room-settings'; import { ClientRoomsNotificationPreferences } from './client/ClientRoomsNotificationPreferences'; import { SpaceSettingsRenderer } from '../features/space-settings'; +import { CallProvider } from './client/call/CallProvider'; +import { PersistentCallContainer } from './client/call/PersistentCallContainer'; export const createRouter = (clientConfig: ClientConfig, screenSize: ScreenSize) => { const { hashRouter } = clientConfig; @@ -116,15 +118,19 @@ export const createRouter = (clientConfig: ClientConfig, screenSize: ScreenSize) - - - - } - > - - + + + + + } + > + + + + + diff --git a/src/app/pages/client/call/CallProvider.tsx b/src/app/pages/client/call/CallProvider.tsx new file mode 100644 index 00000000..364670eb --- /dev/null +++ b/src/app/pages/client/call/CallProvider.tsx @@ -0,0 +1,536 @@ +import React, { + createContext, + useState, + useContext, + useMemo, + useCallback, + ReactNode, + useEffect, +} from 'react'; +import { logger } from 'matrix-js-sdk/lib/logger'; +import { WidgetApiToWidgetAction, WidgetApiAction, ClientWidgetApi } from 'matrix-widget-api'; +import { useParams } from 'react-router-dom'; +import { SmallWidget } from '../../../features/call/SmallWidget'; + +interface MediaStatePayload { + data?: { + audio_enabled?: boolean; + video_enabled?: boolean; + }; +} + +const WIDGET_MEDIA_STATE_UPDATE_ACTION = 'io.element.device_mute'; +const WIDGET_HANGUP_ACTION = 'im.vector.hangup'; +const WIDGET_ON_SCREEN_ACTION = 'set_always_on_screen'; +const WIDGET_JOIN_ACTION = 'io.element.join'; +const WIDGET_TILE_UPDATE = 'io.element.tile_layout'; + +interface CallContextState { + activeCallRoomId: string | null; + setActiveCallRoomId: (roomId: string | null) => void; + viewedCallRoomId: string | null; + setViewedCallRoomId: (roomId: string | null) => void; + hangUp: (room: string) => void; + activeClientWidgetApi: ClientWidgetApi | null; + activeClientWidget: SmallWidget | null; + registerActiveClientWidgetApi: ( + roomId: string | null, + clientWidgetApi: ClientWidgetApi | null, + clientWidget: SmallWidget + ) => void; + viewedClientWidgetApi: ClientWidgetApi | null; + viewedClientWidget: SmallWidget | null; + registerViewedClientWidgetApi: ( + roomId: string | null, + clientWidgetApi: ClientWidgetApi | null, + clientWidget: SmallWidget + ) => void; + sendWidgetAction: ( + action: WidgetApiToWidgetAction | string, + data: T + ) => Promise; + isAudioEnabled: boolean; + isVideoEnabled: boolean; + isChatOpen: boolean; + isCallActive: boolean; + isPrimaryIframe: boolean; + toggleAudio: () => Promise; + toggleVideo: () => Promise; + toggleChat: () => Promise; + toggleIframe: () => Promise; +} + +const CallContext = createContext(undefined); + +interface CallProviderProps { + children: ReactNode; +} + +const DEFAULT_AUDIO_ENABLED = true; +const DEFAULT_VIDEO_ENABLED = false; +const DEFAULT_CHAT_OPENED = false; +const DEFAULT_CALL_ACTIVE = false; +const DEFAULT_PRIMARY_IFRAME = true; + +export function CallProvider({ children }: CallProviderProps) { + const [activeCallRoomId, setActiveCallRoomIdState] = useState(null); + const [viewedCallRoomId, setViewedCallRoomIdState] = useState(null); + const [activeClientWidgetApi, setActiveClientWidgetApiState] = useState( + null + ); + const [activeClientWidget, setActiveClientWidget] = useState(null); + const [activeClientWidgetApiRoomId, setActiveClientWidgetApiRoomId] = useState( + null + ); + const [viewedClientWidgetApi, setViewedClientWidgetApiState] = useState( + null + ); + const [viewedClientWidget, setViewedClientWidget] = useState(null); + const [viewedClientWidgetApiRoomId, setViewedClientWidgetApiRoomId] = useState( + null + ); + + const [isAudioEnabled, setIsAudioEnabledState] = useState(DEFAULT_AUDIO_ENABLED); + const [isVideoEnabled, setIsVideoEnabledState] = useState(DEFAULT_VIDEO_ENABLED); + const [isChatOpen, setIsChatOpenState] = useState(DEFAULT_CHAT_OPENED); + const [isCallActive, setIsCallActive] = useState(DEFAULT_CALL_ACTIVE); + const [isPrimaryIframe, setIsPrimaryIframe] = useState(DEFAULT_PRIMARY_IFRAME); + + const { roomIdOrAlias: viewedRoomId } = useParams<{ roomIdOrAlias: string }>(); + const [lastViewedRoomDuringCall, setLastViewedRoomDuringCall] = useState(null); + + const resetMediaState = useCallback(() => { + logger.debug('CallContext: Resetting media state to defaults.'); + setIsAudioEnabledState(DEFAULT_AUDIO_ENABLED); + setIsVideoEnabledState(DEFAULT_VIDEO_ENABLED); + }, []); + + const setActiveCallRoomId = useCallback( + (roomId: string | null) => { + logger.warn(`CallContext: Setting activeCallRoomId to ${roomId}`); + const previousRoomId = activeCallRoomId; + setActiveCallRoomIdState(roomId); + + if (roomId !== previousRoomId) { + logger.debug(`CallContext: Active call room changed, resetting media state.`); + resetMediaState(); + } + + if (roomId === null || roomId !== activeClientWidgetApiRoomId) { + logger.warn( + `CallContext: Clearing active clientWidgetApi because active room changed to ${roomId} or was cleared.` + ); + } + }, + [activeClientWidgetApiRoomId, resetMediaState, activeCallRoomId] + ); + + const setViewedCallRoomId = useCallback( + (roomId: string | null) => { + logger.warn(`CallContext: Setting activeCallRoomId to ${roomId}`); + setViewedCallRoomIdState(roomId); + }, + [setViewedCallRoomIdState] + ); + + const setActiveClientWidgetApi = useCallback( + ( + clientWidgetApi: ClientWidgetApi | null, + clientWidget: SmallWidget | null, + roomId: string | null + ) => { + setActiveClientWidgetApiState(clientWidgetApi); + setActiveClientWidget(clientWidget); + setActiveClientWidgetApiRoomId(roomId); + }, + [] + ); + + const registerActiveClientWidgetApi = useCallback( + ( + roomId: string | null, + clientWidgetApi: ClientWidgetApi | null, + clientWidget: SmallWidget | null + ) => { + if (activeClientWidgetApi && activeClientWidgetApi !== clientWidgetApi) { + logger.debug(`CallContext: Cleaning up listeners for previous clientWidgetApi instance.`); + } + + if (roomId && clientWidgetApi) { + logger.debug(`CallContext: Registering active clientWidgetApi for room ${roomId}.`); + setActiveClientWidgetApi(clientWidgetApi, clientWidget, roomId); + } else if (roomId === activeClientWidgetApiRoomId || roomId === null) { + setActiveClientWidgetApi(null, null, null); + resetMediaState(); + } + }, + [activeClientWidgetApi, activeClientWidgetApiRoomId, setActiveClientWidgetApi, resetMediaState] + ); + + const setViewedClientWidgetApi = useCallback( + ( + clientWidgetApi: ClientWidgetApi | null, + clientWidget: SmallWidget | null, + roomId: string | null + ) => { + setViewedClientWidgetApiState(clientWidgetApi); + setViewedClientWidget(clientWidget); + setViewedClientWidgetApiRoomId(roomId); + }, + [] + ); + + const registerViewedClientWidgetApi = useCallback( + ( + roomId: string | null, + clientWidgetApi: ClientWidgetApi | null, + clientWidget: SmallWidget | null + ) => { + if (viewedClientWidgetApi && viewedClientWidgetApi !== clientWidgetApi) { + logger.debug(`CallContext: Cleaning up listeners for previous clientWidgetApi instance.`); + } + + if (roomId && clientWidgetApi) { + logger.debug(`CallContext: Registering viewed clientWidgetApi for room ${roomId}.`); + setViewedClientWidgetApi(clientWidgetApi, clientWidget, roomId); + } else if (roomId === viewedClientWidgetApiRoomId || roomId === null) { + logger.debug( + `CallContext: Clearing viewed clientWidgetApi for room ${viewedClientWidgetApiRoomId}.` + ); + setViewedClientWidgetApi(null, null, null); + } + }, + [viewedClientWidgetApi, viewedClientWidgetApiRoomId, setViewedClientWidgetApi] + ); + + const hangUp = useCallback( + (nextRoom: string) => { + if (typeof nextRoom === 'string') { + logger.debug('1 Hangup'); + setActiveClientWidgetApi(null, null, null); + setActiveCallRoomIdState(null); + activeClientWidgetApi?.transport.send(`${WIDGET_HANGUP_ACTION}`, {}); + } else if (viewedRoomId !== activeCallRoomId) { + logger.debug('2 Hangup'); + setActiveClientWidgetApi(null, null, null); + setActiveCallRoomIdState(null); + activeClientWidgetApi?.transport.send(`${WIDGET_HANGUP_ACTION}`, {}); + } else if (activeClientWidget) { + logger.debug('3 Hangup'); + const iframeDoc = + activeClientWidget?.iframe?.contentDocument || + activeClientWidget?.iframe?.contentWindow.document; + const button = iframeDoc.querySelector('[data-testid="incall_leave"]'); + button.click(); + } + setIsCallActive(false); + + logger.debug(`CallContext: Hang up called.`); + }, + [ + activeCallRoomId, + activeClientWidget, + activeClientWidgetApi?.transport, + setActiveClientWidgetApi, + viewedRoomId, + ] + ); + + useEffect(() => { + if (!activeCallRoomId && !viewedCallRoomId) { + return; + } + if (!lastViewedRoomDuringCall) { + if (activeCallRoomId) + setLastViewedRoomDuringCall((prevLastRoom) => prevLastRoom || activeCallRoomId); + } + if ( + lastViewedRoomDuringCall && + lastViewedRoomDuringCall !== viewedRoomId && + activeCallRoomId && + isCallActive + ) { + setLastViewedRoomDuringCall(activeCallRoomId); + } + + const handleHangup = (ev: CustomEvent) => { + ev.preventDefault(); + if (isCallActive && ev.detail.widgetId === activeClientWidgetApi?.widget.id) { + activeClientWidgetApi?.transport.reply(ev.detail, {}); + } + logger.debug( + `CallContext: Received hangup action from widget in room ${activeCallRoomId}.`, + ev + ); + }; + + const handleMediaStateUpdate = (ev: CustomEvent) => { + ev.preventDefault(); + logger.debug( + `CallContext: Received media state update from widget in room ${activeCallRoomId}:`, + ev.detail + ); + + /* eslint-disable camelcase */ + const { audio_enabled, video_enabled } = ev.detail.data ?? {}; + + if (typeof audio_enabled === 'boolean' && audio_enabled !== isAudioEnabled) { + logger.debug(`CallContext: Updating audio enabled state from widget: ${audio_enabled}`); + setIsAudioEnabledState(audio_enabled); + } + if (typeof video_enabled === 'boolean' && video_enabled !== isVideoEnabled) { + logger.debug(`CallContext: Updating video enabled state from widget: ${video_enabled}`); + setIsVideoEnabledState(video_enabled); + } + /* eslint-enable camelcase */ + }; + + const handleOnScreenStateUpdate = (ev: CustomEvent) => { + ev.preventDefault(); + if (isPrimaryIframe) { + activeClientWidgetApi?.transport.reply(ev.detail, {}); + } else { + viewedClientWidgetApi?.transport.reply(ev.detail, {}); + } + }; + + const handleOnTileLayout = (ev: CustomEvent) => { + ev.preventDefault(); + if (isPrimaryIframe) { + activeClientWidgetApi?.transport.reply(ev.detail, {}); + } else { + viewedClientWidgetApi?.transport.reply(ev.detail, {}); + } + }; + + const handleJoin = (ev: CustomEvent) => { + ev.preventDefault(); + const setViewedAsActive = () => { + if (viewedCallRoomId !== activeCallRoomId) setIsPrimaryIframe(!isPrimaryIframe); + setActiveClientWidgetApi(viewedClientWidgetApi, viewedClientWidget, viewedCallRoomId); + setActiveCallRoomIdState(viewedCallRoomId); + setIsCallActive(true); + const iframeDoc = + viewedClientWidget?.iframe?.contentDocument || + viewedClientWidget?.iframe?.contentWindow.document; + const observer = new MutationObserver(() => { + const button = iframeDoc.querySelector('[data-testid="incall_leave"]'); + if (button) { + button.addEventListener('click', () => { + setIsCallActive(false); + }); + } + observer.disconnect(); + }); + observer.observe(iframeDoc, { childList: true, subtree: true }); + }; + + if (ev.detail.widgetId === activeClientWidgetApi?.widget.id) { + activeClientWidgetApi?.transport.reply(ev.detail, {}); + const iframeDoc = + activeClientWidget?.iframe?.contentDocument || + activeClientWidget?.iframe?.contentWindow.document; + const observer = new MutationObserver(() => { + const button = iframeDoc.querySelector('[data-testid="incall_leave"]'); + if (button) { + button.addEventListener('click', () => { + setIsCallActive(false); + }); + } + observer.disconnect(); + }); + logger.debug('1 Join'); + observer.observe(iframeDoc, { childList: true, subtree: true }); + setIsCallActive(true); + return; + } + if ( + lastViewedRoomDuringCall && + viewedRoomId === activeCallRoomId && + lastViewedRoomDuringCall === activeCallRoomId + ) { + logger.debug('2 Join'); + setIsCallActive(true); + return; + } + if (activeClientWidgetApi) { + if (isCallActive && viewedClientWidgetApi && viewedCallRoomId) { + activeClientWidgetApi?.removeAllListeners(); + activeClientWidgetApi?.transport.send(WIDGET_HANGUP_ACTION, {}).then(() => { + logger.debug('3 Join'); + setViewedAsActive(); + }); + } else { + logger.debug('4 Join'); + setViewedAsActive(); + setIsCallActive(true); + } + } else if (viewedCallRoomId !== viewedRoomId) { + logger.debug('5 Join'); + setIsCallActive(true); + } else { + logger.debug('6 Join'); + setViewedAsActive(); + } + }; + + logger.debug( + `CallContext: Setting up listeners for clientWidgetApi in room ${activeCallRoomId}` + ); + activeClientWidgetApi?.on(`action:${WIDGET_HANGUP_ACTION}`, handleHangup); + activeClientWidgetApi?.on(`action:${WIDGET_MEDIA_STATE_UPDATE_ACTION}`, handleMediaStateUpdate); + viewedClientWidgetApi?.on(`action:${WIDGET_TILE_UPDATE}`, handleOnTileLayout); + activeClientWidgetApi?.on(`action:${WIDGET_ON_SCREEN_ACTION}`, handleOnScreenStateUpdate); + activeClientWidgetApi?.on(`action:${WIDGET_JOIN_ACTION}`, handleJoin); + + viewedClientWidgetApi?.on(`action:${WIDGET_JOIN_ACTION}`, handleJoin); + viewedClientWidgetApi?.on(`action:${WIDGET_MEDIA_STATE_UPDATE_ACTION}`, handleMediaStateUpdate); + viewedClientWidgetApi?.on(`action:${WIDGET_TILE_UPDATE}`, handleOnTileLayout); + viewedClientWidgetApi?.on(`action:${WIDGET_ON_SCREEN_ACTION}`, handleOnScreenStateUpdate); + viewedClientWidgetApi?.on(`action:${WIDGET_HANGUP_ACTION}`, handleHangup); + }, [ + activeClientWidgetApi, + activeCallRoomId, + activeClientWidgetApiRoomId, + hangUp, + isChatOpen, + isAudioEnabled, + isVideoEnabled, + isCallActive, + viewedRoomId, + viewedClientWidgetApi, + isPrimaryIframe, + viewedCallRoomId, + setViewedClientWidgetApi, + setActiveClientWidgetApi, + viewedClientWidget, + setViewedCallRoomId, + lastViewedRoomDuringCall, + activeClientWidget?.iframe?.contentDocument, + activeClientWidget?.iframe?.contentWindow?.document, + ]); + + const sendWidgetAction = useCallback( + async (action: WidgetApiToWidgetAction | string, data: T): Promise => { + if (!activeClientWidgetApi) { + logger.warn( + `CallContext: Cannot send action '${action}', no active API clientWidgetApi registered.` + ); + return Promise.reject(new Error('No active call clientWidgetApi')); + } + if (!activeClientWidgetApiRoomId || activeClientWidgetApiRoomId !== activeCallRoomId) { + logger.debug( + `CallContext: Cannot send action '${action}', clientWidgetApi room (${activeClientWidgetApiRoomId}) does not match active call room (${activeCallRoomId}). Stale clientWidgetApi?` + ); + return Promise.reject(new Error('Mismatched active call clientWidgetApi')); + } + logger.debug( + `CallContext: Sending action '${action}' via active clientWidgetApi (room: ${activeClientWidgetApiRoomId}) with data:`, + data + ); + await activeClientWidgetApi.transport.send(action as WidgetApiAction, data); + }, + [activeClientWidgetApi, activeCallRoomId, activeClientWidgetApiRoomId] + ); + + const toggleAudio = useCallback(async () => { + const newState = !isAudioEnabled; + logger.debug(`CallContext: Toggling audio. New state: enabled=${newState}`); + setIsAudioEnabledState(newState); + try { + await sendWidgetAction(WIDGET_MEDIA_STATE_UPDATE_ACTION, { + audio_enabled: newState, + video_enabled: isVideoEnabled, + }); + logger.debug(`CallContext: Successfully sent audio toggle action.`); + } catch (error) { + setIsAudioEnabledState(!newState); + throw error; + } + }, [isAudioEnabled, isVideoEnabled, sendWidgetAction]); + + const toggleVideo = useCallback(async () => { + const newState = !isVideoEnabled; + logger.debug(`CallContext: Toggling video. New state: enabled=${newState}`); + setIsVideoEnabledState(newState); + try { + await sendWidgetAction(WIDGET_MEDIA_STATE_UPDATE_ACTION, { + audio_enabled: isAudioEnabled, + video_enabled: newState, + }); + logger.debug(`CallContext: Successfully sent video toggle action.`); + } catch (error) { + setIsVideoEnabledState(!newState); + throw error; + } + }, [isVideoEnabled, isAudioEnabled, sendWidgetAction]); + + const toggleChat = useCallback(async () => { + const newState = !isChatOpen; + setIsChatOpenState(newState); + }, [isChatOpen]); + + const toggleIframe = useCallback(async () => { + const newState = !isPrimaryIframe; + setIsPrimaryIframe(newState); + }, [isPrimaryIframe]); + + const contextValue = useMemo( + () => ({ + activeCallRoomId, + setActiveCallRoomId, + viewedCallRoomId, + setViewedCallRoomId, + hangUp, + activeClientWidgetApi, + registerActiveClientWidgetApi, + activeClientWidget, + viewedClientWidgetApi, + registerViewedClientWidgetApi, + viewedClientWidget, + sendWidgetAction, + isChatOpen, + isAudioEnabled, + isVideoEnabled, + isCallActive, + isPrimaryIframe, + toggleAudio, + toggleVideo, + toggleChat, + toggleIframe, + }), + [ + activeCallRoomId, + setActiveCallRoomId, + viewedCallRoomId, + setViewedCallRoomId, + hangUp, + activeClientWidgetApi, + registerActiveClientWidgetApi, + activeClientWidget, + viewedClientWidgetApi, + registerViewedClientWidgetApi, + viewedClientWidget, + sendWidgetAction, + isChatOpen, + isAudioEnabled, + isVideoEnabled, + isCallActive, + isPrimaryIframe, + toggleAudio, + toggleVideo, + toggleChat, + toggleIframe, + ] + ); + + return {children}; +} + +export function useCallState(): CallContextState { + const context = useContext(CallContext); + if (context === undefined) { + throw new Error('useCallState must be used within a CallProvider'); + } + return context; +} diff --git a/src/app/pages/client/call/PersistentCallContainer.tsx b/src/app/pages/client/call/PersistentCallContainer.tsx new file mode 100644 index 00000000..bf35b45e --- /dev/null +++ b/src/app/pages/client/call/PersistentCallContainer.tsx @@ -0,0 +1,237 @@ +import React, { createContext, ReactNode, useCallback, useEffect, useMemo, useRef } from 'react'; +import { logger } from 'matrix-js-sdk/lib/logger'; +import { ClientWidgetApi } from 'matrix-widget-api'; +import { Box } from 'folds'; +import { useParams } from 'react-router-dom'; +import { useCallState } from './CallProvider'; +import { + createVirtualWidget, + SmallWidget, + getWidgetData, + getWidgetUrl, +} from '../../../features/call/SmallWidget'; +import { useMatrixClient } from '../../../hooks/useMatrixClient'; +import { useClientConfig } from '../../../hooks/useClientConfig'; +import { ScreenSize, useScreenSizeContext } from '../../../hooks/useScreenSize'; + +interface PersistentCallContainerProps { + children: ReactNode; +} + +export const PrimaryRefContext = createContext(null); +export const BackupRefContext = createContext(null); + +export function PersistentCallContainer({ children }: PersistentCallContainerProps) { + const primaryIframeRef = useRef(null); + const primaryWidgetApiRef = useRef(null); + const primarySmallWidgetRef = useRef(null); + + const backupIframeRef = useRef(null); + const backupWidgetApiRef = useRef(null); + const backupSmallWidgetRef = useRef(null); + const { + activeCallRoomId, + viewedCallRoomId, + isChatOpen, + isCallActive, + isPrimaryIframe, + registerActiveClientWidgetApi, + activeClientWidget, + registerViewedClientWidgetApi, + viewedClientWidget, + } = useCallState(); + const mx = useMatrixClient(); + const clientConfig = useClientConfig(); + const screenSize = useScreenSizeContext(); + const isMobile = screenSize === ScreenSize.Mobile; + const { roomIdOrAlias: viewedRoomId } = useParams(); + const isViewingActiveCall = useMemo( + () => activeCallRoomId !== null && activeCallRoomId === viewedRoomId, + [activeCallRoomId, viewedRoomId] + ); + /* eslint-disable no-param-reassign */ + + const setupWidget = useCallback( + ( + widgetApiRef: { current: ClientWidgetApi }, + smallWidgetRef: { current: SmallWidget }, + iframeRef: { current: { src: string } }, + skipLobby: { toString: () => any } + ) => { + if (mx?.getUserId()) { + if ( + (activeCallRoomId !== viewedCallRoomId && isCallActive) || + (activeCallRoomId && !isCallActive) || + (!activeCallRoomId && viewedCallRoomId && !isCallActive) + ) { + const roomIdToSet = (skipLobby ? activeCallRoomId : viewedCallRoomId) ?? ''; + if (roomIdToSet === '') { + return; + } + const widgetId = `element-call-${roomIdToSet}-${Date.now()}`; + const newUrl = getWidgetUrl( + mx, + roomIdToSet, + clientConfig.elementCallUrl ?? '', + widgetId, + { + skipLobby: skipLobby.toString(), + returnToLobby: 'true', + perParticipantE2EE: 'true', + } + ); + + if ( + (primarySmallWidgetRef.current?.roomId || backupSmallWidgetRef.current?.roomId) && + (skipLobby + ? activeClientWidget?.roomId && + //activeCallRoomId === activeClientWidget.roomId && + (activeClientWidget.roomId === primarySmallWidgetRef.current?.roomId || + activeClientWidget.roomId === backupSmallWidgetRef.current?.roomId) + : viewedClientWidget?.roomId && + viewedCallRoomId === viewedClientWidget.roomId && + (viewedClientWidget.roomId === primarySmallWidgetRef.current?.roomId || + viewedClientWidget.roomId === backupSmallWidgetRef.current?.roomId)) + ) { + return; + } + + if (iframeRef.current && iframeRef.current.src !== newUrl.toString()) { + iframeRef.current.src = newUrl.toString(); + } else if (iframeRef.current && !iframeRef.current.src) { + iframeRef.current.src = newUrl.toString(); + } + + const iframeElement = iframeRef.current; + if (!iframeElement) { + return; + } + + const userId = mx.getUserId() ?? ''; + const app = createVirtualWidget( + mx, + widgetId, + userId, + 'Element Call', + 'm.call', + newUrl, + true, + getWidgetData(mx, roomIdToSet, {}, { skipLobby: true }), + roomIdToSet + ); + + const smallWidget = new SmallWidget(app); + smallWidgetRef.current = smallWidget; + + const widgetApiInstance = smallWidget.startMessaging(iframeElement); + widgetApiRef.current = widgetApiInstance; + if (skipLobby) { + registerActiveClientWidgetApi(activeCallRoomId, widgetApiRef.current, smallWidget); + } else { + registerViewedClientWidgetApi(viewedCallRoomId, widgetApiRef.current, smallWidget); + } + + widgetApiInstance.once('ready', () => { + logger.info(`PersistentCallContainer: Widget for ${roomIdToSet} is ready.`); + }); + } + } + }, + [ + mx, + activeCallRoomId, + viewedCallRoomId, + isCallActive, + clientConfig.elementCallUrl, + viewedClientWidget, + activeClientWidget, + viewedRoomId, + registerActiveClientWidgetApi, + registerViewedClientWidgetApi, + ] + ); + + useEffect(() => { + if ((activeCallRoomId && !viewedCallRoomId) || (activeCallRoomId && viewedCallRoomId)) + setupWidget(primaryWidgetApiRef, primarySmallWidgetRef, primaryIframeRef, isPrimaryIframe); + if ((!activeCallRoomId && viewedCallRoomId) || (viewedCallRoomId && activeCallRoomId)) + setupWidget(backupWidgetApiRef, backupSmallWidgetRef, backupIframeRef, !isPrimaryIframe); + }, [ + setupWidget, + primaryWidgetApiRef, + primarySmallWidgetRef, + primaryIframeRef, + backupWidgetApiRef, + backupSmallWidgetRef, + backupIframeRef, + registerActiveClientWidgetApi, + registerViewedClientWidgetApi, + activeCallRoomId, + viewedCallRoomId, + isCallActive, + isPrimaryIframe, + ]); + + const memoizedIframeRef = useMemo(() => primaryIframeRef, [primaryIframeRef]); + const memoizedBackupIframeRef = useMemo(() => backupIframeRef, [backupIframeRef]); + + return ( + + + + + +