From 8df10eeecac15ddb45ed4e350d33814ac4690f89 Mon Sep 17 00:00:00 2001 From: trilene Date: Thu, 18 Feb 2021 15:55:29 -0500 Subject: [PATCH 01/11] Support desktop screen sharing on X11 --- resources/icons/ui/screen-share.png | Bin 0 -> 773 bytes resources/qml/TimelineView.qml | 2 +- resources/qml/voip/ActiveCallBar.qml | 52 ++++++++++- resources/qml/voip/CallDevices.qml | 2 +- resources/qml/voip/CallInvite.qml | 8 +- resources/qml/voip/CallInviteBar.qml | 8 +- resources/qml/voip/PlaceCall.qml | 23 ++++- resources/qml/voip/ScreenShare.qml | 95 +++++++++++++++++++ resources/res.qrc | 2 + src/CallManager.cpp | 45 ++++++--- src/CallManager.h | 25 +++-- src/UserSettingsPage.cpp | 38 ++++++-- src/UserSettingsPage.h | 12 +++ src/WebRTCSession.cpp | 134 +++++++++++++++++++-------- src/WebRTCSession.h | 20 +++- 15 files changed, 376 insertions(+), 90 deletions(-) create mode 100644 resources/icons/ui/screen-share.png create mode 100644 resources/qml/voip/ScreenShare.qml diff --git a/resources/icons/ui/screen-share.png b/resources/icons/ui/screen-share.png new file mode 100644 index 0000000000000000000000000000000000000000..d6cee4277f6ba194c43a627532c7b3666f029fab GIT binary patch literal 773 zcmV+g1N!`lP)n-*ntJ3vZ_0%2|4{mkpWFVHyzBH*oQ?1)Jham710r0 zWWboAlMMJ&MW>x)zzrM}X6-VF#_c2n;3y8@XS+J=Gy~u)cHvt)I_pFO;IdGOen}X! zD;|N7+{7k)D6^BQ4HyzOTc>b3^8H=fzGi;2Ksn zB4h+LJZ+)x3$dW>dIScs4_ok{$ruCJfP0BFfe9&t74TA+%w8vmHCT`JLZaPj*(2bI z@T;Xro}l-r=8}&tgJq6@ci1f53_cO(urpCSlxM&Y_DGlf6rniUk|^FE$}-?MZjIu) zxtNEk!b$G)$Tc%?73*8n)X*lYm1XcFW=CR9!gd_PePR1M^1lAaJ@bTBBgGQj7fZB^ z5pYD9t%vEwA?y}6rifl)i@ZFYw7*Ghz;~Rizn_T1!XAHegK?G#$uCMLElz3!uHZcu z7=_h5y-9Exy(E5DSZv9RCjkKH&_$3U&RQ`fE4vML0V=f&Ib`HA;_y z0%5F>>wu~SR82OZY5`T#Bk?hFCX>lzGMP*!(;fT0y70lMx00000NkvXXu0mjf D 0 + visible: CallManager.callType == CallType.VIDEO && CallManager.cameras.length > 0 Image { Layout.preferredWidth: 22 diff --git a/resources/qml/voip/CallInvite.qml b/resources/qml/voip/CallInvite.qml index 00dcc77f..df3343ed 100644 --- a/resources/qml/voip/CallInvite.qml +++ b/resources/qml/voip/CallInvite.qml @@ -53,7 +53,7 @@ Popup { Layout.bottomMargin: msgView.height / 25 Image { - property string image: CallManager.isVideo ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png" + property string image: CallManager.callType == CallType.VIDEO ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png" Layout.alignment: Qt.AlignCenter Layout.preferredWidth: msgView.height / 10 @@ -63,7 +63,7 @@ Popup { Label { Layout.alignment: Qt.AlignCenter - text: CallManager.isVideo ? qsTr("Video Call") : qsTr("Voice Call") + text: CallManager.callType == CallType.VIDEO ? qsTr("Video Call") : qsTr("Voice Call") font.pointSize: fontMetrics.font.pointSize * 2 color: colors.windowText } @@ -97,7 +97,7 @@ Popup { } RowLayout { - visible: CallManager.isVideo && CallManager.cameras.length > 0 + visible: CallManager.callType == CallType.VIDEO && CallManager.cameras.length > 0 Layout.alignment: Qt.AlignCenter Image { @@ -159,7 +159,7 @@ Popup { RoundButton { id: acceptButton - property string image: CallManager.isVideo ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png" + property string image: CallManager.callType == CallType.VIDEO ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png" implicitWidth: buttonLayout.buttonSize implicitHeight: buttonLayout.buttonSize diff --git a/resources/qml/voip/CallInviteBar.qml b/resources/qml/voip/CallInviteBar.qml index 65749c35..bf630e9e 100644 --- a/resources/qml/voip/CallInviteBar.qml +++ b/resources/qml/voip/CallInviteBar.qml @@ -52,12 +52,12 @@ Rectangle { Layout.leftMargin: 4 Layout.preferredWidth: 24 Layout.preferredHeight: 24 - source: CallManager.isVideo ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png" + source: CallManager.callType == CallType.VIDEO ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png" } Label { font.pointSize: fontMetrics.font.pointSize * 1.1 - text: CallManager.isVideo ? qsTr("Video Call") : qsTr("Voice Call") + text: CallManager.callType == CallType.VIDEO ? qsTr("Video Call") : qsTr("Voice Call") color: "#000000" } @@ -83,7 +83,7 @@ Rectangle { Button { Layout.rightMargin: 4 - icon.source: CallManager.isVideo ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png" + icon.source: CallManager.callType == CallType.VIDEO ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png" text: qsTr("Accept") palette: colors onClicked: { @@ -102,7 +102,7 @@ Rectangle { dialog.open(); return ; } - if (CallManager.isVideo && CallManager.cameras.length > 0 && !CallManager.cameras.includes(Settings.camera)) { + if (CallManager.callType == CallType.VIDEO && CallManager.cameras.length > 0 && !CallManager.cameras.includes(Settings.camera)) { var dialog = deviceError.createObject(timelineRoot, { "errorString": qsTr("Unknown camera: %1").arg(Settings.camera), "image": ":/icons/icons/ui/video-call.png" diff --git a/resources/qml/voip/PlaceCall.qml b/resources/qml/voip/PlaceCall.qml index 41cbd54c..5dbeb6e1 100644 --- a/resources/qml/voip/PlaceCall.qml +++ b/resources/qml/voip/PlaceCall.qml @@ -23,6 +23,14 @@ Popup { } + Component { + id: screenShareDialog + + ScreenShare { + } + + } + ColumnLayout { id: columnLayout @@ -76,7 +84,7 @@ Popup { onClicked: { if (buttonLayout.validateMic()) { Settings.microphone = micCombo.currentText; - CallManager.sendInvite(TimelineManager.timeline.roomId(), false); + CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.VOICE); close(); } } @@ -90,12 +98,23 @@ Popup { if (buttonLayout.validateMic()) { Settings.microphone = micCombo.currentText; Settings.camera = cameraCombo.currentText; - CallManager.sendInvite(TimelineManager.timeline.roomId(), true); + CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.VIDEO); close(); } } } + Button { + visible: CallManager.screenShareSupported + text: qsTr("Screen") + icon.source: "qrc:/icons/icons/ui/screen-share.png" + onClicked: { + var dialog = screenShareDialog.createObject(timelineRoot); + dialog.open(); + close(); + } + } + Button { text: qsTr("Cancel") onClicked: { diff --git a/resources/qml/voip/ScreenShare.qml b/resources/qml/voip/ScreenShare.qml new file mode 100644 index 00000000..b21a26fd --- /dev/null +++ b/resources/qml/voip/ScreenShare.qml @@ -0,0 +1,95 @@ +import "../" +import QtQuick 2.9 +import QtQuick.Controls 2.3 +import QtQuick.Layouts 1.2 +import im.nheko 1.0 + +Popup { + modal: true + // only set the anchors on Qt 5.12 or higher + // see https://doc.qt.io/qt-5/qml-qtquick-controls2-popup.html#anchors.centerIn-prop + Component.onCompleted: { + if (anchors) + anchors.centerIn = parent; + + frameRateCombo.currentIndex = frameRateCombo.find(Settings.screenShareFrameRate); + remoteVideoCheckBox.checked = Settings.screenShareRemoteVideo; + } + palette: colors + + ColumnLayout { + Label { + Layout.margins: 8 + Layout.alignment: Qt.AlignLeft + text: qsTr("Share desktop with %1?").arg(TimelineManager.timeline.roomName) + color: colors.windowText + } + + RowLayout { + Layout.leftMargin: 8 + Layout.rightMargin: 8 + + Label { + Layout.alignment: Qt.AlignLeft + text: qsTr("Frame rate:") + color: colors.windowText + } + + ComboBox { + id: frameRateCombo + + Layout.alignment: Qt.AlignRight + model: ["25", "20", "15", "10", "5", "2", "1"] + } + + } + + CheckBox { + id: remoteVideoCheckBox + + Layout.alignment: Qt.AlignLeft + Layout.leftMargin: 8 + Layout.rightMargin: 8 + text: qsTr("Request remote camera") + ToolTip.text: qsTr("View your callee's camera like a regular video call") + ToolTip.visible: hovered + } + + RowLayout { + Layout.margins: 8 + + Item { + Layout.fillWidth: true + } + + Button { + text: qsTr("Share") + icon.source: "qrc:/icons/icons/ui/screen-share.png" + onClicked: { + if (buttonLayout.validateMic()) { + Settings.microphone = micCombo.currentText; + Settings.screenShareFrameRate = frameRateCombo.currentText; + Settings.screenShareRemoteVideo = remoteVideoCheckBox.checked; + CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.SCREEN); + close(); + } + } + } + + Button { + text: qsTr("Cancel") + onClicked: { + close(); + } + } + + } + + } + + background: Rectangle { + color: colors.window + border.color: colors.windowText + } + +} diff --git a/resources/res.qrc b/resources/res.qrc index 308d81a6..2387fa75 100644 --- a/resources/res.qrc +++ b/resources/res.qrc @@ -74,6 +74,7 @@ icons/ui/end-call.png icons/ui/microphone-mute.png icons/ui/microphone-unmute.png + icons/ui/screen-share.png icons/ui/toggle-camera-view.png icons/ui/video-call.png @@ -165,6 +166,7 @@ qml/voip/CallInviteBar.qml qml/voip/DeviceError.qml qml/voip/PlaceCall.qml + qml/voip/ScreenShare.qml qml/voip/VideoCall.qml diff --git a/src/CallManager.cpp b/src/CallManager.cpp index 7acd9592..51bb7b33 100644 --- a/src/CallManager.cpp +++ b/src/CallManager.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -24,6 +25,8 @@ Q_DECLARE_METATYPE(mtx::responses::TurnServer) using namespace mtx::events; using namespace mtx::events::msg; +using webrtc::CallType; + namespace { std::vector getTurnURIs(const mtx::responses::TurnServer &turnServer); @@ -148,10 +151,12 @@ CallManager::CallManager(QObject *parent) } void -CallManager::sendInvite(const QString &roomid, bool isVideo) +CallManager::sendInvite(const QString &roomid, CallType callType) { if (isOnCall()) return; + if (callType == CallType::SCREEN && !screenShareSupported()) + return; auto roomInfo = cache::singleRoomInfo(roomid.toStdString()); if (roomInfo.member_count != 2) { @@ -161,17 +166,20 @@ CallManager::sendInvite(const QString &roomid, bool isVideo) std::string errorMessage; if (!session_.havePlugins(false, &errorMessage) || - (isVideo && !session_.havePlugins(true, &errorMessage))) { + ((callType == CallType::VIDEO || callType == CallType::SCREEN) && + !session_.havePlugins(true, &errorMessage))) { emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage)); return; } - isVideo_ = isVideo; - roomid_ = roomid; + callType_ = callType; + roomid_ = roomid; session_.setTurnServers(turnURIs_); generateCallID(); - nhlog::ui()->debug( - "WebRTC: call id: {} - creating {} invite", callid_, isVideo ? "video" : "voice"); + std::string strCallType = callType_ == CallType::VOICE + ? "voice" + : (callType_ == CallType::VIDEO ? "video" : "screen"); + nhlog::ui()->debug("WebRTC: call id: {} - creating {} invite", callid_, strCallType); std::vector members(cache::getMembers(roomid.toStdString())); const RoomMember &callee = members.front().user_id == utils::localUser() ? members.back() : members.front(); @@ -179,7 +187,7 @@ CallManager::sendInvite(const QString &roomid, bool isVideo) callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url); emit newInviteState(); playRingtone(QUrl("qrc:/media/media/ringback.ogg"), true); - if (!session_.createOffer(isVideo)) { + if (!session_.createOffer(callType)) { emit ChatPage::instance()->showNotification("Problem setting up call."); endCall(); } @@ -280,7 +288,7 @@ CallManager::handleEvent(const RoomEvent &callInviteEvent) callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url); haveCallInvite_ = true; - isVideo_ = isVideo; + callType_ = isVideo ? CallType::VIDEO : CallType::VOICE; inviteSDP_ = callInviteEvent.content.sdp; CallDevices::instance().refresh(); emit newInviteState(); @@ -295,7 +303,7 @@ CallManager::acceptInvite() stopRingtone(); std::string errorMessage; if (!session_.havePlugins(false, &errorMessage) || - (isVideo_ && !session_.havePlugins(true, &errorMessage))) { + (callType_ == CallType::VIDEO && !session_.havePlugins(true, &errorMessage))) { emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage)); hangUp(); return; @@ -383,7 +391,7 @@ CallManager::toggleMicMute() } bool -CallManager::callsSupported() const +CallManager::callsSupported() { #ifdef GSTREAMER_AVAILABLE return true; @@ -392,6 +400,21 @@ CallManager::callsSupported() const #endif } +bool +CallManager::screenShareSupported() +{ + return std::getenv("DISPLAY") != nullptr; +} + +bool +CallManager::haveVideo() const +{ + return callType() == CallType::VIDEO || + (callType() == CallType::SCREEN && + (ChatPage::instance()->userSettings()->screenShareRemoteVideo() && + !session_.isRemoteVideoRecvOnly())); +} + QStringList CallManager::devices(bool isVideo) const { @@ -424,7 +447,7 @@ CallManager::clear() callParty_.clear(); callPartyAvatarUrl_.clear(); callid_.clear(); - isVideo_ = false; + callType_ = CallType::VOICE; haveCallInvite_ = false; emit newInviteState(); inviteSDP_.clear(); diff --git a/src/CallManager.h b/src/CallManager.h index 97cffbc8..ed745b5b 100644 --- a/src/CallManager.h +++ b/src/CallManager.h @@ -25,34 +25,39 @@ class CallManager : public QObject Q_OBJECT Q_PROPERTY(bool haveCallInvite READ haveCallInvite NOTIFY newInviteState) Q_PROPERTY(bool isOnCall READ isOnCall NOTIFY newCallState) - Q_PROPERTY(bool isVideo READ isVideo NOTIFY newInviteState) - Q_PROPERTY(bool haveLocalVideo READ haveLocalVideo NOTIFY newCallState) + Q_PROPERTY(webrtc::CallType callType READ callType NOTIFY newInviteState) Q_PROPERTY(webrtc::State callState READ callState NOTIFY newCallState) Q_PROPERTY(QString callParty READ callParty NOTIFY newInviteState) Q_PROPERTY(QString callPartyAvatarUrl READ callPartyAvatarUrl NOTIFY newInviteState) Q_PROPERTY(bool isMicMuted READ isMicMuted NOTIFY micMuteChanged) - Q_PROPERTY(bool callsSupported READ callsSupported CONSTANT) + Q_PROPERTY(bool haveLocalCamera READ haveLocalCamera NOTIFY newCallState) + Q_PROPERTY(bool haveVideo READ haveVideo NOTIFY newInviteState) Q_PROPERTY(QStringList mics READ mics NOTIFY devicesChanged) Q_PROPERTY(QStringList cameras READ cameras NOTIFY devicesChanged) + Q_PROPERTY(bool callsSupported READ callsSupported CONSTANT) + Q_PROPERTY(bool screenShareSupported READ screenShareSupported CONSTANT) public: CallManager(QObject *); bool haveCallInvite() const { return haveCallInvite_; } bool isOnCall() const { return session_.state() != webrtc::State::DISCONNECTED; } - bool isVideo() const { return isVideo_; } - bool haveLocalVideo() const { return session_.haveLocalVideo(); } + webrtc::CallType callType() const { return callType_; } webrtc::State callState() const { return session_.state(); } QString callParty() const { return callParty_; } QString callPartyAvatarUrl() const { return callPartyAvatarUrl_; } bool isMicMuted() const { return session_.isMicMuted(); } - bool callsSupported() const; + bool haveLocalCamera() const { return session_.haveLocalCamera(); } + bool haveVideo() const; QStringList mics() const { return devices(false); } QStringList cameras() const { return devices(true); } void refreshTurnServer(); + static bool callsSupported(); + static bool screenShareSupported(); + public slots: - void sendInvite(const QString &roomid, bool isVideo); + void sendInvite(const QString &roomid, webrtc::CallType); void syncEvent(const mtx::events::collections::TimelineEvents &event); void refreshDevices() { CallDevices::instance().refresh(); } void toggleMicMute(); @@ -81,9 +86,9 @@ private: QString callParty_; QString callPartyAvatarUrl_; std::string callid_; - const uint32_t timeoutms_ = 120000; - bool isVideo_ = false; - bool haveCallInvite_ = false; + const uint32_t timeoutms_ = 120000; + webrtc::CallType callType_ = webrtc::CallType::VOICE; + bool haveCallInvite_ = false; std::string inviteSDP_; std::vector remoteICECandidates_; std::vector turnURIs_; diff --git a/src/UserSettingsPage.cpp b/src/UserSettingsPage.cpp index b6fdf504..186a03bb 100644 --- a/src/UserSettingsPage.cpp +++ b/src/UserSettingsPage.cpp @@ -107,13 +107,15 @@ UserSettings::load(std::optional profile) auto presenceValue = QMetaEnum::fromType().keyToValue(tempPresence.c_str()); if (presenceValue < 0) presenceValue = 0; - presence_ = static_cast(presenceValue); - ringtone_ = settings.value("user/ringtone", "Default").toString(); - microphone_ = settings.value("user/microphone", QString()).toString(); - camera_ = settings.value("user/camera", QString()).toString(); - cameraResolution_ = settings.value("user/camera_resolution", QString()).toString(); - cameraFrameRate_ = settings.value("user/camera_frame_rate", QString()).toString(); - useStunServer_ = settings.value("user/use_stun_server", false).toBool(); + presence_ = static_cast(presenceValue); + ringtone_ = settings.value("user/ringtone", "Default").toString(); + microphone_ = settings.value("user/microphone", QString()).toString(); + camera_ = settings.value("user/camera", QString()).toString(); + cameraResolution_ = settings.value("user/camera_resolution", QString()).toString(); + cameraFrameRate_ = settings.value("user/camera_frame_rate", QString()).toString(); + screenShareFrameRate_ = settings.value("user/screen_share_frame_rate", 5).toInt(); + screenShareRemoteVideo_ = settings.value("user/screen_share_remote_video", false).toBool(); + useStunServer_ = settings.value("user/use_stun_server", false).toBool(); if (profile) // set to "" if it's the default to maintain compatibility profile_ = (*profile == "default") ? "" : *profile; @@ -444,6 +446,26 @@ UserSettings::setCameraFrameRate(QString frameRate) save(); } +void +UserSettings::setScreenShareFrameRate(int frameRate) +{ + if (frameRate == screenShareFrameRate_) + return; + screenShareFrameRate_ = frameRate; + emit screenShareFrameRateChanged(frameRate); + save(); +} + +void +UserSettings::setScreenShareRemoteVideo(bool state) +{ + if (state == screenShareRemoteVideo_) + return; + screenShareRemoteVideo_ = state; + emit screenShareRemoteVideoChanged(state); + save(); +} + void UserSettings::setProfile(QString profile) { @@ -593,6 +615,8 @@ UserSettings::save() settings.setValue("camera", camera_); settings.setValue("camera_resolution", cameraResolution_); settings.setValue("camera_frame_rate", cameraFrameRate_); + settings.setValue("screen_share_frame_rate", screenShareFrameRate_); + settings.setValue("screen_share_remote_video", screenShareRemoteVideo_); settings.setValue("use_stun_server", useStunServer_); settings.setValue("currentProfile", profile_); diff --git a/src/UserSettingsPage.h b/src/UserSettingsPage.h index 49de94b3..4de9913a 100644 --- a/src/UserSettingsPage.h +++ b/src/UserSettingsPage.h @@ -86,6 +86,10 @@ class UserSettings : public QObject cameraResolutionChanged) Q_PROPERTY(QString cameraFrameRate READ cameraFrameRate WRITE setCameraFrameRate NOTIFY cameraFrameRateChanged) + Q_PROPERTY(int screenShareFrameRate READ screenShareFrameRate WRITE setScreenShareFrameRate + NOTIFY screenShareFrameRateChanged) + Q_PROPERTY(bool screenShareRemoteVideo READ screenShareRemoteVideo WRITE + setScreenShareRemoteVideo NOTIFY screenShareRemoteVideoChanged) Q_PROPERTY( bool useStunServer READ useStunServer WRITE setUseStunServer NOTIFY useStunServerChanged) Q_PROPERTY(bool shareKeysWithTrustedUsers READ shareKeysWithTrustedUsers WRITE @@ -143,6 +147,8 @@ public: void setCamera(QString camera); void setCameraResolution(QString resolution); void setCameraFrameRate(QString frameRate); + void setScreenShareFrameRate(int frameRate); + void setScreenShareRemoteVideo(bool state); void setUseStunServer(bool state); void setShareKeysWithTrustedUsers(bool state); void setProfile(QString profile); @@ -191,6 +197,8 @@ public: QString camera() const { return camera_; } QString cameraResolution() const { return cameraResolution_; } QString cameraFrameRate() const { return cameraFrameRate_; } + int screenShareFrameRate() const { return screenShareFrameRate_; } + bool screenShareRemoteVideo() const { return screenShareRemoteVideo_; } bool useStunServer() const { return useStunServer_; } bool shareKeysWithTrustedUsers() const { return shareKeysWithTrustedUsers_; } QString profile() const { return profile_; } @@ -229,6 +237,8 @@ signals: void cameraChanged(QString camera); void cameraResolutionChanged(QString resolution); void cameraFrameRateChanged(QString frameRate); + void screenShareFrameRateChanged(int frameRate); + void screenShareRemoteVideoChanged(bool state); void useStunServerChanged(bool state); void shareKeysWithTrustedUsersChanged(bool state); void profileChanged(QString profile); @@ -272,6 +282,8 @@ private: QString camera_; QString cameraResolution_; QString cameraFrameRate_; + int screenShareFrameRate_; + bool screenShareRemoteVideo_; bool useStunServer_; QString profile_; QString userId_; diff --git a/src/WebRTCSession.cpp b/src/WebRTCSession.cpp index b6d98058..9c01ddc4 100644 --- a/src/WebRTCSession.cpp +++ b/src/WebRTCSession.cpp @@ -10,6 +10,7 @@ #include #include +#include "CallDevices.h" #include "ChatPage.h" #include "Logging.h" #include "UserSettingsPage.h" @@ -29,14 +30,20 @@ extern "C" // https://github.com/vector-im/riot-web/issues/10173 #define STUN_SERVER "stun://turn.matrix.org:3478" +Q_DECLARE_METATYPE(webrtc::CallType) Q_DECLARE_METATYPE(webrtc::State) +using webrtc::CallType; using webrtc::State; WebRTCSession::WebRTCSession() : QObject() , devices_(CallDevices::instance()) { + qRegisterMetaType(); + qmlRegisterUncreatableMetaObject( + webrtc::staticMetaObject, "im.nheko", 1, 0, "CallType", "Can't instantiate enum"); + qRegisterMetaType(); qmlRegisterUncreatableMetaObject( webrtc::staticMetaObject, "im.nheko", 1, 0, "WebRTCState", "Can't instantiate enum"); @@ -455,7 +462,8 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe) nhlog::ui()->info("WebRTC: incoming video resolution: {}x{}", videoCallSize.first, videoCallSize.second); - addCameraView(pipe, videoCallSize); + if (session->callType() == CallType::VIDEO) + addCameraView(pipe, videoCallSize); } else { g_free(mediaType); nhlog::ui()->error("WebRTC: unknown pad type: {}", GST_PAD_NAME(newpad)); @@ -467,7 +475,7 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe) if (GST_PAD_LINK_FAILED(gst_pad_link(newpad, queuepad))) nhlog::ui()->error("WebRTC: unable to link new pad"); else { - if (!session->isVideo() || + if (session->callType() == CallType::VOICE || (haveAudioStream_ && (haveVideoStream_ || session->isRemoteVideoRecvOnly()))) { emit session->stateChanged(State::CONNECTED); @@ -523,14 +531,17 @@ getMediaAttributes(const GstSDPMessage *sdp, const char *mediaType, const char *encoding, int &payloadType, - bool &recvOnly) + bool &recvOnly, + bool &sendOnly) { payloadType = -1; recvOnly = false; + sendOnly = false; for (guint mlineIndex = 0; mlineIndex < gst_sdp_message_medias_len(sdp); ++mlineIndex) { const GstSDPMedia *media = gst_sdp_message_get_media(sdp, mlineIndex); if (!std::strcmp(gst_sdp_media_get_media(media), mediaType)) { recvOnly = gst_sdp_media_get_attribute_val(media, "recvonly") != nullptr; + sendOnly = gst_sdp_media_get_attribute_val(media, "sendonly") != nullptr; const gchar *rtpval = nullptr; for (guint n = 0; n == 0 || rtpval; ++n) { rtpval = gst_sdp_media_get_attribute_val_n(media, "rtpmap", n); @@ -603,11 +614,12 @@ WebRTCSession::havePlugins(bool isVideo, std::string *errorMessage) } bool -WebRTCSession::createOffer(bool isVideo) +WebRTCSession::createOffer(CallType callType) { isOffering_ = true; - isVideo_ = isVideo; + callType_ = callType; isRemoteVideoRecvOnly_ = false; + isRemoteVideoSendOnly_ = false; videoItem_ = nullptr; haveAudioStream_ = false; haveVideoStream_ = false; @@ -630,8 +642,10 @@ WebRTCSession::acceptOffer(const std::string &sdp) if (state_ != State::DISCONNECTED) return false; + callType_ = webrtc::CallType::VOICE; isOffering_ = false; isRemoteVideoRecvOnly_ = false; + isRemoteVideoSendOnly_ = false; videoItem_ = nullptr; haveAudioStream_ = false; haveVideoStream_ = false; @@ -645,7 +659,8 @@ WebRTCSession::acceptOffer(const std::string &sdp) int opusPayloadType; bool recvOnly; - if (getMediaAttributes(offer->sdp, "audio", "opus", opusPayloadType, recvOnly)) { + bool sendOnly; + if (getMediaAttributes(offer->sdp, "audio", "opus", opusPayloadType, recvOnly, sendOnly)) { if (opusPayloadType == -1) { nhlog::ui()->error("WebRTC: remote audio offer - no opus encoding"); gst_webrtc_session_description_free(offer); @@ -658,13 +673,18 @@ WebRTCSession::acceptOffer(const std::string &sdp) } int vp8PayloadType; - isVideo_ = - getMediaAttributes(offer->sdp, "video", "vp8", vp8PayloadType, isRemoteVideoRecvOnly_); - if (isVideo_ && vp8PayloadType == -1) { + bool isVideo = getMediaAttributes(offer->sdp, + "video", + "vp8", + vp8PayloadType, + isRemoteVideoRecvOnly_, + isRemoteVideoSendOnly_); + if (isVideo && vp8PayloadType == -1) { nhlog::ui()->error("WebRTC: remote video offer - no vp8 encoding"); gst_webrtc_session_description_free(offer); return false; } + callType_ = isVideo ? CallType::VIDEO : CallType::VOICE; if (!startPipeline(opusPayloadType, vp8PayloadType)) { gst_webrtc_session_description_free(offer); @@ -695,10 +715,14 @@ WebRTCSession::acceptAnswer(const std::string &sdp) return false; } - if (isVideo_) { + if (callType_ != CallType::VOICE) { int unused; - if (!getMediaAttributes( - answer->sdp, "video", "vp8", unused, isRemoteVideoRecvOnly_)) + if (!getMediaAttributes(answer->sdp, + "video", + "vp8", + unused, + isRemoteVideoRecvOnly_, + isRemoteVideoSendOnly_)) isRemoteVideoRecvOnly_ = true; } @@ -855,39 +879,59 @@ WebRTCSession::createPipeline(int opusPayloadType, int vp8PayloadType) return false; } - return isVideo_ ? addVideoPipeline(vp8PayloadType) : true; + return callType_ == CallType::VOICE || isRemoteVideoSendOnly_ + ? true + : addVideoPipeline(vp8PayloadType); } bool WebRTCSession::addVideoPipeline(int vp8PayloadType) { // allow incoming video calls despite localUser having no webcam - if (!devices_.haveCamera()) + if (callType_ == CallType::VIDEO && !devices_.haveCamera()) return !isOffering_; - std::pair resolution; - std::pair frameRate; - GstDevice *device = devices_.videoDevice(resolution, frameRate); - if (!device) - return false; + GstElement *source = nullptr; + GstCaps *caps = nullptr; + if (callType_ == CallType::VIDEO) { + std::pair resolution; + std::pair frameRate; + GstDevice *device = devices_.videoDevice(resolution, frameRate); + if (!device) + return false; + source = gst_device_create_element(device, nullptr); + caps = gst_caps_new_simple("video/x-raw", + "width", + G_TYPE_INT, + resolution.first, + "height", + G_TYPE_INT, + resolution.second, + "framerate", + GST_TYPE_FRACTION, + frameRate.first, + frameRate.second, + nullptr); + } else { + source = gst_element_factory_make("ximagesrc", nullptr); + if (!source) { + nhlog::ui()->error("WebRTC: failed to create ximagesrc"); + return false; + } + g_object_set(source, "use-damage", 0, nullptr); + g_object_set(source, "xid", 0, nullptr); + + int frameRate = ChatPage::instance()->userSettings()->screenShareFrameRate(); + caps = gst_caps_new_simple( + "video/x-raw", "framerate", GST_TYPE_FRACTION, frameRate, 1, nullptr); + nhlog::ui()->debug("WebRTC: screen share frame rate: {} fps", frameRate); + } - GstElement *source = gst_device_create_element(device, nullptr); GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr); GstElement *capsfilter = gst_element_factory_make("capsfilter", "camerafilter"); - GstCaps *caps = gst_caps_new_simple("video/x-raw", - "width", - G_TYPE_INT, - resolution.first, - "height", - G_TYPE_INT, - resolution.second, - "framerate", - GST_TYPE_FRACTION, - frameRate.first, - frameRate.second, - nullptr); g_object_set(capsfilter, "caps", caps, nullptr); gst_caps_unref(caps); + GstElement *tee = gst_element_factory_make("tee", "videosrctee"); GstElement *queue = gst_element_factory_make("queue", nullptr); GstElement *vp8enc = gst_element_factory_make("vp8enc", nullptr); @@ -938,14 +982,25 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType) gst_object_unref(webrtcbin); return false; } + + if (callType_ == CallType::SCREEN && + !ChatPage::instance()->userSettings()->screenShareRemoteVideo()) { + GArray *transceivers; + g_signal_emit_by_name(webrtcbin, "get-transceivers", &transceivers); + GstWebRTCRTPTransceiver *transceiver = + g_array_index(transceivers, GstWebRTCRTPTransceiver *, 1); + transceiver->direction = GST_WEBRTC_RTP_TRANSCEIVER_DIRECTION_SENDONLY; + g_array_unref(transceivers); + } + gst_object_unref(webrtcbin); return true; } bool -WebRTCSession::haveLocalVideo() const +WebRTCSession::haveLocalCamera() const { - if (isVideo_ && state_ >= State::INITIATED) { + if (callType_ == CallType::VIDEO && state_ >= State::INITIATED) { GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe_), "videosrctee"); if (tee) { gst_object_unref(tee); @@ -1008,9 +1063,10 @@ WebRTCSession::end() } webrtc_ = nullptr; - isVideo_ = false; + callType_ = CallType::VOICE; isOffering_ = false; isRemoteVideoRecvOnly_ = false; + isRemoteVideoSendOnly_ = false; videoItem_ = nullptr; insetSinkPad_ = nullptr; if (state_ != State::DISCONNECTED) @@ -1026,16 +1082,12 @@ WebRTCSession::havePlugins(bool, std::string *) } bool -WebRTCSession::haveLocalVideo() const +WebRTCSession::haveLocalCamera() const { return false; } -bool -WebRTCSession::createOffer(bool) -{ - return false; -} +bool WebRTCSession::createOffer(webrtc::CallType) { return false; } bool WebRTCSession::acceptOffer(const std::string &) diff --git a/src/WebRTCSession.h b/src/WebRTCSession.h index 0fe8a864..64eac706 100644 --- a/src/WebRTCSession.h +++ b/src/WebRTCSession.h @@ -5,15 +5,23 @@ #include -#include "CallDevices.h" #include "mtx/events/voip.hpp" typedef struct _GstElement GstElement; +class CallDevices; class QQuickItem; namespace webrtc { Q_NAMESPACE +enum class CallType +{ + VOICE, + VIDEO, + SCREEN // localUser is sharing screen +}; +Q_ENUM_NS(CallType) + enum class State { DISCONNECTED, @@ -42,13 +50,14 @@ public: } bool havePlugins(bool isVideo, std::string *errorMessage = nullptr); + webrtc::CallType callType() const { return callType_; } webrtc::State state() const { return state_; } - bool isVideo() const { return isVideo_; } - bool haveLocalVideo() const; + bool haveLocalCamera() const; bool isOffering() const { return isOffering_; } bool isRemoteVideoRecvOnly() const { return isRemoteVideoRecvOnly_; } + bool isRemoteVideoSendOnly() const { return isRemoteVideoSendOnly_; } - bool createOffer(bool isVideo); + bool createOffer(webrtc::CallType); bool acceptOffer(const std::string &sdp); bool acceptAnswer(const std::string &sdp); void acceptICECandidates(const std::vector &); @@ -81,10 +90,11 @@ private: bool initialised_ = false; bool haveVoicePlugins_ = false; bool haveVideoPlugins_ = false; + webrtc::CallType callType_ = webrtc::CallType::VOICE; webrtc::State state_ = webrtc::State::DISCONNECTED; - bool isVideo_ = false; bool isOffering_ = false; bool isRemoteVideoRecvOnly_ = false; + bool isRemoteVideoSendOnly_ = false; QQuickItem *videoItem_ = nullptr; GstElement *pipe_ = nullptr; GstElement *webrtc_ = nullptr; From 3b26cf4ba37c0944a6a297968d40da309be82b69 Mon Sep 17 00:00:00 2001 From: trilene Date: Thu, 18 Feb 2021 16:53:30 -0500 Subject: [PATCH 02/11] Screen sharing (X11): add hide mouse cursor option --- resources/qml/voip/ScreenShare.qml | 11 +++++++++++ src/UserSettingsPage.cpp | 12 ++++++++++++ src/UserSettingsPage.h | 6 ++++++ src/WebRTCSession.cpp | 9 ++++++--- 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/resources/qml/voip/ScreenShare.qml b/resources/qml/voip/ScreenShare.qml index b21a26fd..331e1c11 100644 --- a/resources/qml/voip/ScreenShare.qml +++ b/resources/qml/voip/ScreenShare.qml @@ -14,6 +14,7 @@ Popup { frameRateCombo.currentIndex = frameRateCombo.find(Settings.screenShareFrameRate); remoteVideoCheckBox.checked = Settings.screenShareRemoteVideo; + hideCursorCheckBox.checked = Settings.screenShareHideCursor; } palette: colors @@ -55,6 +56,15 @@ Popup { ToolTip.visible: hovered } + CheckBox { + id: hideCursorCheckBox + + Layout.alignment: Qt.AlignLeft + Layout.leftMargin: 8 + Layout.rightMargin: 8 + text: qsTr("Hide mouse cursor") + } + RowLayout { Layout.margins: 8 @@ -70,6 +80,7 @@ Popup { Settings.microphone = micCombo.currentText; Settings.screenShareFrameRate = frameRateCombo.currentText; Settings.screenShareRemoteVideo = remoteVideoCheckBox.checked; + Settings.screenShareHideCursor = hideCursorCheckBox.checked; CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.SCREEN); close(); } diff --git a/src/UserSettingsPage.cpp b/src/UserSettingsPage.cpp index 186a03bb..765e1e81 100644 --- a/src/UserSettingsPage.cpp +++ b/src/UserSettingsPage.cpp @@ -115,6 +115,7 @@ UserSettings::load(std::optional profile) cameraFrameRate_ = settings.value("user/camera_frame_rate", QString()).toString(); screenShareFrameRate_ = settings.value("user/screen_share_frame_rate", 5).toInt(); screenShareRemoteVideo_ = settings.value("user/screen_share_remote_video", false).toBool(); + screenShareHideCursor_ = settings.value("user/screen_share_hide_cursor", false).toBool(); useStunServer_ = settings.value("user/use_stun_server", false).toBool(); if (profile) // set to "" if it's the default to maintain compatibility @@ -466,6 +467,16 @@ UserSettings::setScreenShareRemoteVideo(bool state) save(); } +void +UserSettings::setScreenShareHideCursor(bool state) +{ + if (state == screenShareHideCursor_) + return; + screenShareHideCursor_ = state; + emit screenShareHideCursorChanged(state); + save(); +} + void UserSettings::setProfile(QString profile) { @@ -617,6 +628,7 @@ UserSettings::save() settings.setValue("camera_frame_rate", cameraFrameRate_); settings.setValue("screen_share_frame_rate", screenShareFrameRate_); settings.setValue("screen_share_remote_video", screenShareRemoteVideo_); + settings.setValue("screen_share_hide_cursor", screenShareHideCursor_); settings.setValue("use_stun_server", useStunServer_); settings.setValue("currentProfile", profile_); diff --git a/src/UserSettingsPage.h b/src/UserSettingsPage.h index 4de9913a..6e00572a 100644 --- a/src/UserSettingsPage.h +++ b/src/UserSettingsPage.h @@ -90,6 +90,8 @@ class UserSettings : public QObject NOTIFY screenShareFrameRateChanged) Q_PROPERTY(bool screenShareRemoteVideo READ screenShareRemoteVideo WRITE setScreenShareRemoteVideo NOTIFY screenShareRemoteVideoChanged) + Q_PROPERTY(bool screenShareHideCursor READ screenShareHideCursor WRITE + setScreenShareHideCursor NOTIFY screenShareHideCursorChanged) Q_PROPERTY( bool useStunServer READ useStunServer WRITE setUseStunServer NOTIFY useStunServerChanged) Q_PROPERTY(bool shareKeysWithTrustedUsers READ shareKeysWithTrustedUsers WRITE @@ -149,6 +151,7 @@ public: void setCameraFrameRate(QString frameRate); void setScreenShareFrameRate(int frameRate); void setScreenShareRemoteVideo(bool state); + void setScreenShareHideCursor(bool state); void setUseStunServer(bool state); void setShareKeysWithTrustedUsers(bool state); void setProfile(QString profile); @@ -199,6 +202,7 @@ public: QString cameraFrameRate() const { return cameraFrameRate_; } int screenShareFrameRate() const { return screenShareFrameRate_; } bool screenShareRemoteVideo() const { return screenShareRemoteVideo_; } + bool screenShareHideCursor() const { return screenShareHideCursor_; } bool useStunServer() const { return useStunServer_; } bool shareKeysWithTrustedUsers() const { return shareKeysWithTrustedUsers_; } QString profile() const { return profile_; } @@ -239,6 +243,7 @@ signals: void cameraFrameRateChanged(QString frameRate); void screenShareFrameRateChanged(int frameRate); void screenShareRemoteVideoChanged(bool state); + void screenShareHideCursorChanged(bool state); void useStunServerChanged(bool state); void shareKeysWithTrustedUsersChanged(bool state); void profileChanged(QString profile); @@ -284,6 +289,7 @@ private: QString cameraFrameRate_; int screenShareFrameRate_; bool screenShareRemoteVideo_; + bool screenShareHideCursor_; bool useStunServer_; QString profile_; QString userId_; diff --git a/src/WebRTCSession.cpp b/src/WebRTCSession.cpp index 9c01ddc4..acd54b77 100644 --- a/src/WebRTCSession.cpp +++ b/src/WebRTCSession.cpp @@ -918,10 +918,13 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType) nhlog::ui()->error("WebRTC: failed to create ximagesrc"); return false; } - g_object_set(source, "use-damage", 0, nullptr); + g_object_set(source, "use-damage", FALSE, nullptr); g_object_set(source, "xid", 0, nullptr); - - int frameRate = ChatPage::instance()->userSettings()->screenShareFrameRate(); + auto settings = ChatPage::instance()->userSettings(); + g_object_set(source, "show-pointer", !settings->screenShareHideCursor(), nullptr); + nhlog::ui()->debug("WebRTC: screen share hide mouse cursor: {}", + settings->screenShareHideCursor()); + int frameRate = settings->screenShareFrameRate(); caps = gst_caps_new_simple( "video/x-raw", "framerate", GST_TYPE_FRACTION, frameRate, 1, nullptr); nhlog::ui()->debug("WebRTC: screen share frame rate: {} fps", frameRate); From 8ccd2abc6af543a634817b891a955b8729cba659 Mon Sep 17 00:00:00 2001 From: trilene Date: Sat, 20 Feb 2021 11:26:53 -0500 Subject: [PATCH 03/11] Screen sharing (X11): support picture-in-picture --- resources/qml/voip/ScreenShare.qml | 12 ++ src/UserSettingsPage.cpp | 12 ++ src/UserSettingsPage.h | 6 + src/WebRTCSession.cpp | 275 ++++++++++++++++++----------- src/WebRTCSession.h | 1 + 5 files changed, 202 insertions(+), 104 deletions(-) diff --git a/resources/qml/voip/ScreenShare.qml b/resources/qml/voip/ScreenShare.qml index 331e1c11..cb70a36c 100644 --- a/resources/qml/voip/ScreenShare.qml +++ b/resources/qml/voip/ScreenShare.qml @@ -13,6 +13,7 @@ Popup { anchors.centerIn = parent; frameRateCombo.currentIndex = frameRateCombo.find(Settings.screenShareFrameRate); + pipCheckBox.checked = Settings.screenSharePiP; remoteVideoCheckBox.checked = Settings.screenShareRemoteVideo; hideCursorCheckBox.checked = Settings.screenShareHideCursor; } @@ -45,6 +46,16 @@ Popup { } + CheckBox { + id: pipCheckBox + + visible: CallManager.cameras.length > 0 + Layout.alignment: Qt.AlignLeft + Layout.leftMargin: 8 + Layout.rightMargin: 8 + text: qsTr("Include your camera picture-in-picture") + } + CheckBox { id: remoteVideoCheckBox @@ -79,6 +90,7 @@ Popup { if (buttonLayout.validateMic()) { Settings.microphone = micCombo.currentText; Settings.screenShareFrameRate = frameRateCombo.currentText; + Settings.screenSharePiP = pipCheckBox.checked; Settings.screenShareRemoteVideo = remoteVideoCheckBox.checked; Settings.screenShareHideCursor = hideCursorCheckBox.checked; CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.SCREEN); diff --git a/src/UserSettingsPage.cpp b/src/UserSettingsPage.cpp index 765e1e81..1dcf9f63 100644 --- a/src/UserSettingsPage.cpp +++ b/src/UserSettingsPage.cpp @@ -114,6 +114,7 @@ UserSettings::load(std::optional profile) cameraResolution_ = settings.value("user/camera_resolution", QString()).toString(); cameraFrameRate_ = settings.value("user/camera_frame_rate", QString()).toString(); screenShareFrameRate_ = settings.value("user/screen_share_frame_rate", 5).toInt(); + screenSharePiP_ = settings.value("user/screen_share_pip", true).toBool(); screenShareRemoteVideo_ = settings.value("user/screen_share_remote_video", false).toBool(); screenShareHideCursor_ = settings.value("user/screen_share_hide_cursor", false).toBool(); useStunServer_ = settings.value("user/use_stun_server", false).toBool(); @@ -457,6 +458,16 @@ UserSettings::setScreenShareFrameRate(int frameRate) save(); } +void +UserSettings::setScreenSharePiP(bool state) +{ + if (state == screenSharePiP_) + return; + screenSharePiP_ = state; + emit screenSharePiPChanged(state); + save(); +} + void UserSettings::setScreenShareRemoteVideo(bool state) { @@ -627,6 +638,7 @@ UserSettings::save() settings.setValue("camera_resolution", cameraResolution_); settings.setValue("camera_frame_rate", cameraFrameRate_); settings.setValue("screen_share_frame_rate", screenShareFrameRate_); + settings.setValue("screen_share_pip", screenSharePiP_); settings.setValue("screen_share_remote_video", screenShareRemoteVideo_); settings.setValue("screen_share_hide_cursor", screenShareHideCursor_); settings.setValue("use_stun_server", useStunServer_); diff --git a/src/UserSettingsPage.h b/src/UserSettingsPage.h index 6e00572a..dfb5acf4 100644 --- a/src/UserSettingsPage.h +++ b/src/UserSettingsPage.h @@ -88,6 +88,8 @@ class UserSettings : public QObject cameraFrameRateChanged) Q_PROPERTY(int screenShareFrameRate READ screenShareFrameRate WRITE setScreenShareFrameRate NOTIFY screenShareFrameRateChanged) + Q_PROPERTY(bool screenSharePiP READ screenSharePiP WRITE setScreenSharePiP NOTIFY + screenSharePiPChanged) Q_PROPERTY(bool screenShareRemoteVideo READ screenShareRemoteVideo WRITE setScreenShareRemoteVideo NOTIFY screenShareRemoteVideoChanged) Q_PROPERTY(bool screenShareHideCursor READ screenShareHideCursor WRITE @@ -150,6 +152,7 @@ public: void setCameraResolution(QString resolution); void setCameraFrameRate(QString frameRate); void setScreenShareFrameRate(int frameRate); + void setScreenSharePiP(bool state); void setScreenShareRemoteVideo(bool state); void setScreenShareHideCursor(bool state); void setUseStunServer(bool state); @@ -201,6 +204,7 @@ public: QString cameraResolution() const { return cameraResolution_; } QString cameraFrameRate() const { return cameraFrameRate_; } int screenShareFrameRate() const { return screenShareFrameRate_; } + bool screenSharePiP() const { return screenSharePiP_; } bool screenShareRemoteVideo() const { return screenShareRemoteVideo_; } bool screenShareHideCursor() const { return screenShareHideCursor_; } bool useStunServer() const { return useStunServer_; } @@ -242,6 +246,7 @@ signals: void cameraResolutionChanged(QString resolution); void cameraFrameRateChanged(QString frameRate); void screenShareFrameRateChanged(int frameRate); + void screenSharePiPChanged(bool state); void screenShareRemoteVideoChanged(bool state); void screenShareHideCursorChanged(bool state); void useStunServerChanged(bool state); @@ -288,6 +293,7 @@ private: QString cameraResolution_; QString cameraFrameRate_; int screenShareFrameRate_; + bool screenSharePiP_; bool screenShareRemoteVideo_; bool screenShareHideCursor_; bool useStunServer_; diff --git a/src/WebRTCSession.cpp b/src/WebRTCSession.cpp index acd54b77..2281145b 100644 --- a/src/WebRTCSession.cpp +++ b/src/WebRTCSession.cpp @@ -89,9 +89,10 @@ namespace { std::string localsdp_; std::vector localcandidates_; -bool haveAudioStream_; -bool haveVideoStream_; -GstPad *insetSinkPad_ = nullptr; +bool haveAudioStream_ = false; +bool haveVideoStream_ = false; +GstPad *localPiPSinkPad_ = nullptr; +GstPad *remotePiPSinkPad_ = nullptr; gboolean newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer user_data) @@ -364,6 +365,7 @@ newVideoSinkChain(GstElement *pipe) GstElement *glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr); GstElement *qmlglsink = gst_element_factory_make("qmlglsink", nullptr); GstElement *glsinkbin = gst_element_factory_make("glsinkbin", nullptr); + g_object_set(compositor, "background", 1, nullptr); g_object_set(qmlglsink, "widget", WebRTCSession::instance().getVideoItem(), nullptr); g_object_set(glsinkbin, "sink", qmlglsink, nullptr); gst_bin_add_many( @@ -390,8 +392,9 @@ getResolution(GstPad *pad) } void -addCameraView(GstElement *pipe, const std::pair &videoCallSize) +addLocalPiP(GstElement *pipe, const std::pair &videoCallSize) { + // embed localUser's camera into received video GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe), "videosrctee"); if (!tee) return; @@ -407,26 +410,56 @@ addCameraView(GstElement *pipe, const std::pair &videoCallSize) GstElement *camerafilter = gst_bin_get_by_name(GST_BIN(pipe), "camerafilter"); GstPad *filtersinkpad = gst_element_get_static_pad(camerafilter, "sink"); auto cameraResolution = getResolution(filtersinkpad); - int insetWidth = videoCallSize.first / 4; - int insetHeight = - static_cast(cameraResolution.second) / cameraResolution.first * insetWidth; - nhlog::ui()->debug("WebRTC: picture-in-picture size: {}x{}", insetWidth, insetHeight); + int pipWidth = videoCallSize.first / 4; + int pipHeight = + static_cast(cameraResolution.second) / cameraResolution.first * pipWidth; + nhlog::ui()->debug("WebRTC: local picture-in-picture: {}x{}", pipWidth, pipHeight); gst_object_unref(filtersinkpad); gst_object_unref(camerafilter); GstPad *camerapad = gst_element_get_static_pad(videorate, "src"); GstElement *compositor = gst_bin_get_by_name(GST_BIN(pipe), "compositor"); - insetSinkPad_ = gst_element_get_request_pad(compositor, "sink_%u"); - g_object_set(insetSinkPad_, "zorder", 2, nullptr); - g_object_set(insetSinkPad_, "width", insetWidth, "height", insetHeight, nullptr); + localPiPSinkPad_ = gst_element_get_request_pad(compositor, "sink_%u"); + g_object_set(localPiPSinkPad_, "zorder", 2, nullptr); + g_object_set(localPiPSinkPad_, "width", pipWidth, "height", pipHeight, nullptr); gint offset = videoCallSize.first / 80; - g_object_set(insetSinkPad_, "xpos", offset, "ypos", offset, nullptr); - if (GST_PAD_LINK_FAILED(gst_pad_link(camerapad, insetSinkPad_))) - nhlog::ui()->error("WebRTC: failed to link camera view chain"); + g_object_set(localPiPSinkPad_, "xpos", offset, "ypos", offset, nullptr); + if (GST_PAD_LINK_FAILED(gst_pad_link(camerapad, localPiPSinkPad_))) + nhlog::ui()->error("WebRTC: failed to link local PiP elements"); gst_object_unref(camerapad); gst_object_unref(compositor); } +void +addRemotePiP(GstElement *pipe) +{ + // embed localUser's camera into screen image being shared + if (remotePiPSinkPad_) { + GstElement *screen = gst_bin_get_by_name(GST_BIN(pipe), "screenshare"); + GstPad *srcpad = gst_element_get_static_pad(screen, "src"); + auto resolution = getResolution(srcpad); + nhlog::ui()->debug( + "WebRTC: screen share: {}x{}", resolution.first, resolution.second); + gst_object_unref(srcpad); + gst_object_unref(screen); + + int pipWidth = resolution.first / 5; + int pipHeight = + static_cast(resolution.second) / resolution.first * pipWidth; + nhlog::ui()->debug( + "WebRTC: screen share picture-in-picture: {}x{}", pipWidth, pipHeight); + gint offset = resolution.first / 100; + g_object_set(remotePiPSinkPad_, "zorder", 2, nullptr); + g_object_set(remotePiPSinkPad_, "width", pipWidth, "height", pipHeight, nullptr); + g_object_set(remotePiPSinkPad_, + "xpos", + resolution.first - pipWidth - offset, + "ypos", + resolution.second - pipHeight - offset, + nullptr); + } +} + void linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe) { @@ -463,7 +496,7 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe) videoCallSize.first, videoCallSize.second); if (session->callType() == CallType::VIDEO) - addCameraView(pipe, videoCallSize); + addLocalPiP(pipe, videoCallSize); } else { g_free(mediaType); nhlog::ui()->error("WebRTC: unknown pad type: {}", GST_PAD_NAME(newpad)); @@ -485,6 +518,7 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe) keyFrameRequestData_.timerid = g_timeout_add_seconds(3, testPacketLoss, nullptr); } + addRemotePiP(pipe); } } gst_object_unref(queuepad); @@ -616,16 +650,9 @@ WebRTCSession::havePlugins(bool isVideo, std::string *errorMessage) bool WebRTCSession::createOffer(CallType callType) { - isOffering_ = true; - callType_ = callType; - isRemoteVideoRecvOnly_ = false; - isRemoteVideoSendOnly_ = false; - videoItem_ = nullptr; - haveAudioStream_ = false; - haveVideoStream_ = false; - insetSinkPad_ = nullptr; - localsdp_.clear(); - localcandidates_.clear(); + clear(); + isOffering_ = true; + callType_ = callType; // opus and vp8 rtp payload types must be defined dynamically // therefore from the range [96-127] @@ -642,17 +669,7 @@ WebRTCSession::acceptOffer(const std::string &sdp) if (state_ != State::DISCONNECTED) return false; - callType_ = webrtc::CallType::VOICE; - isOffering_ = false; - isRemoteVideoRecvOnly_ = false; - isRemoteVideoSendOnly_ = false; - videoItem_ = nullptr; - haveAudioStream_ = false; - haveVideoStream_ = false; - insetSinkPad_ = nullptr; - localsdp_.clear(); - localcandidates_.clear(); - + clear(); GstWebRTCSessionDescription *offer = parseSDP(sdp, GST_WEBRTC_SDP_TYPE_OFFER); if (!offer) return false; @@ -891,51 +908,106 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType) if (callType_ == CallType::VIDEO && !devices_.haveCamera()) return !isOffering_; - GstElement *source = nullptr; - GstCaps *caps = nullptr; - if (callType_ == CallType::VIDEO) { + auto settings = ChatPage::instance()->userSettings(); + if (callType_ == CallType::SCREEN && settings->screenSharePiP() && !devices_.haveCamera()) + return false; + + GstElement *camerafilter = nullptr; + GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr); + GstElement *tee = gst_element_factory_make("tee", "videosrctee"); + gst_bin_add_many(GST_BIN(pipe_), videoconvert, tee, nullptr); + if (callType_ == CallType::VIDEO || settings->screenSharePiP()) { std::pair resolution; std::pair frameRate; GstDevice *device = devices_.videoDevice(resolution, frameRate); if (!device) return false; - source = gst_device_create_element(device, nullptr); - caps = gst_caps_new_simple("video/x-raw", - "width", - G_TYPE_INT, - resolution.first, - "height", - G_TYPE_INT, - resolution.second, - "framerate", - GST_TYPE_FRACTION, - frameRate.first, - frameRate.second, - nullptr); - } else { - source = gst_element_factory_make("ximagesrc", nullptr); - if (!source) { + + GstElement *camera = gst_device_create_element(device, nullptr); + GstCaps *caps = gst_caps_new_simple("video/x-raw", + "width", + G_TYPE_INT, + resolution.first, + "height", + G_TYPE_INT, + resolution.second, + "framerate", + GST_TYPE_FRACTION, + frameRate.first, + frameRate.second, + nullptr); + camerafilter = gst_element_factory_make("capsfilter", "camerafilter"); + g_object_set(camerafilter, "caps", caps, nullptr); + gst_caps_unref(caps); + + gst_bin_add_many(GST_BIN(pipe_), camera, camerafilter, nullptr); + if (!gst_element_link_many(camera, videoconvert, camerafilter, nullptr)) { + nhlog::ui()->error("WebRTC: failed to link camera elements"); + return false; + } + if (callType_ == CallType::VIDEO && !gst_element_link(camerafilter, tee)) { + nhlog::ui()->error("WebRTC: failed to link camerafilter -> tee"); + return false; + } + } + + if (callType_ == CallType::SCREEN) { + nhlog::ui()->debug("WebRTC: screen share frame rate: {} fps", + settings->screenShareFrameRate()); + nhlog::ui()->debug("WebRTC: screen share picture-in-picture: {}", + settings->screenSharePiP()); + nhlog::ui()->debug("WebRTC: screen share request remote camera: {}", + settings->screenShareRemoteVideo()); + nhlog::ui()->debug("WebRTC: screen share hide mouse cursor: {}", + settings->screenShareHideCursor()); + + GstElement *ximagesrc = gst_element_factory_make("ximagesrc", "screenshare"); + if (!ximagesrc) { nhlog::ui()->error("WebRTC: failed to create ximagesrc"); return false; } - g_object_set(source, "use-damage", FALSE, nullptr); - g_object_set(source, "xid", 0, nullptr); - auto settings = ChatPage::instance()->userSettings(); - g_object_set(source, "show-pointer", !settings->screenShareHideCursor(), nullptr); - nhlog::ui()->debug("WebRTC: screen share hide mouse cursor: {}", - settings->screenShareHideCursor()); - int frameRate = settings->screenShareFrameRate(); - caps = gst_caps_new_simple( - "video/x-raw", "framerate", GST_TYPE_FRACTION, frameRate, 1, nullptr); - nhlog::ui()->debug("WebRTC: screen share frame rate: {} fps", frameRate); + g_object_set(ximagesrc, "use-damage", FALSE, nullptr); + g_object_set(ximagesrc, "xid", 0, nullptr); + g_object_set( + ximagesrc, "show-pointer", !settings->screenShareHideCursor(), nullptr); + + GstCaps *caps = gst_caps_new_simple("video/x-raw", + "framerate", + GST_TYPE_FRACTION, + settings->screenShareFrameRate(), + 1, + nullptr); + GstElement *capsfilter = gst_element_factory_make("capsfilter", nullptr); + g_object_set(capsfilter, "caps", caps, nullptr); + gst_caps_unref(caps); + gst_bin_add_many(GST_BIN(pipe_), ximagesrc, capsfilter, nullptr); + + if (settings->screenSharePiP()) { + GstElement *compositor = gst_element_factory_make("compositor", nullptr); + g_object_set(compositor, "background", 1, nullptr); + gst_bin_add(GST_BIN(pipe_), compositor); + if (!gst_element_link_many( + ximagesrc, compositor, capsfilter, tee, nullptr)) { + nhlog::ui()->error("WebRTC: failed to link screen share elements"); + return false; + } + + GstPad *srcpad = gst_element_get_static_pad(camerafilter, "src"); + remotePiPSinkPad_ = gst_element_get_request_pad(compositor, "sink_%u"); + if (GST_PAD_LINK_FAILED(gst_pad_link(srcpad, remotePiPSinkPad_))) { + nhlog::ui()->error( + "WebRTC: failed to link camerafilter -> compositor"); + gst_object_unref(srcpad); + return false; + } + gst_object_unref(srcpad); + } else if (!gst_element_link_many( + ximagesrc, videoconvert, capsfilter, tee, nullptr)) { + nhlog::ui()->error("WebRTC: failed to link screen share elements"); + return false; + } } - GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr); - GstElement *capsfilter = gst_element_factory_make("capsfilter", "camerafilter"); - g_object_set(capsfilter, "caps", caps, nullptr); - gst_caps_unref(caps); - - GstElement *tee = gst_element_factory_make("tee", "videosrctee"); GstElement *queue = gst_element_factory_make("queue", nullptr); GstElement *vp8enc = gst_element_factory_make("vp8enc", nullptr); g_object_set(vp8enc, "deadline", 1, nullptr); @@ -957,31 +1029,13 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType) g_object_set(rtpcapsfilter, "caps", rtpcaps, nullptr); gst_caps_unref(rtpcaps); - gst_bin_add_many(GST_BIN(pipe_), - source, - videoconvert, - capsfilter, - tee, - queue, - vp8enc, - rtpvp8pay, - rtpqueue, - rtpcapsfilter, - nullptr); + gst_bin_add_many( + GST_BIN(pipe_), queue, vp8enc, rtpvp8pay, rtpqueue, rtpcapsfilter, nullptr); GstElement *webrtcbin = gst_bin_get_by_name(GST_BIN(pipe_), "webrtcbin"); - if (!gst_element_link_many(source, - videoconvert, - capsfilter, - tee, - queue, - vp8enc, - rtpvp8pay, - rtpqueue, - rtpcapsfilter, - webrtcbin, - nullptr)) { - nhlog::ui()->error("WebRTC: failed to link video pipeline elements"); + if (!gst_element_link_many( + tee, queue, vp8enc, rtpvp8pay, rtpqueue, rtpcapsfilter, webrtcbin, nullptr)) { + nhlog::ui()->error("WebRTC: failed to link rtp video elements"); gst_object_unref(webrtcbin); return false; } @@ -1043,13 +1097,32 @@ WebRTCSession::toggleMicMute() void WebRTCSession::toggleCameraView() { - if (insetSinkPad_) { + if (localPiPSinkPad_) { guint zorder; - g_object_get(insetSinkPad_, "zorder", &zorder, nullptr); - g_object_set(insetSinkPad_, "zorder", zorder ? 0 : 2, nullptr); + g_object_get(localPiPSinkPad_, "zorder", &zorder, nullptr); + g_object_set(localPiPSinkPad_, "zorder", zorder ? 0 : 2, nullptr); } } +void +WebRTCSession::clear() +{ + callType_ = webrtc::CallType::VOICE; + isOffering_ = false; + isRemoteVideoRecvOnly_ = false; + isRemoteVideoSendOnly_ = false; + videoItem_ = nullptr; + pipe_ = nullptr; + webrtc_ = nullptr; + busWatchId_ = 0; + haveAudioStream_ = false; + haveVideoStream_ = false; + localPiPSinkPad_ = nullptr; + remotePiPSinkPad_ = nullptr; + localsdp_.clear(); + localcandidates_.clear(); +} + void WebRTCSession::end() { @@ -1065,13 +1138,7 @@ WebRTCSession::end() } } - webrtc_ = nullptr; - callType_ = CallType::VOICE; - isOffering_ = false; - isRemoteVideoRecvOnly_ = false; - isRemoteVideoSendOnly_ = false; - videoItem_ = nullptr; - insetSinkPad_ = nullptr; + clear(); if (state_ != State::DISCONNECTED) emit stateChanged(State::DISCONNECTED); } diff --git a/src/WebRTCSession.h b/src/WebRTCSession.h index 64eac706..24ae9a17 100644 --- a/src/WebRTCSession.h +++ b/src/WebRTCSession.h @@ -105,6 +105,7 @@ private: bool startPipeline(int opusPayloadType, int vp8PayloadType); bool createPipeline(int opusPayloadType, int vp8PayloadType); bool addVideoPipeline(int vp8PayloadType); + void clear(); public: WebRTCSession(WebRTCSession const &) = delete; From c461c0aac03d2ee658ad0b449b0da46c929d4401 Mon Sep 17 00:00:00 2001 From: trilene Date: Sat, 20 Feb 2021 17:14:22 -0500 Subject: [PATCH 04/11] Require GStreamer 1.18 for voip support --- CMakeLists.txt | 4 +-- resources/qml/MessageInput.qml | 1 - resources/qml/voip/CallInviteBar.qml | 1 - src/CallDevices.cpp | 44 -------------------------- src/CallDevices.h | 1 - src/CallManager.cpp | 1 - src/CallManager.h | 1 - src/UserSettingsPage.cpp | 1 - src/WebRTCSession.cpp | 46 +--------------------------- 9 files changed, 3 insertions(+), 97 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 72190947..7f695bc5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -446,11 +446,11 @@ else() endif() include(FindPkgConfig) -pkg_check_modules(GSTREAMER IMPORTED_TARGET gstreamer-sdp-1.0>=1.16 gstreamer-webrtc-1.0>=1.16) +pkg_check_modules(GSTREAMER IMPORTED_TARGET gstreamer-sdp-1.0>=1.18 gstreamer-webrtc-1.0>=1.18) if (TARGET PkgConfig::GSTREAMER) add_feature_info(voip ON "GStreamer found. Call support is enabled automatically.") else() - add_feature_info(voip OFF "GStreamer could not be found on your system. As a consequence call support has been disabled. If you don't want that, make sure gstreamer-sdp-1.0>=1.16 gstreamer-webrtc-1.0>=1.16 can be found via pkgconfig.") + add_feature_info(voip OFF "GStreamer could not be found on your system. As a consequence call support has been disabled. If you don't want that, make sure gstreamer-sdp-1.0>=1.18 gstreamer-webrtc-1.0>=1.18 can be found via pkgconfig.") endif() # single instance functionality diff --git a/resources/qml/MessageInput.qml b/resources/qml/MessageInput.qml index b5c96660..863b09f7 100644 --- a/resources/qml/MessageInput.qml +++ b/resources/qml/MessageInput.qml @@ -44,7 +44,6 @@ Rectangle { } else if (CallManager.isOnCall) { CallManager.hangUp(); } else { - CallManager.refreshDevices(); var dialog = placeCallDialog.createObject(timelineRoot); dialog.open(); } diff --git a/resources/qml/voip/CallInviteBar.qml b/resources/qml/voip/CallInviteBar.qml index bf630e9e..7fc8cd05 100644 --- a/resources/qml/voip/CallInviteBar.qml +++ b/resources/qml/voip/CallInviteBar.qml @@ -75,7 +75,6 @@ Rectangle { ToolTip.visible: hovered ToolTip.text: qsTr("Devices") onClicked: { - CallManager.refreshDevices(); var dialog = devicesDialog.createObject(timelineRoot); dialog.open(); } diff --git a/src/CallDevices.cpp b/src/CallDevices.cpp index 0b9809e5..f182c133 100644 --- a/src/CallDevices.cpp +++ b/src/CallDevices.cpp @@ -152,7 +152,6 @@ addDevice(GstDevice *device) setDefaultDevice(true); } -#if GST_CHECK_VERSION(1, 18, 0) template bool removeDevice(T &sources, GstDevice *device, bool changed) @@ -212,7 +211,6 @@ newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer user_data G_G } return TRUE; } -#endif template std::vector @@ -257,7 +255,6 @@ tokenise(std::string_view str, char delim) void CallDevices::init() { -#if GST_CHECK_VERSION(1, 18, 0) static GstDeviceMonitor *monitor = nullptr; if (!monitor) { monitor = gst_device_monitor_new(); @@ -278,43 +275,6 @@ CallDevices::init() return; } } -#endif -} - -void -CallDevices::refresh() -{ -#if !GST_CHECK_VERSION(1, 18, 0) - - static GstDeviceMonitor *monitor = nullptr; - if (!monitor) { - monitor = gst_device_monitor_new(); - GstCaps *caps = gst_caps_new_empty_simple("audio/x-raw"); - gst_device_monitor_add_filter(monitor, "Audio/Source", caps); - gst_device_monitor_add_filter(monitor, "Audio/Duplex", caps); - gst_caps_unref(caps); - caps = gst_caps_new_empty_simple("video/x-raw"); - gst_device_monitor_add_filter(monitor, "Video/Source", caps); - gst_device_monitor_add_filter(monitor, "Video/Duplex", caps); - gst_caps_unref(caps); - } - - auto clearDevices = [](auto &sources) { - std::for_each( - sources.begin(), sources.end(), [](auto &s) { gst_object_unref(s.device); }); - sources.clear(); - }; - clearDevices(audioSources_); - clearDevices(videoSources_); - - GList *devices = gst_device_monitor_get_devices(monitor); - if (devices) { - for (GList *l = devices; l != nullptr; l = l->next) - addDevice(GST_DEVICE_CAST(l->data)); - g_list_free(devices); - } - emit devicesChanged(); -#endif } bool @@ -400,10 +360,6 @@ CallDevices::videoDevice(std::pair &resolution, std::pair &f #else -void -CallDevices::refresh() -{} - bool CallDevices::haveMic() const { diff --git a/src/CallDevices.h b/src/CallDevices.h index 2b4129f1..6d9e18c6 100644 --- a/src/CallDevices.h +++ b/src/CallDevices.h @@ -19,7 +19,6 @@ public: return instance; } - void refresh(); bool haveMic() const; bool haveCamera() const; std::vector names(bool isVideo, const std::string &defaultDevice) const; diff --git a/src/CallManager.cpp b/src/CallManager.cpp index 51bb7b33..b920cfed 100644 --- a/src/CallManager.cpp +++ b/src/CallManager.cpp @@ -290,7 +290,6 @@ CallManager::handleEvent(const RoomEvent &callInviteEvent) haveCallInvite_ = true; callType_ = isVideo ? CallType::VIDEO : CallType::VOICE; inviteSDP_ = callInviteEvent.content.sdp; - CallDevices::instance().refresh(); emit newInviteState(); } diff --git a/src/CallManager.h b/src/CallManager.h index ed745b5b..cfc50481 100644 --- a/src/CallManager.h +++ b/src/CallManager.h @@ -59,7 +59,6 @@ public: public slots: void sendInvite(const QString &roomid, webrtc::CallType); void syncEvent(const mtx::events::collections::TimelineEvents &event); - void refreshDevices() { CallDevices::instance().refresh(); } void toggleMicMute(); void toggleCameraView() { session_.toggleCameraView(); } void acceptInvite(); diff --git a/src/UserSettingsPage.cpp b/src/UserSettingsPage.cpp index 1dcf9f63..7410e35b 100644 --- a/src/UserSettingsPage.cpp +++ b/src/UserSettingsPage.cpp @@ -1288,7 +1288,6 @@ UserSettingsPage::showEvent(QShowEvent *) timelineMaxWidthSpin_->setValue(settings_->timelineMaxWidth()); privacyScreenTimeout_->setValue(settings_->privacyScreenTimeout()); - CallDevices::instance().refresh(); auto mics = CallDevices::instance().names(false, settings_->microphone().toStdString()); microphoneCombo_->clear(); for (const auto &m : mics) diff --git a/src/WebRTCSession.cpp b/src/WebRTCSession.cpp index 2281145b..c3a28117 100644 --- a/src/WebRTCSession.cpp +++ b/src/WebRTCSession.cpp @@ -174,7 +174,6 @@ createAnswer(GstPromise *promise, gpointer webrtc) g_signal_emit_by_name(webrtc, "create-answer", nullptr, promise); } -#if GST_CHECK_VERSION(1, 18, 0) void iceGatheringStateChanged(GstElement *webrtc, GParamSpec *pspec G_GNUC_UNUSED, @@ -194,23 +193,6 @@ iceGatheringStateChanged(GstElement *webrtc, } } -#else - -gboolean -onICEGatheringCompletion(gpointer timerid) -{ - *(guint *)(timerid) = 0; - if (WebRTCSession::instance().isOffering()) { - emit WebRTCSession::instance().offerCreated(localsdp_, localcandidates_); - emit WebRTCSession::instance().stateChanged(State::OFFERSENT); - } else { - emit WebRTCSession::instance().answerCreated(localsdp_, localcandidates_); - emit WebRTCSession::instance().stateChanged(State::ANSWERSENT); - } - return FALSE; -} -#endif - void addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED, guint mlineIndex, @@ -218,28 +200,7 @@ addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED, gpointer G_GNUC_UNUSED) { nhlog::ui()->debug("WebRTC: local candidate: (m-line:{}):{}", mlineIndex, candidate); - -#if GST_CHECK_VERSION(1, 18, 0) localcandidates_.push_back({std::string() /*max-bundle*/, (uint16_t)mlineIndex, candidate}); - return; -#else - if (WebRTCSession::instance().state() >= State::OFFERSENT) { - emit WebRTCSession::instance().newICECandidate( - {std::string() /*max-bundle*/, (uint16_t)mlineIndex, candidate}); - return; - } - - localcandidates_.push_back({std::string() /*max-bundle*/, (uint16_t)mlineIndex, candidate}); - - // GStreamer v1.16: webrtcbin's notify::ice-gathering-state triggers - // GST_WEBRTC_ICE_GATHERING_STATE_COMPLETE too early. Fixed in v1.18. - // Use a 1s timeout in the meantime - static guint timerid = 0; - if (timerid) - g_source_remove(timerid); - - timerid = g_timeout_add(1000, onICEGatheringCompletion, &timerid); -#endif } void @@ -328,7 +289,6 @@ testPacketLoss(gpointer G_GNUC_UNUSED) return FALSE; } -#if GST_CHECK_VERSION(1, 18, 0) void setWaitForKeyFrame(GstBin *decodebin G_GNUC_UNUSED, GstElement *element, gpointer G_GNUC_UNUSED) { @@ -337,7 +297,6 @@ setWaitForKeyFrame(GstBin *decodebin G_GNUC_UNUSED, GstElement *element, gpointe "rtpvp8depay")) g_object_set(element, "wait-for-keyframe", TRUE, nullptr); } -#endif GstElement * newAudioSinkChain(GstElement *pipe) @@ -537,9 +496,7 @@ addDecodeBin(GstElement *webrtc G_GNUC_UNUSED, GstPad *newpad, GstElement *pipe) // hardware decoding needs investigation; eg rendering fails if vaapi plugin installed g_object_set(decodebin, "force-sw-decoders", TRUE, nullptr); g_signal_connect(decodebin, "pad-added", G_CALLBACK(linkNewPad), pipe); -#if GST_CHECK_VERSION(1, 18, 0) g_signal_connect(decodebin, "element-added", G_CALLBACK(setWaitForKeyFrame), nullptr); -#endif gst_bin_add(GST_BIN(pipe), decodebin); gst_element_sync_state_with_parent(decodebin); GstPad *sinkpad = gst_element_get_static_pad(decodebin, "sink"); @@ -810,11 +767,10 @@ WebRTCSession::startPipeline(int opusPayloadType, int vp8PayloadType) gst_element_set_state(pipe_, GST_STATE_READY); g_signal_connect(webrtc_, "pad-added", G_CALLBACK(addDecodeBin), pipe_); -#if GST_CHECK_VERSION(1, 18, 0) // capture ICE gathering completion g_signal_connect( webrtc_, "notify::ice-gathering-state", G_CALLBACK(iceGatheringStateChanged), nullptr); -#endif + // webrtcbin lifetime is the same as that of the pipeline gst_object_unref(webrtc_); From e8e88e7d79842f1636905f3ce86169bb950767f4 Mon Sep 17 00:00:00 2001 From: trilene Date: Sat, 20 Feb 2021 17:33:04 -0500 Subject: [PATCH 05/11] Refine X11 test --- src/CallManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CallManager.cpp b/src/CallManager.cpp index b920cfed..de04ed3b 100644 --- a/src/CallManager.cpp +++ b/src/CallManager.cpp @@ -402,7 +402,7 @@ CallManager::callsSupported() bool CallManager::screenShareSupported() { - return std::getenv("DISPLAY") != nullptr; + return std::getenv("DISPLAY") && !std::getenv("WAYLAND_DISPLAY"); } bool From 70c77cdc44698104c11c222dc630bbcc26a27f3e Mon Sep 17 00:00:00 2001 From: trilene Date: Sun, 21 Feb 2021 16:30:10 -0500 Subject: [PATCH 06/11] Display screen sharing content locally --- resources/qml/TimelineView.qml | 2 +- resources/qml/voip/ActiveCallBar.qml | 10 +-- resources/qml/voip/ScreenShare.qml | 5 +- src/CallManager.cpp | 9 -- src/CallManager.h | 8 +- src/WebRTCSession.cpp | 120 +++++++++++++++++---------- src/WebRTCSession.h | 4 +- 7 files changed, 90 insertions(+), 68 deletions(-) diff --git a/resources/qml/TimelineView.qml b/resources/qml/TimelineView.qml index 0cd129da..07145c7a 100644 --- a/resources/qml/TimelineView.qml +++ b/resources/qml/TimelineView.qml @@ -249,7 +249,7 @@ Page { } Loader { - source: CallManager.isOnCall && CallManager.haveVideo ? "voip/VideoCall.qml" : "" + source: CallManager.isOnCall && CallManager.callType != CallType.VOICE ? "voip/VideoCall.qml" : "" onLoaded: TimelineManager.setVideoCallItem() } diff --git a/resources/qml/voip/ActiveCallBar.qml b/resources/qml/voip/ActiveCallBar.qml index 5589c79b..d7f3c6fd 100644 --- a/resources/qml/voip/ActiveCallBar.qml +++ b/resources/qml/voip/ActiveCallBar.qml @@ -12,7 +12,7 @@ Rectangle { MouseArea { anchors.fill: parent onClicked: { - if (CallManager.haveVideo) + if (CallManager.callType != CallType.VOICE) stackLayout.currentIndex = stackLayout.currentIndex ? 0 : 1; } @@ -139,7 +139,7 @@ Rectangle { PropertyChanges { target: stackLayout - currentIndex: CallManager.haveVideo ? 1 : 0 + currentIndex: CallManager.callType != CallType.VOICE ? 1 : 0 } }, @@ -196,15 +196,15 @@ Rectangle { } ImageButton { - visible: CallManager.haveLocalCamera + visible: CallManager.haveLocalPiP width: 24 height: 24 buttonTextColor: "#000000" image: ":/icons/icons/ui/toggle-camera-view.png" hoverEnabled: true ToolTip.visible: hovered - ToolTip.text: qsTr("Toggle camera view") - onClicked: CallManager.toggleCameraView() + ToolTip.text: qsTr("Hide/Show Picture-in-Picture") + onClicked: CallManager.toggleLocalPiP() } ImageButton { diff --git a/resources/qml/voip/ScreenShare.qml b/resources/qml/voip/ScreenShare.qml index cb70a36c..3ff74199 100644 --- a/resources/qml/voip/ScreenShare.qml +++ b/resources/qml/voip/ScreenShare.qml @@ -21,7 +21,10 @@ Popup { ColumnLayout { Label { - Layout.margins: 8 + Layout.topMargin: 16 + Layout.bottomMargin: 16 + Layout.leftMargin: 8 + Layout.rightMargin: 8 Layout.alignment: Qt.AlignLeft text: qsTr("Share desktop with %1?").arg(TimelineManager.timeline.roomName) color: colors.windowText diff --git a/src/CallManager.cpp b/src/CallManager.cpp index de04ed3b..58d75c2f 100644 --- a/src/CallManager.cpp +++ b/src/CallManager.cpp @@ -405,15 +405,6 @@ CallManager::screenShareSupported() return std::getenv("DISPLAY") && !std::getenv("WAYLAND_DISPLAY"); } -bool -CallManager::haveVideo() const -{ - return callType() == CallType::VIDEO || - (callType() == CallType::SCREEN && - (ChatPage::instance()->userSettings()->screenShareRemoteVideo() && - !session_.isRemoteVideoRecvOnly())); -} - QStringList CallManager::devices(bool isVideo) const { diff --git a/src/CallManager.h b/src/CallManager.h index cfc50481..19d79f86 100644 --- a/src/CallManager.h +++ b/src/CallManager.h @@ -30,8 +30,7 @@ class CallManager : public QObject Q_PROPERTY(QString callParty READ callParty NOTIFY newInviteState) Q_PROPERTY(QString callPartyAvatarUrl READ callPartyAvatarUrl NOTIFY newInviteState) Q_PROPERTY(bool isMicMuted READ isMicMuted NOTIFY micMuteChanged) - Q_PROPERTY(bool haveLocalCamera READ haveLocalCamera NOTIFY newCallState) - Q_PROPERTY(bool haveVideo READ haveVideo NOTIFY newInviteState) + Q_PROPERTY(bool haveLocalPiP READ haveLocalPiP NOTIFY newCallState) Q_PROPERTY(QStringList mics READ mics NOTIFY devicesChanged) Q_PROPERTY(QStringList cameras READ cameras NOTIFY devicesChanged) Q_PROPERTY(bool callsSupported READ callsSupported CONSTANT) @@ -47,8 +46,7 @@ public: QString callParty() const { return callParty_; } QString callPartyAvatarUrl() const { return callPartyAvatarUrl_; } bool isMicMuted() const { return session_.isMicMuted(); } - bool haveLocalCamera() const { return session_.haveLocalCamera(); } - bool haveVideo() const; + bool haveLocalPiP() const { return session_.haveLocalPiP(); } QStringList mics() const { return devices(false); } QStringList cameras() const { return devices(true); } void refreshTurnServer(); @@ -60,7 +58,7 @@ public slots: void sendInvite(const QString &roomid, webrtc::CallType); void syncEvent(const mtx::events::collections::TimelineEvents &event); void toggleMicMute(); - void toggleCameraView() { session_.toggleCameraView(); } + void toggleLocalPiP() { session_.toggleLocalPiP(); } void acceptInvite(); void hangUp( mtx::events::msg::CallHangUp::Reason = mtx::events::msg::CallHangUp::Reason::User); diff --git a/src/WebRTCSession.cpp b/src/WebRTCSession.cpp index c3a28117..4d38d196 100644 --- a/src/WebRTCSession.cpp +++ b/src/WebRTCSession.cpp @@ -350,42 +350,59 @@ getResolution(GstPad *pad) return ret; } +std::pair +getResolution(GstElement *pipe, const gchar *elementName, const gchar *padName) +{ + GstElement *element = gst_bin_get_by_name(GST_BIN(pipe), elementName); + GstPad *pad = gst_element_get_static_pad(element, padName); + auto ret = getResolution(pad); + gst_object_unref(pad); + gst_object_unref(element); + return ret; +} + +std::pair +getPiPDimensions(const std::pair resolution, int fullWidth, double scaleFactor) +{ + int pipWidth = fullWidth * scaleFactor; + int pipHeight = static_cast(resolution.second) / resolution.first * pipWidth; + return {pipWidth, pipHeight}; +} + void addLocalPiP(GstElement *pipe, const std::pair &videoCallSize) { - // embed localUser's camera into received video + // embed localUser's camera into received video (CallType::VIDEO) + // OR embed screen share into received video (CallType::SCREEN) GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe), "videosrctee"); if (!tee) return; - GstElement *queue = gst_element_factory_make("queue", nullptr); - GstElement *videorate = gst_element_factory_make("videorate", nullptr); - gst_bin_add_many(GST_BIN(pipe), queue, videorate, nullptr); - gst_element_link_many(tee, queue, videorate, nullptr); + GstElement *queue = gst_element_factory_make("queue", nullptr); + gst_bin_add(GST_BIN(pipe), queue); + gst_element_link(tee, queue); gst_element_sync_state_with_parent(queue); - gst_element_sync_state_with_parent(videorate); gst_object_unref(tee); - GstElement *camerafilter = gst_bin_get_by_name(GST_BIN(pipe), "camerafilter"); - GstPad *filtersinkpad = gst_element_get_static_pad(camerafilter, "sink"); - auto cameraResolution = getResolution(filtersinkpad); - int pipWidth = videoCallSize.first / 4; - int pipHeight = - static_cast(cameraResolution.second) / cameraResolution.first * pipWidth; - nhlog::ui()->debug("WebRTC: local picture-in-picture: {}x{}", pipWidth, pipHeight); - gst_object_unref(filtersinkpad); - gst_object_unref(camerafilter); - - GstPad *camerapad = gst_element_get_static_pad(videorate, "src"); GstElement *compositor = gst_bin_get_by_name(GST_BIN(pipe), "compositor"); localPiPSinkPad_ = gst_element_get_request_pad(compositor, "sink_%u"); g_object_set(localPiPSinkPad_, "zorder", 2, nullptr); - g_object_set(localPiPSinkPad_, "width", pipWidth, "height", pipHeight, nullptr); + + bool isVideo = WebRTCSession::instance().callType() == CallType::VIDEO; + const gchar *element = isVideo ? "camerafilter" : "screenshare"; + const gchar *pad = isVideo ? "sink" : "src"; + auto resolution = getResolution(pipe, element, pad); + auto pipSize = getPiPDimensions(resolution, videoCallSize.first, 0.25); + nhlog::ui()->debug( + "WebRTC: local picture-in-picture: {}x{}", pipSize.first, pipSize.second); + g_object_set(localPiPSinkPad_, "width", pipSize.first, "height", pipSize.second, nullptr); gint offset = videoCallSize.first / 80; g_object_set(localPiPSinkPad_, "xpos", offset, "ypos", offset, nullptr); - if (GST_PAD_LINK_FAILED(gst_pad_link(camerapad, localPiPSinkPad_))) + + GstPad *srcpad = gst_element_get_static_pad(queue, "src"); + if (GST_PAD_LINK_FAILED(gst_pad_link(srcpad, localPiPSinkPad_))) nhlog::ui()->error("WebRTC: failed to link local PiP elements"); - gst_object_unref(camerapad); + gst_object_unref(srcpad); gst_object_unref(compositor); } @@ -394,31 +411,37 @@ addRemotePiP(GstElement *pipe) { // embed localUser's camera into screen image being shared if (remotePiPSinkPad_) { - GstElement *screen = gst_bin_get_by_name(GST_BIN(pipe), "screenshare"); - GstPad *srcpad = gst_element_get_static_pad(screen, "src"); - auto resolution = getResolution(srcpad); + auto camRes = getResolution(pipe, "camerafilter", "sink"); + auto shareRes = getResolution(pipe, "screenshare", "src"); + auto pipSize = getPiPDimensions(camRes, shareRes.first, 0.2); nhlog::ui()->debug( - "WebRTC: screen share: {}x{}", resolution.first, resolution.second); - gst_object_unref(srcpad); - gst_object_unref(screen); + "WebRTC: screen share picture-in-picture: {}x{}", pipSize.first, pipSize.second); - int pipWidth = resolution.first / 5; - int pipHeight = - static_cast(resolution.second) / resolution.first * pipWidth; - nhlog::ui()->debug( - "WebRTC: screen share picture-in-picture: {}x{}", pipWidth, pipHeight); - gint offset = resolution.first / 100; + gint offset = shareRes.first / 100; g_object_set(remotePiPSinkPad_, "zorder", 2, nullptr); - g_object_set(remotePiPSinkPad_, "width", pipWidth, "height", pipHeight, nullptr); + g_object_set( + remotePiPSinkPad_, "width", pipSize.first, "height", pipSize.second, nullptr); g_object_set(remotePiPSinkPad_, "xpos", - resolution.first - pipWidth - offset, + shareRes.first - pipSize.first - offset, "ypos", - resolution.second - pipHeight - offset, + shareRes.second - pipSize.second - offset, nullptr); } } +void +addLocalVideo(GstElement *pipe) +{ + GstElement *queue = newVideoSinkChain(pipe); + GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe), "videosrctee"); + GstPad *srcpad = gst_element_get_request_pad(tee, "src_%u"); + GstPad *sinkpad = gst_element_get_static_pad(queue, "sink"); + if (GST_PAD_LINK_FAILED(gst_pad_link(srcpad, sinkpad))) + nhlog::ui()->error("WebRTC: failed to link videosrctee -> video sink chain"); + gst_object_unref(srcpad); +} + void linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe) { @@ -454,8 +477,7 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe) nhlog::ui()->info("WebRTC: incoming video resolution: {}x{}", videoCallSize.first, videoCallSize.second); - if (session->callType() == CallType::VIDEO) - addLocalPiP(pipe, videoCallSize); + addLocalPiP(pipe, videoCallSize); } else { g_free(mediaType); nhlog::ui()->error("WebRTC: unknown pad type: {}", GST_PAD_NAME(newpad)); @@ -478,6 +500,8 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe) g_timeout_add_seconds(3, testPacketLoss, nullptr); } addRemotePiP(pipe); + if (session->isRemoteVideoRecvOnly()) + addLocalVideo(pipe); } } gst_object_unref(queuepad); @@ -1011,13 +1035,19 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType) } bool -WebRTCSession::haveLocalCamera() const +WebRTCSession::haveLocalPiP() const { - if (callType_ == CallType::VIDEO && state_ >= State::INITIATED) { - GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe_), "videosrctee"); - if (tee) { - gst_object_unref(tee); + if (state_ >= State::INITIATED) { + if (callType_ == CallType::VOICE || isRemoteVideoRecvOnly_) + return false; + else if (callType_ == CallType::SCREEN) return true; + else { + GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe_), "videosrctee"); + if (tee) { + gst_object_unref(tee); + return true; + } } } return false; @@ -1051,7 +1081,7 @@ WebRTCSession::toggleMicMute() } void -WebRTCSession::toggleCameraView() +WebRTCSession::toggleLocalPiP() { if (localPiPSinkPad_) { guint zorder; @@ -1108,7 +1138,7 @@ WebRTCSession::havePlugins(bool, std::string *) } bool -WebRTCSession::haveLocalCamera() const +WebRTCSession::haveLocalPiP() const { return false; } @@ -1144,7 +1174,7 @@ WebRTCSession::toggleMicMute() } void -WebRTCSession::toggleCameraView() +WebRTCSession::toggleLocalPiP() {} void diff --git a/src/WebRTCSession.h b/src/WebRTCSession.h index 24ae9a17..fc637193 100644 --- a/src/WebRTCSession.h +++ b/src/WebRTCSession.h @@ -52,7 +52,7 @@ public: bool havePlugins(bool isVideo, std::string *errorMessage = nullptr); webrtc::CallType callType() const { return callType_; } webrtc::State state() const { return state_; } - bool haveLocalCamera() const; + bool haveLocalPiP() const; bool isOffering() const { return isOffering_; } bool isRemoteVideoRecvOnly() const { return isRemoteVideoRecvOnly_; } bool isRemoteVideoSendOnly() const { return isRemoteVideoSendOnly_; } @@ -64,7 +64,7 @@ public: bool isMicMuted() const; bool toggleMicMute(); - void toggleCameraView(); + void toggleLocalPiP(); void end(); void setTurnServers(const std::vector &uris) { turnServers_ = uris; } From efe240d60916f46e9175d3c8e24816d2f497f12f Mon Sep 17 00:00:00 2001 From: trilene Date: Wed, 24 Feb 2021 17:07:01 -0500 Subject: [PATCH 07/11] Allow choice of single window when sharing screen --- CMakeLists.txt | 5 ++ resources/qml/voip/ScreenShare.qml | 37 ++++++++++--- src/CallManager.cpp | 83 ++++++++++++++++++++++++++++-- src/CallManager.h | 4 +- src/WebRTCSession.cpp | 23 ++++----- src/WebRTCSession.h | 3 +- 6 files changed, 131 insertions(+), 24 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f695bc5..9c0eb1ab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -448,6 +448,7 @@ endif() include(FindPkgConfig) pkg_check_modules(GSTREAMER IMPORTED_TARGET gstreamer-sdp-1.0>=1.18 gstreamer-webrtc-1.0>=1.18) if (TARGET PkgConfig::GSTREAMER) + pkg_check_modules(XCB IMPORTED_TARGET xcb xcb-ewmh) add_feature_info(voip ON "GStreamer found. Call support is enabled automatically.") else() add_feature_info(voip OFF "GStreamer could not be found on your system. As a consequence call support has been disabled. If you don't want that, make sure gstreamer-sdp-1.0>=1.18 gstreamer-webrtc-1.0>=1.18 can be found via pkgconfig.") @@ -637,6 +638,10 @@ endif() if (TARGET PkgConfig::GSTREAMER) target_link_libraries(nheko PRIVATE PkgConfig::GSTREAMER) target_compile_definitions(nheko PRIVATE GSTREAMER_AVAILABLE) + if (TARGET PkgConfig::XCB) + target_link_libraries(nheko PRIVATE PkgConfig::XCB) + target_compile_definitions(nheko PRIVATE XCB_AVAILABLE) + endif() endif() if(MSVC) diff --git a/resources/qml/voip/ScreenShare.qml b/resources/qml/voip/ScreenShare.qml index 3ff74199..76991f45 100644 --- a/resources/qml/voip/ScreenShare.qml +++ b/resources/qml/voip/ScreenShare.qml @@ -13,9 +13,6 @@ Popup { anchors.centerIn = parent; frameRateCombo.currentIndex = frameRateCombo.find(Settings.screenShareFrameRate); - pipCheckBox.checked = Settings.screenSharePiP; - remoteVideoCheckBox.checked = Settings.screenShareRemoteVideo; - hideCursorCheckBox.checked = Settings.screenShareHideCursor; } palette: colors @@ -33,6 +30,27 @@ Popup { RowLayout { Layout.leftMargin: 8 Layout.rightMargin: 8 + Layout.bottomMargin: 8 + + Label { + Layout.alignment: Qt.AlignLeft + text: qsTr("Window:") + color: colors.windowText + } + + ComboBox { + id: windowCombo + + Layout.fillWidth: true + model: CallManager.windowList() + } + + } + + RowLayout { + Layout.leftMargin: 8 + Layout.rightMargin: 8 + Layout.bottomMargin: 8 Label { Layout.alignment: Qt.AlignLeft @@ -43,7 +61,7 @@ Popup { ComboBox { id: frameRateCombo - Layout.alignment: Qt.AlignRight + Layout.fillWidth: true model: ["25", "20", "15", "10", "5", "2", "1"] } @@ -52,7 +70,8 @@ Popup { CheckBox { id: pipCheckBox - visible: CallManager.cameras.length > 0 + enabled: CallManager.cameras.length > 0 + checked: Settings.screenSharePiP Layout.alignment: Qt.AlignLeft Layout.leftMargin: 8 Layout.rightMargin: 8 @@ -66,6 +85,7 @@ Popup { Layout.leftMargin: 8 Layout.rightMargin: 8 text: qsTr("Request remote camera") + checked: Settings.screenShareRemoteVideo ToolTip.text: qsTr("View your callee's camera like a regular video call") ToolTip.visible: hovered } @@ -76,7 +96,9 @@ Popup { Layout.alignment: Qt.AlignLeft Layout.leftMargin: 8 Layout.rightMargin: 8 + Layout.bottomMargin: 8 text: qsTr("Hide mouse cursor") + checked: Settings.screenShareHideCursor } RowLayout { @@ -92,11 +114,14 @@ Popup { onClicked: { if (buttonLayout.validateMic()) { Settings.microphone = micCombo.currentText; + if (pipCheckBox.checked) + Settings.camera = cameraCombo.currentText; + Settings.screenShareFrameRate = frameRateCombo.currentText; Settings.screenSharePiP = pipCheckBox.checked; Settings.screenShareRemoteVideo = remoteVideoCheckBox.checked; Settings.screenShareHideCursor = hideCursorCheckBox.checked; - CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.SCREEN); + CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.SCREEN, windowCombo.currentIndex); close(); } } diff --git a/src/CallManager.cpp b/src/CallManager.cpp index 58d75c2f..f5e2c2bb 100644 --- a/src/CallManager.cpp +++ b/src/CallManager.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -18,6 +19,11 @@ #include "mtx/responses/turn_server.hpp" +#ifdef XCB_AVAILABLE +#include +#include +#endif + Q_DECLARE_METATYPE(std::vector) Q_DECLARE_METATYPE(mtx::events::msg::CallCandidates::Candidate) Q_DECLARE_METATYPE(mtx::responses::TurnServer) @@ -151,12 +157,18 @@ CallManager::CallManager(QObject *parent) } void -CallManager::sendInvite(const QString &roomid, CallType callType) +CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int windowIndex) { if (isOnCall()) return; - if (callType == CallType::SCREEN && !screenShareSupported()) - return; + if (callType == CallType::SCREEN) { + if (!screenShareSupported()) + return; + if (windows_.empty() || windowIndex >= windows_.size()) { + nhlog::ui()->error("WebRTC: window index out of range"); + return; + } + } auto roomInfo = cache::singleRoomInfo(roomid.toStdString()); if (roomInfo.member_count != 2) { @@ -187,7 +199,7 @@ CallManager::sendInvite(const QString &roomid, CallType callType) callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url); emit newInviteState(); playRingtone(QUrl("qrc:/media/media/ringback.ogg"), true); - if (!session_.createOffer(callType)) { + if (!session_.createOffer(callType, windows_[windowIndex].second)) { emit ChatPage::instance()->showNotification("Problem setting up call."); endCall(); } @@ -490,6 +502,69 @@ CallManager::stopRingtone() player_.setPlaylist(nullptr); } +QStringList +CallManager::windowList() +{ + windows_.clear(); + windows_.push_back({"Entire screen", 0}); + +#ifdef XCB_AVAILABLE + std::unique_ptr> connection( + xcb_connect(nullptr, nullptr), [](xcb_connection_t *c) { xcb_disconnect(c); }); + if (xcb_connection_has_error(connection.get())) { + nhlog::ui()->error("Failed to connect to X server"); + return {}; + } + + xcb_ewmh_connection_t ewmh; + if (!xcb_ewmh_init_atoms_replies( + &ewmh, xcb_ewmh_init_atoms(connection.get(), &ewmh), nullptr)) { + nhlog::ui()->error("Failed to connect to EWMH server"); + return {}; + } + std::unique_ptr> + ewmhconnection(&ewmh, [](xcb_ewmh_connection_t *c) { xcb_ewmh_connection_wipe(c); }); + + for (int i = 0; i < ewmh.nb_screens; i++) { + xcb_ewmh_get_windows_reply_t clients; + if (!xcb_ewmh_get_client_list_reply( + &ewmh, xcb_ewmh_get_client_list(&ewmh, i), &clients, nullptr)) { + nhlog::ui()->error("Failed to request window list"); + return {}; + } + + for (uint32_t w = 0; w < clients.windows_len; w++) { + xcb_window_t window = clients.windows[w]; + + std::string name; + xcb_ewmh_get_utf8_strings_reply_t data; + auto getName = [](xcb_ewmh_get_utf8_strings_reply_t *r) { + std::string name(r->strings, r->strings_len); + xcb_ewmh_get_utf8_strings_reply_wipe(r); + return name; + }; + + xcb_get_property_cookie_t cookie = xcb_ewmh_get_wm_name(&ewmh, window); + if (xcb_ewmh_get_wm_name_reply(&ewmh, cookie, &data, nullptr)) + name = getName(&data); + + cookie = xcb_ewmh_get_wm_visible_name(&ewmh, window); + if (xcb_ewmh_get_wm_visible_name_reply(&ewmh, cookie, &data, nullptr)) + name = getName(&data); + + windows_.push_back({QString::fromStdString(name), window}); + } + xcb_ewmh_get_windows_reply_wipe(&clients); + } +#endif + QStringList ret; + ret.reserve(windows_.size()); + for (const auto &w : windows_) + ret.append(w.first); + + return ret; +} + namespace { std::vector getTurnURIs(const mtx::responses::TurnServer &turnServer) diff --git a/src/CallManager.h b/src/CallManager.h index 19d79f86..6decdf19 100644 --- a/src/CallManager.h +++ b/src/CallManager.h @@ -55,13 +55,14 @@ public: static bool screenShareSupported(); public slots: - void sendInvite(const QString &roomid, webrtc::CallType); + void sendInvite(const QString &roomid, webrtc::CallType, unsigned int windowIndex = 0); void syncEvent(const mtx::events::collections::TimelineEvents &event); void toggleMicMute(); void toggleLocalPiP() { session_.toggleLocalPiP(); } void acceptInvite(); void hangUp( mtx::events::msg::CallHangUp::Reason = mtx::events::msg::CallHangUp::Reason::User); + QStringList windowList(); signals: void newMessage(const QString &roomid, const mtx::events::msg::CallInvite &); @@ -91,6 +92,7 @@ private: std::vector turnURIs_; QTimer turnServerTimer_; QMediaPlayer player_; + std::vector> windows_; template bool handleEvent_(const mtx::events::collections::TimelineEvents &event); diff --git a/src/WebRTCSession.cpp b/src/WebRTCSession.cpp index 4d38d196..74855835 100644 --- a/src/WebRTCSession.cpp +++ b/src/WebRTCSession.cpp @@ -362,7 +362,7 @@ getResolution(GstElement *pipe, const gchar *elementName, const gchar *padName) } std::pair -getPiPDimensions(const std::pair resolution, int fullWidth, double scaleFactor) +getPiPDimensions(const std::pair &resolution, int fullWidth, double scaleFactor) { int pipWidth = fullWidth * scaleFactor; int pipHeight = static_cast(resolution.second) / resolution.first * pipWidth; @@ -629,11 +629,12 @@ WebRTCSession::havePlugins(bool isVideo, std::string *errorMessage) } bool -WebRTCSession::createOffer(CallType callType) +WebRTCSession::createOffer(CallType callType, uint32_t shareWindowId) { clear(); - isOffering_ = true; - callType_ = callType; + isOffering_ = true; + callType_ = callType; + shareWindowId_ = shareWindowId; // opus and vp8 rtp payload types must be defined dynamically // therefore from the range [96-127] @@ -888,15 +889,12 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType) if (callType_ == CallType::VIDEO && !devices_.haveCamera()) return !isOffering_; - auto settings = ChatPage::instance()->userSettings(); - if (callType_ == CallType::SCREEN && settings->screenSharePiP() && !devices_.haveCamera()) - return false; - + auto settings = ChatPage::instance()->userSettings(); GstElement *camerafilter = nullptr; GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr); GstElement *tee = gst_element_factory_make("tee", "videosrctee"); gst_bin_add_many(GST_BIN(pipe_), videoconvert, tee, nullptr); - if (callType_ == CallType::VIDEO || settings->screenSharePiP()) { + if (callType_ == CallType::VIDEO || (settings->screenSharePiP() && devices_.haveCamera())) { std::pair resolution; std::pair frameRate; GstDevice *device = devices_.videoDevice(resolution, frameRate); @@ -947,7 +945,7 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType) return false; } g_object_set(ximagesrc, "use-damage", FALSE, nullptr); - g_object_set(ximagesrc, "xid", 0, nullptr); + g_object_set(ximagesrc, "xid", shareWindowId_, nullptr); g_object_set( ximagesrc, "show-pointer", !settings->screenShareHideCursor(), nullptr); @@ -962,7 +960,7 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType) gst_caps_unref(caps); gst_bin_add_many(GST_BIN(pipe_), ximagesrc, capsfilter, nullptr); - if (settings->screenSharePiP()) { + if (settings->screenSharePiP() && devices_.haveCamera()) { GstElement *compositor = gst_element_factory_make("compositor", nullptr); g_object_set(compositor, "background", 1, nullptr); gst_bin_add(GST_BIN(pipe_), compositor); @@ -1101,6 +1099,7 @@ WebRTCSession::clear() pipe_ = nullptr; webrtc_ = nullptr; busWatchId_ = 0; + shareWindowId_ = 0; haveAudioStream_ = false; haveVideoStream_ = false; localPiPSinkPad_ = nullptr; @@ -1143,7 +1142,7 @@ WebRTCSession::haveLocalPiP() const return false; } -bool WebRTCSession::createOffer(webrtc::CallType) { return false; } +bool WebRTCSession::createOffer(webrtc::CallType, uint32_t) { return false; } bool WebRTCSession::acceptOffer(const std::string &) diff --git a/src/WebRTCSession.h b/src/WebRTCSession.h index fc637193..7c77a94d 100644 --- a/src/WebRTCSession.h +++ b/src/WebRTCSession.h @@ -57,7 +57,7 @@ public: bool isRemoteVideoRecvOnly() const { return isRemoteVideoRecvOnly_; } bool isRemoteVideoSendOnly() const { return isRemoteVideoSendOnly_; } - bool createOffer(webrtc::CallType); + bool createOffer(webrtc::CallType, uint32_t shareWindowId); bool acceptOffer(const std::string &sdp); bool acceptAnswer(const std::string &sdp); void acceptICECandidates(const std::vector &); @@ -100,6 +100,7 @@ private: GstElement *webrtc_ = nullptr; unsigned int busWatchId_ = 0; std::vector turnServers_; + uint32_t shareWindowId_ = 0; bool init(std::string *errorMessage = nullptr); bool startPipeline(int opusPayloadType, int vp8PayloadType); From 402bd565cb2b8836420f73461acacd886c9d147c Mon Sep 17 00:00:00 2001 From: trilene Date: Thu, 25 Feb 2021 12:00:55 -0500 Subject: [PATCH 08/11] Add screen sharing window preview --- resources/qml/voip/ScreenShare.qml | 7 +++ src/CallManager.cpp | 94 +++++++++++++++++++++++++++++- src/CallManager.h | 3 +- 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/resources/qml/voip/ScreenShare.qml b/resources/qml/voip/ScreenShare.qml index 76991f45..a22b5b68 100644 --- a/resources/qml/voip/ScreenShare.qml +++ b/resources/qml/voip/ScreenShare.qml @@ -127,6 +127,13 @@ Popup { } } + Button { + text: qsTr("Preview") + onClicked: { + CallManager.previewWindow(windowCombo.currentIndex); + } + } + Button { text: qsTr("Cancel") onClicked: { diff --git a/src/CallManager.cpp b/src/CallManager.cpp index f5e2c2bb..67e30c4e 100644 --- a/src/CallManager.cpp +++ b/src/CallManager.cpp @@ -24,6 +24,13 @@ #include #endif +#ifdef GSTREAMER_AVAILABLE +extern "C" +{ +#include "gst/gst.h" +} +#endif + Q_DECLARE_METATYPE(std::vector) Q_DECLARE_METATYPE(mtx::events::msg::CallCandidates::Candidate) Q_DECLARE_METATYPE(mtx::responses::TurnServer) @@ -235,8 +242,8 @@ void CallManager::syncEvent(const mtx::events::collections::TimelineEvents &event) { #ifdef GSTREAMER_AVAILABLE - if (handleEvent_(event) || handleEvent_(event) || - handleEvent_(event) || handleEvent_(event)) + if (handleEvent(event) || handleEvent(event) || + handleEvent(event) || handleEvent(event)) return; #else (void)event; @@ -245,7 +252,7 @@ CallManager::syncEvent(const mtx::events::collections::TimelineEvents &event) template bool -CallManager::handleEvent_(const mtx::events::collections::TimelineEvents &event) +CallManager::handleEvent(const mtx::events::collections::TimelineEvents &event) { if (std::holds_alternative>(event)) { handleEvent(std::get>(event)); @@ -565,6 +572,87 @@ CallManager::windowList() return ret; } +#ifdef GSTREAMER_AVAILABLE +namespace { + +GstElement *pipe_ = nullptr; +unsigned int busWatchId_ = 0; + +gboolean +newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer G_GNUC_UNUSED) +{ + switch (GST_MESSAGE_TYPE(msg)) { + case GST_MESSAGE_EOS: + if (pipe_) { + gst_element_set_state(GST_ELEMENT(pipe_), GST_STATE_NULL); + gst_object_unref(pipe_); + pipe_ = nullptr; + } + if (busWatchId_) { + g_source_remove(busWatchId_); + busWatchId_ = 0; + } + break; + default: + break; + } + return TRUE; +} + +} +#endif + +void +CallManager::previewWindow(unsigned int index) const +{ +#ifdef GSTREAMER_AVAILABLE + if (windows_.empty() || index >= windows_.size() || !gst_is_initialized()) + return; + + GstElement *ximagesrc = gst_element_factory_make("ximagesrc", nullptr); + if (!ximagesrc) { + nhlog::ui()->error("Failed to create ximagesrc"); + return; + } + GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr); + GstElement *videoscale = gst_element_factory_make("videoscale", nullptr); + GstElement *capsfilter = gst_element_factory_make("capsfilter", nullptr); + GstElement *ximagesink = gst_element_factory_make("ximagesink", nullptr); + + g_object_set(ximagesrc, "use-damage", FALSE, nullptr); + g_object_set(ximagesrc, "show-pointer", FALSE, nullptr); + g_object_set(ximagesrc, "xid", windows_[index].second, nullptr); + + GstCaps *caps = gst_caps_new_simple( + "video/x-raw", "width", G_TYPE_INT, 480, "height", G_TYPE_INT, 360, nullptr); + g_object_set(capsfilter, "caps", caps, nullptr); + gst_caps_unref(caps); + + pipe_ = gst_pipeline_new(nullptr); + gst_bin_add_many( + GST_BIN(pipe_), ximagesrc, videoconvert, videoscale, capsfilter, ximagesink, nullptr); + if (!gst_element_link_many( + ximagesrc, videoconvert, videoscale, capsfilter, ximagesink, nullptr)) { + nhlog::ui()->error("Failed to link preview window elements"); + gst_object_unref(pipe_); + pipe_ = nullptr; + return; + } + if (gst_element_set_state(pipe_, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { + nhlog::ui()->error("Unable to start preview pipeline"); + gst_object_unref(pipe_); + pipe_ = nullptr; + return; + } + + GstBus *bus = gst_pipeline_get_bus(GST_PIPELINE(pipe_)); + busWatchId_ = gst_bus_add_watch(bus, newBusMessage, nullptr); + gst_object_unref(bus); +#else + (void)index; +#endif +} + namespace { std::vector getTurnURIs(const mtx::responses::TurnServer &turnServer) diff --git a/src/CallManager.h b/src/CallManager.h index 6decdf19..be3c824d 100644 --- a/src/CallManager.h +++ b/src/CallManager.h @@ -63,6 +63,7 @@ public slots: void hangUp( mtx::events::msg::CallHangUp::Reason = mtx::events::msg::CallHangUp::Reason::User); QStringList windowList(); + void previewWindow(unsigned int windowIndex) const; signals: void newMessage(const QString &roomid, const mtx::events::msg::CallInvite &); @@ -95,7 +96,7 @@ private: std::vector> windows_; template - bool handleEvent_(const mtx::events::collections::TimelineEvents &event); + bool handleEvent(const mtx::events::collections::TimelineEvents &event); void handleEvent(const mtx::events::RoomEvent &); void handleEvent(const mtx::events::RoomEvent &); void handleEvent(const mtx::events::RoomEvent &); From 12e40a13cb53232320397f1704b9b4e7adebb785 Mon Sep 17 00:00:00 2001 From: trilene Date: Thu, 25 Feb 2021 12:44:09 -0500 Subject: [PATCH 09/11] Add missing translation mark --- src/CallManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CallManager.cpp b/src/CallManager.cpp index 67e30c4e..40be3a12 100644 --- a/src/CallManager.cpp +++ b/src/CallManager.cpp @@ -513,7 +513,7 @@ QStringList CallManager::windowList() { windows_.clear(); - windows_.push_back({"Entire screen", 0}); + windows_.push_back({tr("Entire screen"), 0}); #ifdef XCB_AVAILABLE std::unique_ptr> connection( From 6baa775ec87357bd0fa8841e09f6618db0e317f8 Mon Sep 17 00:00:00 2001 From: trilene Date: Thu, 25 Feb 2021 13:27:22 -0500 Subject: [PATCH 10/11] add_feature_info for screen sharing window selection --- CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e0cf575..111f94d1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -449,8 +449,13 @@ endif() include(FindPkgConfig) pkg_check_modules(GSTREAMER IMPORTED_TARGET gstreamer-sdp-1.0>=1.18 gstreamer-webrtc-1.0>=1.18) if (TARGET PkgConfig::GSTREAMER) - pkg_check_modules(XCB IMPORTED_TARGET xcb xcb-ewmh) add_feature_info(voip ON "GStreamer found. Call support is enabled automatically.") + pkg_check_modules(XCB IMPORTED_TARGET xcb xcb-ewmh) + if (TARGET PkgConfig::XCB) + add_feature_info("Window selection when screen sharing (X11)" ON "XCB-EWMH found. Window selection is enabled when screen sharing (X11).") + else() + add_feature_info("Window selection when screen sharing (X11)" OFF "XCB-EWMH could not be found on your system. Screen sharing (X11) is limited to the entire screen only. To enable window selection, make sure xcb and xcb-ewmh can be found via pkgconfig.") + endif() else() add_feature_info(voip OFF "GStreamer could not be found on your system. As a consequence call support has been disabled. If you don't want that, make sure gstreamer-sdp-1.0>=1.18 gstreamer-webrtc-1.0>=1.18 can be found via pkgconfig.") endif() From 099207b88c93d4f79148d86fa8683a19d9888b99 Mon Sep 17 00:00:00 2001 From: trilene Date: Thu, 25 Feb 2021 13:44:08 -0500 Subject: [PATCH 11/11] Restore voice/video calls --- src/CallManager.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CallManager.cpp b/src/CallManager.cpp index 40be3a12..eec5922e 100644 --- a/src/CallManager.cpp +++ b/src/CallManager.cpp @@ -206,7 +206,8 @@ CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int w callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url); emit newInviteState(); playRingtone(QUrl("qrc:/media/media/ringback.ogg"), true); - if (!session_.createOffer(callType, windows_[windowIndex].second)) { + if (!session_.createOffer( + callType, callType == CallType::SCREEN ? windows_[windowIndex].second : 0)) { emit ChatPage::instance()->showNotification("Problem setting up call."); endCall(); }