Use in memory media player instead of storing unencrypted files on disk

CMakeLists.txt

@@ -311,6 +311,7 @@ set(SRC_FILES
 	src/ui/InfoMessage.cpp
 	src/ui/Label.cpp
 	src/ui/LoadingIndicator.cpp
+	src/ui/MxcMediaProxy.cpp
 	src/ui/NhekoCursorShape.cpp
 	src/ui/NhekoDropArea.cpp
 	src/ui/NhekoGlobalObject.cpp
@@ -350,7 +351,7 @@ set(SRC_FILES
 	src/MemberList.cpp
 	src/MxcImageProvider.cpp
 	src/Olm.cpp
-    src/ReadReceiptsModel.cpp
+	src/ReadReceiptsModel.cpp
 	src/RegisterPage.cpp
 	src/SSOHandler.cpp
 	src/CombinedImagePackModel.cpp
@@ -521,6 +522,7 @@ qt5_wrap_cpp(MOC_HEADERS
 	src/ui/InfoMessage.h
 	src/ui/Label.h
 	src/ui/LoadingIndicator.h
+	src/ui/MxcMediaProxy.h
 	src/ui/Menu.h
 	src/ui/NhekoCursorShape.h
 	src/ui/NhekoDropArea.h

resources/qml/ForwardCompleter.qml

@@ -85,11 +85,10 @@ Popup {
                     completerPopup.up();
                 } else if ((event.key == Qt.Key_Down || event.key == Qt.Key_Tab) && completerPopup.opened) {
                     event.accepted = true;
-                    if (event.key == Qt.Key_Tab && (event.modifiers & Qt.ShiftModifier)) {
+                    if (event.key == Qt.Key_Tab && (event.modifiers & Qt.ShiftModifier))
                         completerPopup.up();
-                    } else {
+                    else
                         completerPopup.down();
-                    }
                 } else if (event.matches(StandardKey.InsertParagraphSeparator)) {
                     completerPopup.finishCompletion();
                     event.accepted = true;

resources/qml/MessageInput.qml

@@ -134,9 +134,9 @@ Rectangle {
                         return ;
 
                     room.input.updateState(selectionStart, selectionEnd, cursorPosition, text);
-                    if (popup.opened && cursorPosition <= completerTriggeredAt) {
+                    if (popup.opened && cursorPosition <= completerTriggeredAt)
                         popup.close();
-                    }
+
                     if (popup.opened)
                         popup.completer.setSearchString(messageInput.getText(completerTriggeredAt, cursorPosition));
 
@@ -195,11 +195,10 @@ Rectangle {
                     } else if (event.key == Qt.Key_Tab) {
                         event.accepted = true;
                         if (popup.opened) {
-                            if (event.modifiers & Qt.ShiftModifier) {
+                            if (event.modifiers & Qt.ShiftModifier)
                                 popup.down();
-                            } else {
+                            else
                                 popup.up();
-                            }
                         } else {
                             var pos = cursorPosition - 1;
                             while (pos > -1) {

resources/qml/QuickSwitcher.qml

@@ -44,11 +44,10 @@ Popup {
                 completerPopup.up();
             } else if ((event.key == Qt.Key_Down || event.key == Qt.Key_Tab) && completerPopup.opened) {
                 event.accepted = true;
-                if (event.key == Qt.Key_Tab && (event.modifiers & Qt.ShiftModifier)) {
+                if (event.key == Qt.Key_Tab && (event.modifiers & Qt.ShiftModifier))
                     completerPopup.up();
-                } else {
+                else
                     completerPopup.down();
-                }
             } else if (event.matches(StandardKey.InsertParagraphSeparator)) {
                 completerPopup.finishCompletion();
                 event.accepted = true;

resources/qml/delegates/PlayableMediaMessage.qml

@@ -3,9 +3,9 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 import "../"
-import QtMultimedia 5.6
+import QtMultimedia 5.15
-import QtQuick 2.12
+import QtQuick 2.15
-import QtQuick.Controls 2.1
+import QtQuick.Controls 2.15
 import QtQuick.Layouts 1.2
 import im.nheko 1.0
 
@@ -55,7 +55,8 @@ Rectangle {
                 VideoOutput {
                     anchors.fill: parent
                     fillMode: VideoOutput.PreserveAspectFit
-                    source: media
+                    flushMode: VideoOutput.FirstFrame
+                    source: mxcmedia
                 }
 
             }
@@ -93,15 +94,15 @@ Rectangle {
                         return hh + ":" + mm + ":" + ss;
                     }
 
-                    positionText.text = formatTime(new Date(media.position));
+                    positionText.text = formatTime(new Date(mxcmedia.position));
-                    durationText.text = formatTime(new Date(media.duration));
+                    durationText.text = formatTime(new Date(mxcmedia.duration));
                 }
 
                 Layout.fillWidth: true
-                value: media.position
+                value: mxcmedia.position
                 from: 0
-                to: media.duration
+                to: mxcmedia.duration
-                onMoved: media.seek(value)
+                onMoved: mxcmedia.position = value
                 onValueChanged: updatePositionTexts()
                 palette: Nheko.colors
             }
@@ -132,15 +133,15 @@ Rectangle {
                 onClicked: {
                     switch (button.state) {
                     case "":
-                        room.cacheMedia(eventId);
+                        mxcmedia.eventId = eventId;
                         break;
                     case "stopped":
-                        media.play();
+                        mxcmedia.play();
                         console.log("play");
                         button.state = "playing";
                         break;
                     case "playing":
-                        media.pause();
+                        mxcmedia.pause();
                         console.log("pause");
                         button.state = "stopped";
                         break;
@@ -172,29 +173,22 @@ Rectangle {
                     cursorShape: Qt.PointingHandCursor
                 }
 
-                MediaPlayer {
+                MxcMedia {
-                    id: media
+                    id: mxcmedia
 
+                    roomm: room
                     onError: console.log(errorString)
-                    onStatusChanged: {
+                    onMediaStatusChanged: {
-                        if (status == MediaPlayer.Loaded)
+                        if (status == MxcMedia.LoadedMedia) {
                             progress.updatePositionTexts();
-
-                    }
-                    onStopped: button.state = "stopped"
-                }
-
-                Connections {
-                    function onMediaCached(mxcUrl, cacheUrl) {
-                        if (mxcUrl == url) {
-                            media.source = cacheUrl;
                             button.state = "stopped";
-                            console.log("media loaded: " + mxcUrl + " at " + cacheUrl);
                         }
-                        console.log("media cached: " + mxcUrl + " at " + cacheUrl);
                     }
-
+                    onStateChanged: {
-                    target: room
+                        if (state == MxcMedia.StoppedState) {
+                            button.state = "stopped";
+                        }
+                    }
                 }
 
             }

src/timeline/TimelineModel.h

@@ -293,6 +293,15 @@ public:
         crypto::Trust trustlevel() const;
         int roomMemberCount() const;
 
+        std::optional<mtx::events::collections::TimelineEvents> eventById(const QString &id)
+        {
+                auto e = events.get(id.toStdString(), "");
+                if (e)
+                        return *e;
+                else
+                        return std::nullopt;
+        }
+
 public slots:
         void setCurrentIndex(int index);
         int currentIndex() const { return idToIndex(currentId); }

src/timeline/TimelineViewManager.cpp

@@ -35,6 +35,7 @@
 #include "dialogs/ImageOverlay.h"
 #include "emoji/EmojiModel.h"
 #include "emoji/Provider.h"
+#include "ui/MxcMediaProxy.h"
 #include "ui/NhekoCursorShape.h"
 #include "ui/NhekoDropArea.h"
 #include "ui/NhekoGlobalObject.h"
@@ -176,6 +177,7 @@ TimelineViewManager::TimelineViewManager(CallManager *callManager, ChatPage *par
         qmlRegisterType<DelegateChooser>("im.nheko", 1, 0, "DelegateChooser");
         qmlRegisterType<NhekoDropArea>("im.nheko", 1, 0, "NhekoDropArea");
         qmlRegisterType<NhekoCursorShape>("im.nheko", 1, 0, "CursorShape");
+        qmlRegisterType<MxcMediaProxy>("im.nheko", 1, 0, "MxcMedia");
         qmlRegisterUncreatableType<DeviceVerificationFlow>(
           "im.nheko", 1, 0, "DeviceVerificationFlow", "Can't create verification flow from QML!");
         qmlRegisterUncreatableType<UserProfile>(

src/ui/MxcMediaProxy.cpp

@@ -0,0 +1,142 @@
+// SPDX-FileCopyrightText: 2021 Nheko Contributors
+//
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "MxcMediaProxy.h"
+
+#include <QDir>
+#include <QFile>
+#include <QFileInfo>
+#include <QMediaObject>
+#include <QMediaPlayer>
+#include <QMimeDatabase>
+#include <QStandardPaths>
+#include <QUrl>
+
+#include "EventAccessors.h"
+#include "Logging.h"
+#include "MatrixClient.h"
+#include "timeline/TimelineModel.h"
+
+void
+MxcMediaProxy::setVideoSurface(QAbstractVideoSurface *surface)
+{
+        qDebug() << "Changing surface";
+        m_surface = surface;
+        setVideoOutput(m_surface);
+}
+
+QAbstractVideoSurface *
+MxcMediaProxy::getVideoSurface()
+{
+        return m_surface;
+}
+
+void
+MxcMediaProxy::startDownload()
+{
+        if (!room_)
+                return;
+        if (eventId_.isEmpty())
+                return;
+
+        auto event = room_->eventById(eventId_);
+        if (!event) {
+                nhlog::ui()->error("Failed to load media for event {}, event not found.",
+                                   eventId_.toStdString());
+                return;
+        }
+
+        QString mxcUrl           = QString::fromStdString(mtx::accessors::url(*event));
+        QString originalFilename = QString::fromStdString(mtx::accessors::filename(*event));
+        QString mimeType         = QString::fromStdString(mtx::accessors::mimetype(*event));
+
+        auto encryptionInfo = mtx::accessors::file(*event);
+
+        // If the message is a link to a non mxcUrl, don't download it
+        if (!mxcUrl.startsWith("mxc://")) {
+                return;
+        }
+
+        QString suffix = QMimeDatabase().mimeTypeForName(mimeType).preferredSuffix();
+
+        const auto url  = mxcUrl.toStdString();
+        const auto name = QString(mxcUrl).remove("mxc://");
+        QFileInfo filename(QString("%1/media_cache/media/%2.%3")
+                             .arg(QStandardPaths::writableLocation(QStandardPaths::CacheLocation))
+                             .arg(name)
+                             .arg(suffix));
+        if (QDir::cleanPath(name) != name) {
+                nhlog::net()->warn("mxcUrl '{}' is not safe, not downloading file", url);
+                return;
+        }
+
+        QDir().mkpath(filename.path());
+
+        QPointer<MxcMediaProxy> self = this;
+
+        auto processBuffer = [this, encryptionInfo, filename, self](QIODevice &device) {
+                if (!self)
+                        return;
+
+                if (encryptionInfo) {
+                        QByteArray ba = device.readAll();
+                        std::string temp(ba.constData(), ba.size());
+                        temp = mtx::crypto::to_string(
+                          mtx::crypto::decrypt_file(temp, encryptionInfo.value()));
+                        buffer.setData(temp.data(), temp.size());
+                } else {
+                        buffer.setData(device.readAll());
+                }
+                buffer.open(QIODevice::ReadOnly);
+                buffer.reset();
+
+                QTimer::singleShot(0, this, [this, self, filename] {
+                        nhlog::ui()->info("Playing buffer with size: {}, {}",
+                                          buffer.bytesAvailable(),
+                                          buffer.isOpen());
+                        self->setMedia(QMediaContent(filename.fileName()), &buffer);
+                        emit loadedChanged();
+                });
+        };
+
+        if (filename.isReadable()) {
+                QFile f(filename.filePath());
+                if (f.open(QIODevice::ReadOnly)) {
+                        processBuffer(f);
+                        return;
+                }
+        }
+
+        http::client()->download(
+          url,
+          [filename, url, processBuffer](const std::string &data,
+                                         const std::string &,
+                                         const std::string &,
+                                         mtx::http::RequestErr err) {
+                  if (err) {
+                          nhlog::net()->warn("failed to retrieve media {}: {} {}",
+                                             url,
+                                             err->matrix_error.error,
+                                             static_cast<int>(err->status_code));
+                          return;
+                  }
+
+                  try {
+                          QFile file(filename.filePath());
+
+                          if (!file.open(QIODevice::WriteOnly))
+                                  return;
+
+                          QByteArray ba(data.data(), (int)data.size());
+                          file.write(ba);
+                          file.close();
+
+                          QBuffer buf(&ba);
+                          buf.open(QBuffer::ReadOnly);
+                          processBuffer(buf);
+                  } catch (const std::exception &e) {
+                          nhlog::ui()->warn("Error while saving file to: {}", e.what());
+                  }
+          });
+}

src/ui/MxcMediaProxy.h

@@ -0,0 +1,80 @@
+// SPDX-FileCopyrightText: 2021 Nheko Contributors
+//
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <QAbstractVideoSurface>
+#include <QBuffer>
+#include <QMediaContent>
+#include <QMediaPlayer>
+#include <QObject>
+#include <QPointer>
+#include <QString>
+
+#include "Logging.h"
+
+class TimelineModel;
+
+// I failed to get my own buffer into the MediaPlayer in qml, so just make our own. For that we just
+// need the videoSurface property, so that part is really easy!
+class MxcMediaProxy : public QMediaPlayer
+{
+        Q_OBJECT
+        Q_PROPERTY(TimelineModel *roomm READ room WRITE setRoom NOTIFY roomChanged REQUIRED)
+        Q_PROPERTY(QString eventId READ eventId WRITE setEventId NOTIFY eventIdChanged)
+        Q_PROPERTY(QAbstractVideoSurface *videoSurface READ getVideoSurface WRITE setVideoSurface)
+        Q_PROPERTY(bool loaded READ loaded NOTIFY loadedChanged)
+public:
+        MxcMediaProxy(QObject *parent = nullptr)
+          : QMediaPlayer(parent)
+        {
+                connect(this, &MxcMediaProxy::eventIdChanged, &MxcMediaProxy::startDownload);
+                connect(this, &MxcMediaProxy::roomChanged, &MxcMediaProxy::startDownload);
+                connect(this,
+                        qOverload<QMediaPlayer::Error>(&MxcMediaProxy::error),
+                        [this](QMediaPlayer::Error error) {
+                                nhlog::ui()->info("Media player error {} and errorStr {}",
+                                                  error,
+                                                  this->errorString().toStdString());
+                        });
+                connect(this,
+                        &MxcMediaProxy::mediaStatusChanged,
+                        [this](QMediaPlayer::MediaStatus status) {
+                                nhlog::ui()->info(
+                                  "Media player status {} and error {}", status, this->error());
+                        });
+        }
+
+        bool loaded() const { return buffer.size() > 0; }
+        QString eventId() const { return eventId_; }
+        TimelineModel *room() const { return room_; }
+        void setEventId(QString newEventId)
+        {
+                eventId_ = newEventId;
+                emit eventIdChanged();
+        }
+        void setRoom(TimelineModel *room)
+        {
+                room_ = room;
+                emit roomChanged();
+        }
+        void setVideoSurface(QAbstractVideoSurface *surface);
+        QAbstractVideoSurface *getVideoSurface();
+
+signals:
+        void roomChanged();
+        void eventIdChanged();
+        void loadedChanged();
+        void newBuffer(QMediaContent, QIODevice *buf);
+
+private slots:
+        void startDownload();
+
+private:
+        TimelineModel *room_ = nullptr;
+        QString eventId_;
+        QString filename_;
+        QBuffer buffer;
+        QAbstractVideoSurface *m_surface = nullptr;
+};