rnhmjoj/nheko
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Save timeline messages in cache for faster startup times

Browse Source
This commit is contained in:
Konstantinos Sideris 2018-06-28 16:16:43 +03:00
parent 1d6746e4c9
commit 4344b6964f
14 changed files with 272 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deps/CMakeLists.txt vendored
View File

@ -37,7 +37,7 @@ set(BOOST_SHA256
5721818253e6a0989583192f96782c4a98eb6204965316df9f5ad75819225ca9)
set(MATRIX_STRUCTS_URL https://github.com/mujx/matrix-structs)
set(MATRIX_STRUCTS_TAG c24cb9b38312dfa24b33413847e3238600c678cd)
set(MATRIX_STRUCTS_TAG 3a052a95c555ce3ae12b8a2e0508e8bb73266fa1)
set(MTXCLIENT_URL https://github.com/mujx/mtxclient)
set(MTXCLIENT_TAG 73491268f94ddeb606284836bb5f512d11b0e249)

47
include/Cache.h
View File

@ -19,8 +19,10 @@
#include <boost/optional.hpp>
#include <QDateTime>
#include <QDir>
#include <QImage>
#include <QString>
#include <json.hpp>
#include <lmdb++.h>
@ -46,9 +48,24 @@ struct SearchResult
QString display_name;
};
inline int
numeric_key_comparison(const MDB_val *a, const MDB_val *b)
{
auto lhs = std::stoul(std::string((char *)a->mv_data, a->mv_size));
auto rhs = std::stoul(std::string((char *)b->mv_data, b->mv_size));
if (lhs < rhs)
return 1;
else if (lhs == rhs)
return 0;
return -1;
}
Q_DECLARE_METATYPE(SearchResult)
Q_DECLARE_METATYPE(QVector<SearchResult>)
Q_DECLARE_METATYPE(RoomMember)
Q_DECLARE_METATYPE(mtx::responses::Timeline)
//! Used to uniquely identify a list of read receipts.
struct ReadReceiptKey
@ -70,6 +87,15 @@ from_json(const json &j, ReadReceiptKey &key)
key.room_id = j.at("room_id").get<std::string>();
}
struct DescInfo
{
QString username;
QString userid;
QString body;
QString timestamp;
QDateTime datetime;
};
//! UI info associated with a room.
struct RoomInfo
{
@ -86,6 +112,8 @@ struct RoomInfo
//! Who can access to the room.
JoinRule join_rule = JoinRule::Public;
bool guest_access = false;
//! Metadata describing the last message in the timeline.
DescInfo msgInfo;
};
inline void
@ -289,6 +317,8 @@ public:
bool isFormatValid();
void setCurrentFormat();
std::map<QString, mtx::responses::Timeline> roomMessages();
//! Retrieve all the user ids from a room.
std::vector<std::string> roomMembers(const std::string &room_id);
@ -402,6 +432,13 @@ private:
QString getInviteRoomTopic(lmdb::txn &txn, lmdb::dbi &statesdb);
QString getInviteRoomAvatarUrl(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &membersdb);
DescInfo getLastMessageInfo(lmdb::txn &txn, const std::string &room_id);
void saveTimelineMessages(lmdb::txn &txn,
const std::string &room_id,
const mtx::responses::Timeline &res);
mtx::responses::Timeline getTimelineMessages(lmdb::txn &txn, const std::string &room_id);
//! Remove a room from the cache.
// void removeLeftRoom(lmdb::txn &txn, const std::string &room_id);
template<class T>
@ -500,6 +537,7 @@ private:
mpark::holds_alternative<StateEvent<HistoryVisibility>>(e) ||
mpark::holds_alternative<StateEvent<JoinRules>>(e) ||
mpark::holds_alternative<StateEvent<Name>>(e) ||
mpark::holds_alternative<StateEvent<Member>>(e) ||
mpark::holds_alternative<StateEvent<PowerLevels>>(e) ||
mpark::holds_alternative<StateEvent<Topic>>(e);
}
@ -544,6 +582,15 @@ private:
}
}
lmdb::dbi getMessagesDb(lmdb::txn &txn, const std::string &room_id)
{
auto db =
lmdb::dbi::open(txn, std::string(room_id + "/messages").c_str(), MDB_CREATE);
lmdb::dbi_set_compare(txn, db, numeric_key_comparison);
return db;
}
lmdb::dbi getInviteStatesDb(lmdb::txn &txn, const std::string &room_id)
{
return lmdb::dbi::open(

6
include/ChatPage.h
View File

@ -108,7 +108,6 @@ signals:
void showLoginPage(const QString &msg);
void showUserSettingsPage();
void showOverlayProgressBar();
void startConsesusTimer();
void removeTimelineEvent(const QString &room_id, const QString &event_id);
@ -124,7 +123,7 @@ signals:
void initializeRoomList(QMap<QString, RoomInfo>);
void initializeViews(const mtx::responses::Rooms &rooms);
void initializeEmptyViews(const std::vector<std::string> &rooms);
void initializeEmptyViews(const std::map<QString, mtx::responses::Timeline> &msgs);
void syncUI(const mtx::responses::Rooms &rooms);
void syncRoomlist(const std::map<QString, RoomInfo> &updates);
void syncTopBar(const std::map<QString, RoomInfo> &updates);
@ -206,9 +205,6 @@ private:
TextInputWidget *text_input_;
TypingDisplay *typingDisplay_;
// Safety net if consensus is not possible or too slow.
QTimer *showContentTimer_;
QTimer *consensusTimer_;
QTimer connectivityTimer_;
std::atomic_bool isConnected_;

11
include/RoomInfoListItem.h
View File

@ -22,20 +22,11 @@
#include <QSharedPointer>
#include <QWidget>
#include "Cache.h"
#include <mtx/responses.hpp>
class Menu;
class RippleOverlay;
struct RoomInfo;
struct DescInfo
{
QString username;
QString userid;
QString body;
QString timestamp;
QDateTime datetime;
};
class RoomInfoListItem : public QWidget
{

31
include/Utils.h
View File

@ -41,14 +41,15 @@ template<class T>
QString
messageDescription(const QString &username = "", const QString &body = "")
{
using Audio = mtx::events::RoomEvent<mtx::events::msg::Audio>;
using Emote = mtx::events::RoomEvent<mtx::events::msg::Emote>;
using File = mtx::events::RoomEvent<mtx::events::msg::File>;
using Image = mtx::events::RoomEvent<mtx::events::msg::Image>;
using Notice = mtx::events::RoomEvent<mtx::events::msg::Notice>;
using Sticker = mtx::events::Sticker;
using Text = mtx::events::RoomEvent<mtx::events::msg::Text>;
using Video = mtx::events::RoomEvent<mtx::events::msg::Video>;
using Audio = mtx::events::RoomEvent<mtx::events::msg::Audio>;
using Emote = mtx::events::RoomEvent<mtx::events::msg::Emote>;
using File = mtx::events::RoomEvent<mtx::events::msg::File>;
using Image = mtx::events::RoomEvent<mtx::events::msg::Image>;
using Notice = mtx::events::RoomEvent<mtx::events::msg::Notice>;
using Sticker = mtx::events::Sticker;
using Text = mtx::events::RoomEvent<mtx::events::msg::Text>;
using Video = mtx::events::RoomEvent<mtx::events::msg::Video>;
using Encrypted = mtx::events::EncryptedEvent<mtx::events::msg::Encrypted>;
if (std::is_same<T, AudioItem>::value || std::is_same<T, Audio>::value)
return QString("sent an audio clip");
@ -66,6 +67,8 @@ messageDescription(const QString &username = "", const QString &body = "")
return QString(": %1").arg(body);
else if (std::is_same<T, Emote>::value)
return QString("* %1 %2").arg(username).arg(body);
else if (std::is_same<T, Encrypted>::value)
return QString("sent an encrypted message");
}
template<class T, class Event>
@ -135,6 +138,18 @@ erase_if(ContainerT &items, const PredicateT &predicate)
}
}
inline uint64_t
event_timestamp(const mtx::events::collections::TimelineEvents &event)
{
return mpark::visit([](auto msg) { return msg.origin_server_ts; }, event);
}
inline nlohmann::json
serialize_event(const mtx::events::collections::TimelineEvents &event)
{
return mpark::visit([](auto msg) { return json(msg); }, event);
}
inline mtx::events::EventType
event_type(const mtx::events::collections::TimelineEvents &event)
{

1
include/timeline/TimelineView.h
View File

@ -158,6 +158,7 @@ public:
//! Remove an item from the timeline with the given Event ID.
void removeEvent(const QString &event_id);
void setPrevBatchToken(const QString &token) { prev_batch_token_ = token; }
public slots:
void sliderRangeChanged(int min, int max);

2
include/timeline/TimelineViewManager.h
View File

@ -27,6 +27,7 @@ class QFile;
class RoomInfoListItem;
class TimelineView;
struct DescInfo;
struct SavedMessages;
class TimelineViewManager : public QStackedWidget
{
@ -57,6 +58,7 @@ signals:
public slots:
void removeTimelineEvent(const QString &room_id, const QString &event_id);
void initWithMessages(const std::map<QString, mtx::responses::Timeline> &msgs);
void setHistoryView(const QString &room_id);
void queueTextMessage(const QString &msg);

121
src/Cache.cc
View File

@ -21,8 +21,10 @@
#include <QByteArray>
#include <QFile>
#include <QHash>
#include <QSettings>
#include <QStandardPaths>
#include <mtx/responses/common.hpp>
#include <variant.hpp>
#include "Cache.h"
@ -38,6 +40,8 @@ static const lmdb::val NEXT_BATCH_KEY("next_batch");
static const lmdb::val OLM_ACCOUNT_KEY("olm_account");
static const lmdb::val CACHE_FORMAT_VERSION_KEY("cache_format_version");
constexpr size_t MAX_RESTORED_MESSAGES = 30;
//! Cache databases and their format.
//!
//! Contains UI information for the joined rooms. (i.e name, topic, avatar url etc).
@ -85,6 +89,7 @@ init(const QString &user_id)
qRegisterMetaType<RoomInfo>();
qRegisterMetaType<QMap<QString, RoomInfo>>();
qRegisterMetaType<std::map<QString, RoomInfo>>();
qRegisterMetaType<std::map<QString, mtx::responses::Timeline>>();
instance_ = std::make_unique<Cache>(user_id);
}
@ -744,6 +749,8 @@ Cache::saveState(const mtx::responses::Sync &res)
saveStateEvents(txn, statesdb, membersdb, room.first, room.second.state.events);
saveStateEvents(txn, statesdb, membersdb, room.first, room.second.timeline.events);
saveTimelineMessages(txn, room.first, room.second.timeline);
RoomInfo updatedInfo;
updatedInfo.name = getRoomName(txn, statesdb, membersdb).toStdString();
updatedInfo.topic = getRoomTopic(txn, statesdb).toStdString();
@ -944,6 +951,57 @@ Cache::getRoomInfo(const std::vector<std::string> &rooms)
return room_info;
}
std::map<QString, mtx::responses::Timeline>
Cache::roomMessages()
{
auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY);
std::map<QString, mtx::responses::Timeline> msgs;
std::string room_id, unused;
auto roomsCursor = lmdb::cursor::open(txn, roomsDb_);
while (roomsCursor.get(room_id, unused, MDB_NEXT))
msgs.emplace(QString::fromStdString(room_id), getTimelineMessages(txn, room_id));
roomsCursor.close();
txn.commit();
return msgs;
}
mtx::responses::Timeline
Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id)
{
auto db = getMessagesDb(txn, room_id);
mtx::responses::Timeline timeline;
std::string timestamp, msg;
auto cursor = lmdb::cursor::open(txn, db);
size_t index = 0;
while (cursor.get(timestamp, msg, MDB_NEXT) && index < MAX_RESTORED_MESSAGES) {
auto obj = json::parse(msg);
if (obj.count("event") == 0 || obj.count("token") == 0)
continue;
mtx::events::collections::TimelineEvents event;
mtx::events::collections::from_json(obj.at("event"), event);
index += 1;
timeline.events.push_back(event);
timeline.prev_batch = obj.at("token").get<std::string>();
}
cursor.close();
std::reverse(timeline.events.begin(), timeline.events.end());
return timeline;
}
QMap<QString, RoomInfo>
Cache::roomInfo(bool withInvites)
{
@ -959,6 +1017,8 @@ Cache::roomInfo(bool withInvites)
while (roomsCursor.get(room_id, room_data, MDB_NEXT)) {
RoomInfo tmp = json::parse(std::move(room_data));
tmp.member_count = getMembersDb(txn, room_id).size(txn);
tmp.msgInfo = getLastMessageInfo(txn, room_id);
result.insert(QString::fromStdString(std::move(room_id)), std::move(tmp));
}
roomsCursor.close();
@ -979,6 +1039,38 @@ Cache::roomInfo(bool withInvites)
return result;
}
DescInfo
Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id)
{
auto db = getMessagesDb(txn, room_id);
if (db.size(txn) == 0)
return DescInfo{};
std::string timestamp, msg;
QSettings settings;
auto local_user = settings.value("auth/user_id").toString();
auto cursor = lmdb::cursor::open(txn, db);
while (cursor.get(timestamp, msg, MDB_NEXT)) {
auto obj = json::parse(msg);
if (obj.count("event") == 0)
continue;
mtx::events::collections::TimelineEvents event;
mtx::events::collections::from_json(obj.at("event"), event);
cursor.close();
return utils::getMessageDescription(
event, local_user, QString::fromStdString(room_id));
}
cursor.close();
return DescInfo{};
}
std::map<QString, bool>
Cache::invites()
{
@ -1512,6 +1604,35 @@ Cache::getMembers(const std::string &room_id, std::size_t startIndex, std::size_
return members;
}
void
Cache::saveTimelineMessages(lmdb::txn &txn,
const std::string &room_id,
const mtx::responses::Timeline &res)
{
auto db = getMessagesDb(txn, room_id);
using namespace mtx::events;
using namespace mtx::events::state;
for (const auto &e : res.events) {
if (isStateEvent(e))
continue;
if (mpark::holds_alternative<RedactionEvent<msg::Redaction>>(e))
continue;
json obj = json::object();
obj["event"] = utils::serialize_event(e);
obj["token"] = res.prev_batch;
lmdb::dbi_put(txn,
db,
lmdb::val(std::to_string(utils::event_timestamp(e))),
lmdb::val(obj.dump()));
}
}
void
Cache::markSentNotification(const std::string &event_id)
{

41
src/ChatPage.cc
View File

@ -516,23 +516,6 @@ ChatPage::ChatPage(QSharedPointer<UserSettings> userSettings, QWidget *parent)
connect(this, &ChatPage::leftRoom, this, &ChatPage::removeRoom);
connect(this, &ChatPage::notificationsRetrieved, this, &ChatPage::sendDesktopNotifications);
showContentTimer_ = new QTimer(this);
showContentTimer_->setSingleShot(true);
connect(showContentTimer_, &QTimer::timeout, this, [this]() {
consensusTimer_->stop();
emit contentLoaded();
});
consensusTimer_ = new QTimer(this);
connect(consensusTimer_, &QTimer::timeout, this, [this]() {
if (view_manager_->hasLoaded()) {
// Remove the spinner overlay.
emit contentLoaded();
showContentTimer_->stop();
consensusTimer_->stop();
}
});
connect(communitiesList_,
&CommunitiesList::communityChanged,
this,
@ -552,20 +535,15 @@ ChatPage::ChatPage(QSharedPointer<UserSettings> userSettings, QWidget *parent)
this,
&ChatPage::setGroupViewState);
connect(this, &ChatPage::startConsesusTimer, this, [this]() {
consensusTimer_->start(CONSENSUS_TIMEOUT);
showContentTimer_->start(SHOW_CONTENT_TIMEOUT);
});
connect(this, &ChatPage::initializeRoomList, room_list_, &RoomList::initialize);
connect(this,
&ChatPage::initializeViews,
view_manager_,
[this](const mtx::responses::Rooms &rooms) { view_manager_->initialize(rooms); });
connect(
this,
&ChatPage::initializeEmptyViews,
this,
[this](const std::vector<std::string> &rooms) { view_manager_->initialize(rooms); });
connect(this,
&ChatPage::initializeEmptyViews,
view_manager_,
&TimelineViewManager::initWithMessages);
connect(this, &ChatPage::syncUI, this, [this](const mtx::responses::Rooms &rooms) {
try {
room_list_->cleanupInvites(cache::client()->invites());
@ -817,6 +795,8 @@ ChatPage::showUnreadMessageNotification(int count)
void
ChatPage::loadStateFromCache()
{
emit contentLoaded();
nhlog::db()->info("restoring state from cache");
getProfileInfo();
@ -829,8 +809,9 @@ ChatPage::loadStateFromCache()
cache::client()->populateMembers();
emit initializeEmptyViews(cache::client()->joinedRooms());
emit initializeEmptyViews(cache::client()->roomMessages());
emit initializeRoomList(cache::client()->roomInfo());
} catch (const mtx::crypto::olm_exception &e) {
nhlog::crypto()->critical("failed to restore olm account: {}", e.what());
emit dropToLoginPageCb(
@ -841,6 +822,9 @@ ChatPage::loadStateFromCache()
emit dropToLoginPageCb(
tr("Failed to restore save data. Please login again."));
return;
} catch (const json::exception &e) {
nhlog::db()->critical("failed to parse cache data: {}", e.what());
return;
}
nhlog::crypto()->info("ed25519 : {}", olm::client()->identity_keys().ed25519);
@ -848,9 +832,6 @@ ChatPage::loadStateFromCache()
// Start receiving events.
emit trySyncCb();
// Check periodically if the timelines have been loaded.
emit startConsesusTimer();
});
}

3
src/MainWindow.cc
View File

@ -234,7 +234,8 @@ MainWindow::showChatPage()
showOverlayProgressBar();
QTimer::singleShot(100, this, [this]() { pageStack_->setCurrentWidget(chat_page_); });
welcome_page_->hide();
pageStack_->setCurrentWidget(chat_page_);
login_page_->reset();
chat_page_->bootstrap(userid, homeserver, token);

8
src/RoomList.cc
View File

@ -15,6 +15,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <QApplication>
#include <QBuffer>
#include <QObject>
#include <QTimer>
@ -171,6 +172,8 @@ RoomList::initialize(const QMap<QString, RoomInfo> &info)
rooms_.clear();
setUpdatesEnabled(false);
for (auto it = info.begin(); it != info.end(); it++) {
if (it.value().is_invite)
addInvitedRoom(it.key(), it.value());
@ -178,6 +181,11 @@ RoomList::initialize(const QMap<QString, RoomInfo> &info)
addRoom(it.key(), it.value());
}
for (auto it = info.begin(); it != info.end(); it++)
updateRoomDescription(it.key(), it.value().msgInfo);
setUpdatesEnabled(true);
if (rooms_.empty())
return;

36
src/Utils.cc
View File

@ -28,13 +28,14 @@ utils::getMessageDescription(const TimelineEvent &event,
const QString &localUser,
const QString &room_id)
{
using Audio = mtx::events::RoomEvent<mtx::events::msg::Audio>;
using Emote = mtx::events::RoomEvent<mtx::events::msg::Emote>;
using File = mtx::events::RoomEvent<mtx::events::msg::File>;
using Image = mtx::events::RoomEvent<mtx::events::msg::Image>;
using Notice = mtx::events::RoomEvent<mtx::events::msg::Notice>;
using Text = mtx::events::RoomEvent<mtx::events::msg::Text>;
using Video = mtx::events::RoomEvent<mtx::events::msg::Video>;
using Audio = mtx::events::RoomEvent<mtx::events::msg::Audio>;
using Emote = mtx::events::RoomEvent<mtx::events::msg::Emote>;
using File = mtx::events::RoomEvent<mtx::events::msg::File>;
using Image = mtx::events::RoomEvent<mtx::events::msg::Image>;
using Notice = mtx::events::RoomEvent<mtx::events::msg::Notice>;
using Text = mtx::events::RoomEvent<mtx::events::msg::Text>;
using Video = mtx::events::RoomEvent<mtx::events::msg::Video>;
using Encrypted = mtx::events::EncryptedEvent<mtx::events::msg::Encrypted>;
if (mpark::holds_alternative<Audio>(event)) {
return createDescriptionInfo<Audio>(event, localUser, room_id);
@ -52,6 +53,27 @@ utils::getMessageDescription(const TimelineEvent &event,
return createDescriptionInfo<Video>(event, localUser, room_id);
} else if (mpark::holds_alternative<mtx::events::Sticker>(event)) {
return createDescriptionInfo<mtx::events::Sticker>(event, localUser, room_id);
} else if (mpark::holds_alternative<Encrypted>(event)) {
const auto msg = mpark::get<Encrypted>(event);
const auto sender = QString::fromStdString(msg.sender);
const auto username = Cache::displayName(room_id, sender);
const auto ts = QDateTime::fromMSecsSinceEpoch(msg.origin_server_ts);
DescInfo info;
if (sender == localUser)
info.username = "You";
else
info.username = username;
info.userid = sender;
info.body = QString(" %1").arg(messageDescription<Encrypted>());
info.timestamp = utils::descriptiveTime(ts);
info.datetime = ts;
return info;
} else {
std::cout << "type not found: " << serialize_event(event).dump(2) << '\n';
}
return DescInfo{};

5
src/timeline/TimelineView.cc
View File

@ -378,7 +378,7 @@ TimelineView::renderBottomEvents(const std::vector<TimelineEvent> &events)
// Prevent blocking of the event-loop
// by calling processEvents every 10 items we render.
if (counter % 10 == 0)
if (counter % 4 == 0)
QApplication::processEvents();
}
}
@ -1035,7 +1035,8 @@ TimelineEvent
TimelineView::findLastViewableEvent(const std::vector<TimelineEvent> &events)
{
auto it = std::find_if(events.rbegin(), events.rend(), [](const auto &event) {
return mtx::events::EventType::RoomMessage == utils::event_type(event);
return (mtx::events::EventType::RoomMessage == utils::event_type(event)) ||
(mtx::events::EventType::RoomEncrypted == utils::event_type(event));
});
return (it == std::rend(events)) ? events.back() : *it;

22
src/timeline/TimelineViewManager.cc
View File

@ -21,6 +21,7 @@
#include <QFileInfo>
#include <QSettings>
#include "Cache.h"
#include "Logging.hpp"
#include "timeline/TimelineView.h"
#include "timeline/TimelineViewManager.h"
@ -146,6 +147,27 @@ TimelineViewManager::initialize(const mtx::responses::Rooms &rooms)
sync(rooms);
}
void
TimelineViewManager::initWithMessages(const std::map<QString, mtx::responses::Timeline> &msgs)
{
for (auto it = msgs.cbegin(); it != msgs.cend(); ++it) {
if (timelineViewExists(it->first))
return;
// Create a history view with the room events.
TimelineView *view = new TimelineView(it->second, it->first);
views_.emplace(it->first, QSharedPointer<TimelineView>(view));
connect(view,
&TimelineView::updateLastTimelineMessage,
this,
&TimelineViewManager::updateRoomsLastMessage);
// Add the view in the widget stack.
addWidget(view);
}
}
void
TimelineViewManager::initialize(const std::vector<std::string> &rooms)
{