Remove sender key from megolm session index

implements MSC3700

see https://github.com/matrix-org/matrix-spec-proposals/pull/3700

src/Cache.cpp

@@ -37,7 +37,7 @@
 
 //! Should be changed when a breaking change occurs in the cache format.
 //! This will reset client's data.
-static const std::string CURRENT_CACHE_FORMAT_VERSION("2021.08.31");
+static const std::string CURRENT_CACHE_FORMAT_VERSION{"2022.04.08"};
 
 //! Keys used for the DB
 static const std::string_view NEXT_BATCH_KEY("next_batch");
@@ -165,7 +165,6 @@ Cache::isHiddenEvent(lmdb::txn &txn,
         MegolmSessionIndex index;
         index.room_id    = room_id;
         index.session_id = encryptedEvent->content.session_id;
-        index.sender_key = encryptedEvent->content.sender_key;
 
         auto result = olm::decryptEvent(index, *encryptedEvent, true);
         if (!result.error)
@@ -600,8 +599,25 @@ Cache::exportSessionKeys()
             continue;
         }
 
+        try {
+            using namespace mtx::crypto;
+
+            std::string_view v;
+            if (megolmSessionDataDb_.get(txn, json(index).dump(), v)) {
+                auto data           = nlohmann::json::parse(v).get<GroupSessionData>();
+                exported.sender_key = data.sender_key;
+                if (!data.sender_claimed_ed25519_key.empty())
+                    exported.sender_claimed_keys["ed25519"] = data.sender_claimed_ed25519_key;
+                exported.forwarding_curve25519_key_chain = data.forwarding_curve25519_key_chain;
+            }
+
+            continue;
+        } catch (std::exception &e) {
+            nhlog::db()->error("Failed to retrieve Megolm Session Data: {}", e.what());
+            continue;
+        }
+
         exported.room_id     = index.room_id;
-        exported.sender_key  = index.sender_key;
         exported.session_id  = index.session_id;
         exported.session_key = export_session(saved_session.get(), -1);
 
@@ -620,9 +636,9 @@ Cache::importSessionKeys(const mtx::crypto::ExportedSessionKeys &keys)
         MegolmSessionIndex index;
         index.room_id    = s.room_id;
         index.session_id = s.session_id;
-        index.sender_key = s.sender_key;
 
         GroupSessionData data{};
+        data.sender_key                      = s.sender_key;
         data.forwarding_curve25519_key_chain = s.forwarding_curve25519_key_chain;
         if (s.sender_claimed_keys.count("ed25519"))
             data.sender_claimed_ed25519_key = s.sender_claimed_keys.at("ed25519");
@@ -717,7 +733,6 @@ Cache::updateOutboundMegolmSession(const std::string &room_id,
     data.message_index    = olm_outbound_group_session_message_index(ptr.get());
     MegolmSessionIndex index;
     index.room_id    = room_id;
-    index.sender_key = olm::client()->identity_keys().ed25519;
     index.session_id = mtx::crypto::session_id(ptr.get());
 
     // Save the updated pickled data for the session.
@@ -758,7 +773,6 @@ Cache::saveOutboundMegolmSession(const std::string &room_id,
     data.message_index    = olm_outbound_group_session_message_index(session.get());
     MegolmSessionIndex index;
     index.room_id    = room_id;
-    index.sender_key = olm::client()->identity_keys().ed25519;
     index.session_id = mtx::crypto::session_id(session.get());
 
     json j;
@@ -799,7 +813,6 @@ Cache::getOutboundMegolmSession(const std::string &room_id)
 
         MegolmSessionIndex index;
         index.room_id    = room_id;
-        index.sender_key = olm::client()->identity_keys().ed25519;
         index.session_id = mtx::crypto::session_id(ref.session.get());
 
         if (megolmSessionDataDb_.get(txn, json(index).dump(), value)) {
@@ -1290,6 +1303,56 @@ Cache::runMigrations()
            storeSecret("pickle_secret", "secret", true);
            return true;
        }},
+      {"2022.04.08",
+       [this]() {
+           try {
+               auto txn = lmdb::txn::begin(env_, nullptr);
+               auto inboundMegolmSessionDb =
+                 lmdb::dbi::open(txn, INBOUND_MEGOLM_SESSIONS_DB, MDB_CREATE);
+               auto outboundMegolmSessionDb =
+                 lmdb::dbi::open(txn, OUTBOUND_MEGOLM_SESSIONS_DB, MDB_CREATE);
+               auto megolmSessionDataDb = lmdb::dbi::open(txn, MEGOLM_SESSIONS_DATA_DB, MDB_CREATE);
+               try {
+                   outboundMegolmSessionDb.drop(txn, false);
+               } catch (std::exception &e) {
+                   nhlog::db()->warn("Failed to drop outbound sessions: {}", e.what());
+               }
+
+               std::string_view key, value;
+               auto cursor = lmdb::cursor::open(txn, inboundMegolmSessionDb);
+               std::map<std::string, std::string> inboundSessions;
+               std::map<std::string, std::string> megolmSessionData;
+               while (cursor.get(key, value, MDB_NEXT)) {
+                   auto indexVal   = nlohmann::json::parse(key);
+                   auto sender_key = indexVal["sender_key"];
+                   indexVal.erase("sender_key");
+
+                   std::string_view dataVal;
+                   bool res = megolmSessionDataDb.get(txn, key, dataVal);
+                   if (res) {
+                       auto data                          = nlohmann::json::parse(dataVal);
+                       data["sender_key"]                 = sender_key;
+                       inboundSessions[indexVal.dump()]   = std::string(value);
+                       megolmSessionData[indexVal.dump()] = data.dump();
+                   }
+               }
+               inboundMegolmSessionDb.drop(txn, false);
+               megolmSessionDataDb.drop(txn, false);
+
+               for (const auto &[k, v] : inboundSessions) {
+                   inboundMegolmSessionDb.put(txn, k, v);
+               }
+               for (const auto &[k, v] : megolmSessionData) {
+                   megolmSessionDataDb.put(txn, k, v);
+               }
+               txn.commit();
+               return true;
+           } catch (std::exception &e) {
+               nhlog::db()->warn(
+                 "Failed to migrate stored megolm session to have no sender key: {}", e.what());
+               return false;
+           }
+       }},
     };
 
     nhlog::db()->info("Running migrations, this may take a while!");
@@ -4709,6 +4772,7 @@ to_json(nlohmann::json &obj, const GroupSessionData &msg)
     obj["ts"]            = msg.timestamp;
     obj["trust"]         = msg.trusted;
 
+    obj["sender_key"]                      = msg.sender_key;
     obj["sender_claimed_ed25519_key"]      = msg.sender_claimed_ed25519_key;
     obj["forwarding_curve25519_key_chain"] = msg.forwarding_curve25519_key_chain;
 
@@ -4724,6 +4788,7 @@ from_json(const nlohmann::json &obj, GroupSessionData &msg)
     msg.timestamp     = obj.value("ts", 0ULL);
     msg.trusted       = obj.value("trust", true);
 
+    msg.sender_key                 = obj.value("sender_key", "");
     msg.sender_claimed_ed25519_key = obj.value("sender_claimed_ed25519_key", "");
     msg.forwarding_curve25519_key_chain =
       obj.value("forwarding_curve25519_key_chain", std::vector<std::string>{});
@@ -4752,7 +4817,6 @@ to_json(nlohmann::json &obj, const MegolmSessionIndex &msg)
 {
     obj["room_id"]    = msg.room_id;
     obj["session_id"] = msg.session_id;
-    obj["sender_key"] = msg.sender_key;
 }
 
 void
@@ -4760,7 +4824,6 @@ from_json(const nlohmann::json &obj, MegolmSessionIndex &msg)
 {
     msg.room_id    = obj.at("room_id");
     msg.session_id = obj.at("session_id");
-    msg.sender_key = obj.at("sender_key");
 }
 
 void

src/CacheCryptoStructs.h

@@ -53,6 +53,9 @@ struct GroupSessionData
     // TODO(Nico): What about forwards? They might come from key backup?
     bool trusted = true;
 
+    // the original 25519 key
+    std::string sender_key;
+
     std::string sender_claimed_ed25519_key;
     std::vector<std::string> forwarding_curve25519_key_chain;
 
@@ -93,15 +96,12 @@ struct MegolmSessionIndex
     MegolmSessionIndex(std::string room_id_, const mtx::events::msg::Encrypted &e)
       : room_id(std::move(room_id_))
       , session_id(e.session_id)
-      , sender_key(e.sender_key)
     {}
 
     //! The room in which this session exists.
     std::string room_id;
     //! The session_id of the megolm session.
     std::string session_id;
-    //! The curve25519 public key of the sender.
-    std::string sender_key;
 };
 
 void

src/encryption/Olm.cpp

@@ -614,6 +614,7 @@ encrypt_group_message(const std::string &room_id, const std::string &device_id,
         session_data.message_index              = 0;
         session_data.timestamp                  = QDateTime::currentMSecsSinceEpoch();
         session_data.sender_claimed_ed25519_key = olm::client()->identity_keys().ed25519;
+        session_data.sender_key                 = olm::client()->identity_keys().curve25519;
 
         sendSessionTo.clear();
 
@@ -635,7 +636,6 @@ encrypt_group_message(const std::string &room_id, const std::string &device_id,
             MegolmSessionIndex index;
             index.room_id       = room_id;
             index.session_id    = session_id;
-            index.sender_key    = olm::client()->identity_keys().curve25519;
             auto megolm_session = olm::client()->init_inbound_group_session(session_key);
             backup_session_key(index, session_data, megolm_session);
             cache::saveInboundMegolmSession(index, std::move(megolm_session), session_data);
@@ -734,12 +734,12 @@ create_inbound_megolm_session(const mtx::events::DeviceEvent<mtx::events::msg::R
     MegolmSessionIndex index;
     index.room_id    = roomKey.content.room_id;
     index.session_id = roomKey.content.session_id;
-    index.sender_key = sender_key;
 
     try {
         GroupSessionData data{};
         data.forwarding_curve25519_key_chain = {sender_key};
         data.sender_claimed_ed25519_key      = sender_ed25519;
+        data.sender_key                      = sender_key;
 
         auto megolm_session =
           olm::client()->init_inbound_group_session(roomKey.content.session_key);
@@ -766,7 +766,6 @@ import_inbound_megolm_session(
     MegolmSessionIndex index;
     index.room_id    = roomKey.content.room_id;
     index.session_id = roomKey.content.session_id;
-    index.sender_key = roomKey.content.sender_key;
 
     try {
         auto megolm_session =
@@ -775,6 +774,7 @@ import_inbound_megolm_session(
         GroupSessionData data{};
         data.forwarding_curve25519_key_chain = roomKey.content.forwarding_curve25519_key_chain;
         data.sender_claimed_ed25519_key      = roomKey.content.sender_claimed_ed25519_key;
+        data.sender_key                      = roomKey.content.sender_key;
         // may have come from online key backup, so we can't trust it...
         data.trusted = false;
         // if we got it forwarded from the sender, assume it is trusted. They may still have
@@ -832,7 +832,7 @@ backup_session_key(const MegolmSessionIndex &idx,
         sessionData.algorithm                       = mtx::crypto::MEGOLM_ALGO;
         sessionData.forwarding_curve25519_key_chain = data.forwarding_curve25519_key_chain;
         sessionData.sender_claimed_keys["ed25519"]  = data.sender_claimed_ed25519_key;
-        sessionData.sender_key                      = idx.sender_key;
+        sessionData.sender_key                      = data.sender_key;
         sessionData.session_key = mtx::crypto::export_session(session.get(), -1);
 
         auto encrypt_session = mtx::crypto::encrypt_session(sessionData, public_key);
@@ -920,11 +920,11 @@ lookup_keybackup(const std::string room, const std::string session_id)
               MegolmSessionIndex index;
               index.room_id    = room;
               index.session_id = session_id;
-              index.sender_key = session.sender_key;
 
               GroupSessionData data{};
               data.forwarding_curve25519_key_chain = session.forwarding_curve25519_key_chain;
               data.sender_claimed_ed25519_key      = session.sender_claimed_keys["ed25519"];
+              data.sender_key                      = session.sender_key;
               // online key backup can't be trusted, because anyone can upload to it.
               data.trusted = false;
 
@@ -982,7 +982,7 @@ send_key_request_for(mtx::events::EncryptedEvent<mtx::events::msg::Encrypted> e,
     nhlog::crypto()->debug("m.room_key_request: {}", json(request).dump(2));
 
     std::map<mtx::identifiers::User, std::map<std::string, decltype(request)>> body;
-    body[mtx::identifiers::parse<mtx::identifiers::User>(e.sender)][e.content.device_id] = request;
+    body[mtx::identifiers::parse<mtx::identifiers::User>(e.sender)]["*"] = request;
     body[http::client()->user_id()]["*"]                                 = request;
 
     http::client()->send_to_device(
@@ -1011,24 +1011,10 @@ handle_key_request_message(const mtx::events::DeviceEvent<mtx::events::msg::KeyR
         return;
     }
 
-    // Check if we were the sender of the session being requested (unless it is actually us
-    // requesting the session).
-    if (req.sender != http::client()->user_id().to_string() &&
-        req.content.sender_key != olm::client()->identity_keys().curve25519) {
-        nhlog::crypto()->debug(
-          "ignoring key request {} because we did not create the requested session: "
-          "\nrequested({}) ours({})",
-          req.content.request_id,
-          req.content.sender_key,
-          olm::client()->identity_keys().curve25519);
-        return;
-    }
-
     // Check that the requested session_id and the one we have saved match.
     MegolmSessionIndex index{};
     index.room_id    = req.content.room_id;
     index.session_id = req.content.session_id;
-    index.sender_key = req.content.sender_key;
 
     // Check if we have the keys for the requested session.
     auto sessionData = cache::getMegolmSessionData(index);
@@ -1037,6 +1023,19 @@ handle_key_request_message(const mtx::events::DeviceEvent<mtx::events::msg::KeyR
         return;
     }
 
+    // Check if we were the sender of the session being requested (unless it is actually us
+    // requesting the session).
+    if (req.sender != http::client()->user_id().to_string() &&
+        sessionData->sender_key != olm::client()->identity_keys().curve25519) {
+        nhlog::crypto()->debug(
+          "ignoring key request {} because we did not create the requested session: "
+          "\nrequested({}) ours({})",
+          req.content.request_id,
+          sessionData->sender_key,
+          olm::client()->identity_keys().curve25519);
+        return;
+    }
+
     const auto session = cache::getInboundMegolmSession(index);
     if (!session) {
         nhlog::crypto()->warn("No session with id {} in db", req.content.session_id);
@@ -1098,7 +1097,7 @@ handle_key_request_message(const mtx::events::DeviceEvent<mtx::events::msg::KeyR
         forward_key.room_id     = index.room_id;
         forward_key.session_id  = index.session_id;
         forward_key.session_key = session_key;
-        forward_key.sender_key  = index.sender_key;
+        forward_key.sender_key  = sessionData->sender_key;
 
         // TODO(Nico): Figure out if this is correct
         forward_key.sender_claimed_ed25519_key      = sessionData->sender_claimed_ed25519_key;
@@ -1196,8 +1195,9 @@ calculate_trust(const std::string &user_id, const MegolmSessionIndex &index)
     auto megolmData          = cache::client()->getMegolmSessionData(index);
     crypto::Trust trustlevel = crypto::Trust::Unverified;
 
-    if (megolmData && megolmData->trusted && status.verified_device_keys.count(index.sender_key))
+    if (megolmData && megolmData->trusted &&
-        trustlevel = status.verified_device_keys.at(index.sender_key);
+        status.verified_device_keys.count(megolmData->sender_key))
+        trustlevel = status.verified_device_keys.at(megolmData->sender_key);
 
     return trustlevel;
 }

src/timeline/EventStore.cpp

@@ -691,17 +691,15 @@ EventStore::decryptEvent(const IdIndex &idx,
             break;
         }
         case olm::DecryptionErrorCode::DbError:
-            nhlog::db()->critical("failed to retrieve megolm session with index ({}, {}, {})",
+            nhlog::db()->critical("failed to retrieve megolm session with index ({}, {})",
                                   index.room_id,
                                   index.session_id,
-                                  index.sender_key,
                                   decryptionResult.error_message.value_or(""));
             break;
         case olm::DecryptionErrorCode::DecryptionFailed:
-            nhlog::crypto()->critical("failed to decrypt message with index ({}, {}, {}): {}",
+            nhlog::crypto()->critical("failed to decrypt message with index ({},  {}): {}",
                                       index.room_id,
                                       index.session_id,
-                                      index.sender_key,
                                       decryptionResult.error_message.value_or(""));
             break;
         case olm::DecryptionErrorCode::ParsingFailed:
@@ -710,7 +708,7 @@ EventStore::decryptEvent(const IdIndex &idx,
             nhlog::crypto()->critical("Reply attack while decryptiong event {} in room {} from {}!",
                                       e.event_id,
                                       room_id_,
-                                      index.sender_key);
+                                      e.sender);
             break;
         case olm::DecryptionErrorCode::NoError:
             // unreachable