From 17399da3994bd3195e816a513d55bdbea1253f57 Mon Sep 17 00:00:00 2001 From: imdef <63346462+imdef@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:55:29 +0000 Subject: [PATCH 01/18] Added example docker-compose.yml --- contrib/docker/docker-compose.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 contrib/docker/docker-compose.yml diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml new file mode 100644 index 00000000..bc365c53 --- /dev/null +++ b/contrib/docker/docker-compose.yml @@ -0,0 +1,13 @@ +services: + i2pd: + container_name: i2pd2 + image: purplei2p/i2pd + #optional + entrypoint: ["./entrypoint.sh", "--loglevel error"] + ports: + - 127.0.0.1:7656:7656 + - 127.0.0.1:7070:7070 + - 127.0.0.1:4444:4444 + volumes: + - /path/to/i2pd/data:/home/i2pd/data # make sure data directory and it's contents are owned by 100:65533 + - /path/to/i2pd/i2pd_certificates:/i2pd_certificates # make sure i2pd_certificates is owned by root:root and 755 permissions on the directory From 8f9874570a01698fe739b759feed4e0a8d67148b Mon Sep 17 00:00:00 2001 From: mittwerk Date: Mon, 28 Oct 2024 09:11:09 +0200 Subject: [PATCH 02/18] hardening iterator --- libi2pd_client/UDPTunnel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libi2pd_client/UDPTunnel.cpp b/libi2pd_client/UDPTunnel.cpp index cd17bbf0..9f7fe864 100644 --- a/libi2pd_client/UDPTunnel.cpp +++ b/libi2pd_client/UDPTunnel.cpp @@ -203,7 +203,7 @@ namespace client std::vector > sessions; std::lock_guard lock (m_SessionsMutex); - for (auto it: m_Sessions) + for (const auto &it: m_Sessions) { auto s = it.second; if (!s->m_Destination) continue; From 0e8d624d861bd08c705e8390f98609e3876d0acf Mon Sep 17 00:00:00 2001 From: orignal Date: Mon, 28 Oct 2024 08:38:04 -0400 Subject: [PATCH 03/18] move UpdatePacingTime out of loop --- libi2pd/Streaming.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libi2pd/Streaming.cpp b/libi2pd/Streaming.cpp index f9257974..966c172d 100644 --- a/libi2pd/Streaming.cpp +++ b/libi2pd/Streaming.cpp @@ -1276,10 +1276,12 @@ namespace stream else m_WindowSize += (m_WindowSize - (1 - PREV_SPEED_KEEP_TIME_COEFF)) / m_WindowSize; if (m_WindowSize > MAX_WINDOW_SIZE) m_WindowSize = MAX_WINDOW_SIZE; - m_WindowIncCounter --; - UpdatePacingTime (); + m_WindowIncCounter--; } + else + break; } + UpdatePacingTime (); } if (m_IsNAcked) ResendPacket (); From ec67f48d855c196669938a2da71064da8a501d4b Mon Sep 17 00:00:00 2001 From: orignal Date: Mon, 28 Oct 2024 08:46:01 -0400 Subject: [PATCH 04/18] fixed possible memory leak --- libi2pd/NTCP2.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libi2pd/NTCP2.cpp b/libi2pd/NTCP2.cpp index 728ac01d..a05469f1 100644 --- a/libi2pd/NTCP2.cpp +++ b/libi2pd/NTCP2.cpp @@ -1822,7 +1822,7 @@ namespace transport LogPrint(eLogError, "NTCP2: HTTP proxy write error ", ec.message()); }); - boost::asio::streambuf * readbuff = new boost::asio::streambuf; + auto readbuff = std::make_shared(); boost::asio::async_read_until(conn->GetSocket(), *readbuff, "\r\n\r\n", [readbuff, timer, conn] (const boost::system::error_code & ec, std::size_t transferred) { @@ -1842,7 +1842,6 @@ namespace transport { timer->cancel(); conn->ClientLogin(); - delete readbuff; return; } else @@ -1852,7 +1851,6 @@ namespace transport LogPrint(eLogError, "NTCP2: HTTP proxy gave malformed response"); timer->cancel(); conn->Terminate(); - delete readbuff; } }); break; From 23e66671c2fcd1fc11abd11f07ec18f93a346aab Mon Sep 17 00:00:00 2001 From: orignal Date: Mon, 28 Oct 2024 20:36:50 -0400 Subject: [PATCH 05/18] intermediate queue for transport sessions. use std::list instead std::vector for multiple I2NP messages --- libi2pd/NTCP2.cpp | 40 ++++++++++++++++++++++++++++---------- libi2pd/NTCP2.h | 9 ++++++--- libi2pd/NetDb.cpp | 2 +- libi2pd/SSU2Session.cpp | 25 +++++++++++++++++++----- libi2pd/SSU2Session.h | 6 ++++-- libi2pd/TransportSession.h | 8 ++++++-- libi2pd/Transports.cpp | 36 ++++++++++++++++++++++++---------- libi2pd/Transports.h | 7 ++++--- libi2pd/TunnelGateway.cpp | 2 +- 9 files changed, 98 insertions(+), 37 deletions(-) diff --git a/libi2pd/NTCP2.cpp b/libi2pd/NTCP2.cpp index a05469f1..747dc0b0 100644 --- a/libi2pd/NTCP2.cpp +++ b/libi2pd/NTCP2.cpp @@ -375,6 +375,8 @@ namespace transport m_Socket.close (); transports.PeerDisconnected (shared_from_this ()); m_Server.RemoveNTCP2Session (shared_from_this ()); + if (!m_IntermediateQueue.empty ()) + m_SendQueue.splice (m_SendQueue.end (), m_IntermediateQueue); for (auto& it: m_SendQueue) it->Drop (); m_SendQueue.clear (); @@ -1207,7 +1209,7 @@ namespace transport void NTCP2Session::MoveSendQueue (std::shared_ptr other) { if (!other || m_SendQueue.empty ()) return; - std::vector > msgs; + std::list > msgs; auto ts = i2p::util::GetMillisecondsSinceEpoch (); for (auto it: m_SendQueue) if (!it->IsExpired (ts)) @@ -1216,7 +1218,7 @@ namespace transport it->Drop (); m_SendQueue.clear (); if (!msgs.empty ()) - other->PostI2NPMessages (msgs); + other->SendI2NPMessages (msgs); } size_t NTCP2Session::CreatePaddingBlock (size_t msgLen, uint8_t * buf, size_t len) @@ -1297,20 +1299,38 @@ namespace transport m_Server.GetService ().post (std::bind (&NTCP2Session::Terminate, shared_from_this ())); // let termination message go } - void NTCP2Session::SendI2NPMessages (const std::vector >& msgs) + void NTCP2Session::SendI2NPMessages (std::list >& msgs) { - m_Server.GetService ().post (std::bind (&NTCP2Session::PostI2NPMessages, shared_from_this (), msgs)); + if (m_IsTerminated || msgs.empty ()) return; + bool empty = false; + { + std::lock_guard l(m_IntermediateQueueMutex); + empty = m_IntermediateQueue.empty (); + m_IntermediateQueue.splice (m_IntermediateQueue.end (), msgs); + } + if (empty) + m_Server.GetService ().post (std::bind (&NTCP2Session::PostI2NPMessages, shared_from_this ())); } - void NTCP2Session::PostI2NPMessages (std::vector > msgs) + void NTCP2Session::PostI2NPMessages () { if (m_IsTerminated) return; + std::list > msgs; + { + std::lock_guard l(m_IntermediateQueueMutex); + m_IntermediateQueue.swap (msgs); + } bool isSemiFull = m_SendQueue.size () > NTCP2_MAX_OUTGOING_QUEUE_SIZE/2; - for (auto it: msgs) - if (isSemiFull && it->onDrop) - it->Drop (); // drop earlier because we can handle it - else - m_SendQueue.push_back (std::move (it)); + if (isSemiFull) + { + for (auto it: msgs) + if (it->onDrop) + it->Drop (); // drop earlier because we can handle it + else + m_SendQueue.push_back (std::move (it)); + } + else + m_SendQueue.splice (m_SendQueue.end (), msgs); if (!m_IsSending && m_IsEstablished) SendQueue (); diff --git a/libi2pd/NTCP2.h b/libi2pd/NTCP2.h index f7912b54..27acb529 100644 --- a/libi2pd/NTCP2.h +++ b/libi2pd/NTCP2.h @@ -153,7 +153,7 @@ namespace transport void ServerLogin (); // Bob void SendLocalRouterInfo (bool update) override; // after handshake or by update - void SendI2NPMessages (const std::vector >& msgs) override; + void SendI2NPMessages (std::list >& msgs) override; void MoveSendQueue (std::shared_ptr other); private: @@ -196,7 +196,7 @@ namespace transport void SendRouterInfo (); void SendTermination (NTCP2TerminationReason reason); void SendTerminationAndTerminate (NTCP2TerminationReason reason); - void PostI2NPMessages (std::vector > msgs); + void PostI2NPMessages (); private: @@ -229,7 +229,10 @@ namespace transport bool m_IsSending, m_IsReceiving; std::list > m_SendQueue; uint64_t m_NextRouterInfoResendTime; // seconds since epoch - + + std::list > m_IntermediateQueue; // from transports + mutable std::mutex m_IntermediateQueueMutex; + uint16_t m_PaddingSizes[16]; int m_NextPaddingSize; }; diff --git a/libi2pd/NetDb.cpp b/libi2pd/NetDb.cpp index 341d617e..c96bcf95 100644 --- a/libi2pd/NetDb.cpp +++ b/libi2pd/NetDb.cpp @@ -480,7 +480,7 @@ namespace data void NetDb::ReseedFromFloodfill(const RouterInfo & ri, int numRouters, int numFloodfills) { LogPrint(eLogInfo, "NetDB: Reseeding from floodfill ", ri.GetIdentHashBase64()); - std::vector > requests; + std::list > requests; i2p::data::IdentHash ourIdent = i2p::context.GetIdentHash(); i2p::data::IdentHash ih = ri.GetIdentHash(); diff --git a/libi2pd/SSU2Session.cpp b/libi2pd/SSU2Session.cpp index 3489a6ba..9ceb491e 100644 --- a/libi2pd/SSU2Session.cpp +++ b/libi2pd/SSU2Session.cpp @@ -293,6 +293,8 @@ namespace transport m_SentHandshakePacket.reset (nullptr); m_SessionConfirmedFragment.reset (nullptr); m_PathChallenge.reset (nullptr); + if (!m_IntermediateQueue.empty ()) + m_SendQueue.splice (m_SendQueue.end (), m_IntermediateQueue); for (auto& it: m_SendQueue) it->Drop (); m_SendQueue.clear (); @@ -372,14 +374,27 @@ namespace transport } - void SSU2Session::SendI2NPMessages (const std::vector >& msgs) + void SSU2Session::SendI2NPMessages (std::list >& msgs) { - m_Server.GetService ().post (std::bind (&SSU2Session::PostI2NPMessages, shared_from_this (), msgs)); + if (m_State == eSSU2SessionStateTerminated || msgs.empty ()) return; + bool empty = false; + { + std::lock_guard l(m_IntermediateQueueMutex); + empty = m_IntermediateQueue.empty (); + m_IntermediateQueue.splice (m_IntermediateQueue.end (), msgs); + } + if (empty) + m_Server.GetService ().post (std::bind (&SSU2Session::PostI2NPMessages, shared_from_this ())); } - void SSU2Session::PostI2NPMessages (std::vector > msgs) + void SSU2Session::PostI2NPMessages () { if (m_State == eSSU2SessionStateTerminated) return; + std::list > msgs; + { + std::lock_guard l(m_IntermediateQueueMutex); + m_IntermediateQueue.swap (msgs); + } uint64_t mts = i2p::util::GetMonotonicMicroseconds (); bool isSemiFull = false; if (m_SendQueue.size ()) @@ -415,7 +430,7 @@ namespace transport void SSU2Session::MoveSendQueue (std::shared_ptr other) { if (!other || m_SendQueue.empty ()) return; - std::vector > msgs; + std::list > msgs; auto ts = i2p::util::GetMillisecondsSinceEpoch (); for (auto it: m_SendQueue) if (!it->IsExpired (ts)) @@ -424,7 +439,7 @@ namespace transport it->Drop (); m_SendQueue.clear (); if (!msgs.empty ()) - other->PostI2NPMessages (msgs); + other->SendI2NPMessages (msgs); } bool SSU2Session::SendQueue () diff --git a/libi2pd/SSU2Session.h b/libi2pd/SSU2Session.h index d54731dc..4b3139a7 100644 --- a/libi2pd/SSU2Session.h +++ b/libi2pd/SSU2Session.h @@ -261,7 +261,7 @@ namespace transport void FlushData (); void Done () override; void SendLocalRouterInfo (bool update) override; - void SendI2NPMessages (const std::vector >& msgs) override; + void SendI2NPMessages (std::list >& msgs) override; void MoveSendQueue (std::shared_ptr other); uint32_t GetRelayTag () const override { return m_RelayTag; }; size_t Resend (uint64_t ts); // return number of resent packets @@ -307,7 +307,7 @@ namespace transport void Established (); void ScheduleConnectTimer (); void HandleConnectTimer (const boost::system::error_code& ecode); - void PostI2NPMessages (std::vector > msgs); + void PostI2NPMessages (); bool SendQueue (); // returns true if ack block was sent bool SendFragmentedMessage (std::shared_ptr msg); void ResendHandshakePacket (); @@ -381,6 +381,8 @@ namespace transport std::unordered_map, uint64_t > > m_RelaySessions; // nonce->(Alice, timestamp) for Bob or nonce->(Charlie, timestamp) for Alice std::list > m_SendQueue; i2p::I2NPMessagesHandler m_Handler; + std::list > m_IntermediateQueue; // from transports + mutable std::mutex m_IntermediateQueueMutex; bool m_IsDataReceived; double m_RTT; int m_MsgLocalExpirationTimeout; diff --git a/libi2pd/TransportSession.h b/libi2pd/TransportSession.h index c6bf0de3..6c878d11 100644 --- a/libi2pd/TransportSession.h +++ b/libi2pd/TransportSession.h @@ -144,8 +144,12 @@ namespace transport void SetLastActivityTimestamp (uint64_t ts) { m_LastActivityTimestamp = ts; }; virtual uint32_t GetRelayTag () const { return 0; }; - virtual void SendLocalRouterInfo (bool update = false) { SendI2NPMessages ({ CreateDatabaseStoreMsg () }); }; - virtual void SendI2NPMessages (const std::vector >& msgs) = 0; + virtual void SendLocalRouterInfo (bool update = false) + { + std::list > msgs{ CreateDatabaseStoreMsg () }; + SendI2NPMessages (msgs); + }; + virtual void SendI2NPMessages (std::list >& msgs) = 0; virtual bool IsEstablished () const = 0; private: diff --git a/libi2pd/Transports.cpp b/libi2pd/Transports.cpp index 34bc6142..3954e2cf 100644 --- a/libi2pd/Transports.cpp +++ b/libi2pd/Transports.cpp @@ -450,15 +450,23 @@ namespace transport void Transports::SendMessage (const i2p::data::IdentHash& ident, std::shared_ptr msg) { if (m_IsOnline) - SendMessages (ident, std::vector > {msg }); + SendMessages (ident, { msg }); } - void Transports::SendMessages (const i2p::data::IdentHash& ident, const std::vector >& msgs) + void Transports::SendMessages (const i2p::data::IdentHash& ident, const std::list >& msgs) { m_Service->post (std::bind (&Transports::PostMessages, this, ident, msgs)); } - void Transports::PostMessages (i2p::data::IdentHash ident, std::vector > msgs) + void Transports::SendMessages (const i2p::data::IdentHash& ident, std::list >&& msgs) + { + m_Service->post ([this, ident, msgs = std::move(msgs)] () + { + PostMessages (ident, msgs); + }); + } + + void Transports::PostMessages (i2p::data::IdentHash ident, std::list > msgs) { if (ident == i2p::context.GetRouterInfo ().GetIdentHash ()) { @@ -517,11 +525,16 @@ namespace transport return; } } - for (auto& it1: msgs) - if (sz > MAX_NUM_DELAYED_MESSAGES/2 && it1->onDrop) - it1->Drop (); // drop earlier because we can handle it - else - peer->delayedMessages.push_back (it1); + if (sz > MAX_NUM_DELAYED_MESSAGES/2) + { + for (auto& it1: msgs) + if (it1->onDrop) + it1->Drop (); // drop earlier because we can handle it + else + peer->delayedMessages.push_back (it1); + } + else + peer->delayedMessages.splice (peer->delayedMessages.end (), msgs); } else { @@ -865,7 +878,7 @@ namespace transport if (it->second->delayedMessages.size () > 0) { // check if first message is our DatabaseStore (publishing) - auto firstMsg = peer->delayedMessages[0]; + auto firstMsg = peer->delayedMessages.front (); if (firstMsg && firstMsg->GetTypeID () == eI2NPDatabaseStore && i2p::data::IdentHash(firstMsg->GetPayload () + DATABASE_STORE_KEY_OFFSET) == i2p::context.GetIdentHash ()) sendDatabaseStore = false; // we have it in the list already @@ -887,7 +900,10 @@ namespace transport return; } if (!session->IsOutgoing ()) // incoming - session->SendI2NPMessages ({ CreateDatabaseStoreMsg () }); // send DatabaseStore + { + std::list > msgs{ CreateDatabaseStoreMsg () }; + session->SendI2NPMessages (msgs); // send DatabaseStore + } auto r = i2p::data::netdb.FindRouter (ident); // router should be in netdb after SessionConfirmed if (r) r->GetProfile ()->Connected (); auto ts = i2p::util::GetSecondsSinceEpoch (); diff --git a/libi2pd/Transports.h b/libi2pd/Transports.h index 70273094..e18ec29d 100644 --- a/libi2pd/Transports.h +++ b/libi2pd/Transports.h @@ -71,7 +71,7 @@ namespace transport std::shared_ptr router; std::list > sessions; uint64_t creationTime, nextRouterInfoUpdateTime, lastSelectionTime; - std::vector > delayedMessages; + std::list > delayedMessages; std::vector priority; bool isHighBandwidth, isEligible; @@ -141,7 +141,8 @@ namespace transport void ReuseX25519KeysPair (std::shared_ptr pair); void SendMessage (const i2p::data::IdentHash& ident, std::shared_ptr msg); - void SendMessages (const i2p::data::IdentHash& ident, const std::vector >& msgs); + void SendMessages (const i2p::data::IdentHash& ident, const std::list >& msgs); + void SendMessages (const i2p::data::IdentHash& ident, std::list >&& msgs); void PeerConnected (std::shared_ptr session); void PeerDisconnected (std::shared_ptr session); @@ -185,7 +186,7 @@ namespace transport void Run (); void RequestComplete (std::shared_ptr r, const i2p::data::IdentHash& ident); void HandleRequestComplete (std::shared_ptr r, i2p::data::IdentHash ident); - void PostMessages (i2p::data::IdentHash ident, std::vector > msgs); + void PostMessages (i2p::data::IdentHash ident, std::list > msgs); bool ConnectToPeer (const i2p::data::IdentHash& ident, std::shared_ptr peer); void SetPriority (std::shared_ptr peer) const; void HandlePeerCleanupTimer (const boost::system::error_code& ecode); diff --git a/libi2pd/TunnelGateway.cpp b/libi2pd/TunnelGateway.cpp index 85ff224e..77110c39 100644 --- a/libi2pd/TunnelGateway.cpp +++ b/libi2pd/TunnelGateway.cpp @@ -221,7 +221,7 @@ namespace tunnel void TunnelGateway::SendBuffer () { m_Buffer.CompleteCurrentTunnelDataMessage (); - std::vector > newTunnelMsgs; + std::list > newTunnelMsgs; const auto& tunnelDataMsgs = m_Buffer.GetTunnelDataMsgs (); for (auto& tunnelMsg : tunnelDataMsgs) { From 4c90a88b85c736dad7e70ef1e2503a9360358042 Mon Sep 17 00:00:00 2001 From: orignal Date: Mon, 28 Oct 2024 21:10:30 -0400 Subject: [PATCH 06/18] eliminate extra copy of I2NP messages list --- libi2pd/TransitTunnel.cpp | 5 ++--- libi2pd/Transports.cpp | 9 +++++---- libi2pd/Transports.h | 2 +- libi2pd/TunnelGateway.cpp | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/libi2pd/TransitTunnel.cpp b/libi2pd/TransitTunnel.cpp index 6c2c52a7..72d7b8c2 100644 --- a/libi2pd/TransitTunnel.cpp +++ b/libi2pd/TransitTunnel.cpp @@ -1,5 +1,5 @@ /* -* Copyright (c) 2013-2022, The PurpleI2P Project +* Copyright (c) 2013-2024, The PurpleI2P Project * * This file is part of Purple i2pd project and licensed under BSD3 * @@ -59,8 +59,7 @@ namespace tunnel auto num = m_TunnelDataMsgs.size (); if (num > 1) LogPrint (eLogDebug, "TransitTunnel: ", GetTunnelID (), "->", GetNextTunnelID (), " ", num); - i2p::transport::transports.SendMessages (GetNextIdentHash (), m_TunnelDataMsgs); - m_TunnelDataMsgs.clear (); + i2p::transport::transports.SendMessages (GetNextIdentHash (), m_TunnelDataMsgs); // send and clear } } diff --git a/libi2pd/Transports.cpp b/libi2pd/Transports.cpp index 3954e2cf..d18d3429 100644 --- a/libi2pd/Transports.cpp +++ b/libi2pd/Transports.cpp @@ -453,9 +453,11 @@ namespace transport SendMessages (ident, { msg }); } - void Transports::SendMessages (const i2p::data::IdentHash& ident, const std::list >& msgs) + void Transports::SendMessages (const i2p::data::IdentHash& ident, std::list >& msgs) { - m_Service->post (std::bind (&Transports::PostMessages, this, ident, msgs)); + std::list > msgs1; + msgs.swap (msgs1); + SendMessages (ident, std::move (msgs1)); } void Transports::SendMessages (const i2p::data::IdentHash& ident, std::list >&& msgs) @@ -888,8 +890,7 @@ namespace transport else session->SetTerminationTimeout (10); // most likely it's publishing, no follow-up messages expected, set timeout to 10 seconds peer->sessions.push_back (session); - session->SendI2NPMessages (peer->delayedMessages); - peer->delayedMessages.clear (); + session->SendI2NPMessages (peer->delayedMessages); // send and clear } else // incoming connection or peer test { diff --git a/libi2pd/Transports.h b/libi2pd/Transports.h index e18ec29d..bfabc6b3 100644 --- a/libi2pd/Transports.h +++ b/libi2pd/Transports.h @@ -141,7 +141,7 @@ namespace transport void ReuseX25519KeysPair (std::shared_ptr pair); void SendMessage (const i2p::data::IdentHash& ident, std::shared_ptr msg); - void SendMessages (const i2p::data::IdentHash& ident, const std::list >& msgs); + void SendMessages (const i2p::data::IdentHash& ident, std::list >& msgs); void SendMessages (const i2p::data::IdentHash& ident, std::list >&& msgs); void PeerConnected (std::shared_ptr session); diff --git a/libi2pd/TunnelGateway.cpp b/libi2pd/TunnelGateway.cpp index 77110c39..78a63fc4 100644 --- a/libi2pd/TunnelGateway.cpp +++ b/libi2pd/TunnelGateway.cpp @@ -234,7 +234,7 @@ namespace tunnel m_NumSentBytes += TUNNEL_DATA_MSG_SIZE; } m_Buffer.ClearTunnelDataMsgs (); - i2p::transport::transports.SendMessages (m_Tunnel->GetNextIdentHash (), newTunnelMsgs); + i2p::transport::transports.SendMessages (m_Tunnel->GetNextIdentHash (), std::move (newTunnelMsgs)); } } } From 361f3649667593dd95d2ad6dd79b38959ceaf1b2 Mon Sep 17 00:00:00 2001 From: orignal Date: Mon, 28 Oct 2024 21:15:16 -0400 Subject: [PATCH 07/18] intermediate queue for transport sessions. use std::list instead std::vector for multiple I2NP messages --- libi2pd/TransitTunnel.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libi2pd/TransitTunnel.h b/libi2pd/TransitTunnel.h index f83007a9..b3381fb7 100644 --- a/libi2pd/TransitTunnel.h +++ b/libi2pd/TransitTunnel.h @@ -1,5 +1,5 @@ /* -* Copyright (c) 2013-2023, The PurpleI2P Project +* Copyright (c) 2013-2024, The PurpleI2P Project * * This file is part of Purple i2pd project and licensed under BSD3 * @@ -10,7 +10,7 @@ #define TRANSIT_TUNNEL_H__ #include -#include +#include #include #include #include "Crypto.h" @@ -61,7 +61,7 @@ namespace tunnel private: size_t m_NumTransmittedBytes; - std::vector > m_TunnelDataMsgs; + std::list > m_TunnelDataMsgs; }; class TransitTunnelGateway: public TransitTunnel From f04048717dc1633c2d21ef2d223f79138b2b03bd Mon Sep 17 00:00:00 2001 From: orignal Date: Mon, 28 Oct 2024 21:34:33 -0400 Subject: [PATCH 08/18] cleanup messages to send if session was terminated --- libi2pd/NTCP2.cpp | 6 +++++- libi2pd/SSU2Session.cpp | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/libi2pd/NTCP2.cpp b/libi2pd/NTCP2.cpp index 747dc0b0..33c33596 100644 --- a/libi2pd/NTCP2.cpp +++ b/libi2pd/NTCP2.cpp @@ -1301,7 +1301,11 @@ namespace transport void NTCP2Session::SendI2NPMessages (std::list >& msgs) { - if (m_IsTerminated || msgs.empty ()) return; + if (m_IsTerminated || msgs.empty ()) + { + msgs.clear (); + return; + } bool empty = false; { std::lock_guard l(m_IntermediateQueueMutex); diff --git a/libi2pd/SSU2Session.cpp b/libi2pd/SSU2Session.cpp index 9ceb491e..5d5d5249 100644 --- a/libi2pd/SSU2Session.cpp +++ b/libi2pd/SSU2Session.cpp @@ -376,7 +376,11 @@ namespace transport void SSU2Session::SendI2NPMessages (std::list >& msgs) { - if (m_State == eSSU2SessionStateTerminated || msgs.empty ()) return; + if (m_State == eSSU2SessionStateTerminated || msgs.empty ()) + { + msgs.clear (); + return; + } bool empty = false; { std::lock_guard l(m_IntermediateQueueMutex); From 9bc595a9a2849743e85b3707e6f30163ede676d1 Mon Sep 17 00:00:00 2001 From: orignal Date: Tue, 29 Oct 2024 08:41:59 -0400 Subject: [PATCH 09/18] eliminate extra copy --- libi2pd/Transports.cpp | 4 ++-- libi2pd/Transports.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libi2pd/Transports.cpp b/libi2pd/Transports.cpp index d18d3429..178188f2 100644 --- a/libi2pd/Transports.cpp +++ b/libi2pd/Transports.cpp @@ -462,13 +462,13 @@ namespace transport void Transports::SendMessages (const i2p::data::IdentHash& ident, std::list >&& msgs) { - m_Service->post ([this, ident, msgs = std::move(msgs)] () + m_Service->post ([this, ident, msgs = std::move(msgs)] () mutable { PostMessages (ident, msgs); }); } - void Transports::PostMessages (i2p::data::IdentHash ident, std::list > msgs) + void Transports::PostMessages (const i2p::data::IdentHash& ident, std::list >& msgs) { if (ident == i2p::context.GetRouterInfo ().GetIdentHash ()) { diff --git a/libi2pd/Transports.h b/libi2pd/Transports.h index bfabc6b3..095cc81a 100644 --- a/libi2pd/Transports.h +++ b/libi2pd/Transports.h @@ -186,7 +186,7 @@ namespace transport void Run (); void RequestComplete (std::shared_ptr r, const i2p::data::IdentHash& ident); void HandleRequestComplete (std::shared_ptr r, i2p::data::IdentHash ident); - void PostMessages (i2p::data::IdentHash ident, std::list > msgs); + void PostMessages (const i2p::data::IdentHash& ident, std::list >& msgs); bool ConnectToPeer (const i2p::data::IdentHash& ident, std::shared_ptr peer); void SetPriority (std::shared_ptr peer) const; void HandlePeerCleanupTimer (const boost::system::error_code& ecode); From 3f10f6651d7b966a23c68621d63389ea35e55839 Mon Sep 17 00:00:00 2001 From: orignal Date: Tue, 29 Oct 2024 12:46:14 -0400 Subject: [PATCH 10/18] use splice if queue is not semi-full --- libi2pd/SSU2Session.cpp | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/libi2pd/SSU2Session.cpp b/libi2pd/SSU2Session.cpp index 5d5d5249..aba8195e 100644 --- a/libi2pd/SSU2Session.cpp +++ b/libi2pd/SSU2Session.cpp @@ -412,16 +412,24 @@ namespace transport " is semi-full (size = ", m_SendQueue.size (), ", lag = ", queueLag / 1000, ", rtt = ", (int)m_RTT, ")"); } } - for (auto it: msgs) - { - if (isSemiFull && it->onDrop) - it->Drop (); // drop earlier because we can handle it - else + if (isSemiFull) + { + for (auto it: msgs) { - it->SetEnqueueTime (mts); - m_SendQueue.push_back (std::move (it)); + if (it->onDrop) + it->Drop (); // drop earlier because we can handle it + else + { + it->SetEnqueueTime (mts); + m_SendQueue.push_back (std::move (it)); + } } - } + } + else + { + for (auto& it: msgs) it->SetEnqueueTime (mts); + m_SendQueue.splice (m_SendQueue.end (), msgs); + } if (IsEstablished ()) { SendQueue (); From 8a8277edda0392a3fd99278409b811fc44f9e739 Mon Sep 17 00:00:00 2001 From: orignal Date: Tue, 29 Oct 2024 13:59:21 -0400 Subject: [PATCH 11/18] check for empty URL string --- libi2pd/HTTP.cpp | 1 + libi2pd_client/AddressBook.cpp | 9 ++++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libi2pd/HTTP.cpp b/libi2pd/HTTP.cpp index 990781bc..258d3ada 100644 --- a/libi2pd/HTTP.cpp +++ b/libi2pd/HTTP.cpp @@ -103,6 +103,7 @@ namespace http bool URL::parse(std::string_view url) { + if (url.empty ()) return false; std::size_t pos_p = 0; /* < current parse position */ std::size_t pos_c = 0; /* < work position */ if(url.at(0) != '/' || pos_p > 0) diff --git a/libi2pd_client/AddressBook.cpp b/libi2pd_client/AddressBook.cpp index 14599cf7..802b7996 100644 --- a/libi2pd_client/AddressBook.cpp +++ b/libi2pd_client/AddressBook.cpp @@ -582,16 +582,15 @@ namespace client } else { - LogPrint (eLogInfo, "Addressbook: Loading subscriptions from config file"); + LogPrint (eLogInfo, "Addressbook: Loading subscriptions from config"); // using config file items std::string subscriptionURLs; i2p::config::GetOption("addressbook.subscriptions", subscriptionURLs); std::vector subsList; boost::split(subsList, subscriptionURLs, boost::is_any_of(","), boost::token_compress_on); for (const auto& s: subsList) - { - m_Subscriptions.push_back (std::make_shared (*this, s)); - } + if (!s.empty ()) + m_Subscriptions.push_back (std::make_shared (*this, s)); LogPrint (eLogInfo, "Addressbook: ", m_Subscriptions.size (), " subscriptions urls loaded"); } } @@ -823,7 +822,7 @@ namespace client } } - AddressBookSubscription::AddressBookSubscription (AddressBook& book, const std::string& link): + AddressBookSubscription::AddressBookSubscription (AddressBook& book, std::string_view link): m_Book (book), m_Link (link) { } From 0086f8e27ad0f4fd702838c4782bb057df74ca77 Mon Sep 17 00:00:00 2001 From: orignal Date: Tue, 29 Oct 2024 15:32:06 -0400 Subject: [PATCH 12/18] use std::async for address book download --- libi2pd_client/AddressBook.cpp | 51 +++++++++++++++++++++------------- libi2pd_client/AddressBook.h | 7 +++-- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/libi2pd_client/AddressBook.cpp b/libi2pd_client/AddressBook.cpp index 802b7996..c9452cc6 100644 --- a/libi2pd_client/AddressBook.cpp +++ b/libi2pd_client/AddressBook.cpp @@ -305,7 +305,7 @@ namespace client identHash = hash; } - AddressBook::AddressBook (): m_Storage(nullptr), m_IsLoaded (false), m_IsDownloading (false), + AddressBook::AddressBook (): m_Storage(nullptr), m_IsLoaded (false), m_NumRetries (0), m_DefaultSubscription (nullptr), m_SubscriptionsUpdateTimer (nullptr), m_IsEnabled (true) { @@ -344,20 +344,28 @@ namespace client delete m_SubscriptionsUpdateTimer; m_SubscriptionsUpdateTimer = nullptr; } - if (m_IsDownloading) + bool isDownloading = m_Downloading.valid (); + if (isDownloading) { - LogPrint (eLogInfo, "Addressbook: Subscriptions are downloading, abort"); - for (int i = 0; i < 30; i++) - { - if (!m_IsDownloading) + if (m_Downloading.wait_for(std::chrono::seconds(0)) == std::future_status::ready) + isDownloading = false; + else + { + LogPrint (eLogInfo, "Addressbook: Subscriptions are downloading, abort"); + for (int i = 0; i < 30; i++) { - LogPrint (eLogInfo, "Addressbook: Subscriptions download complete"); - break; + if (m_Downloading.wait_for(std::chrono::seconds(1)) == std::future_status::ready) // wait for 1 seconds + { + isDownloading = false; + LogPrint (eLogInfo, "Addressbook: Subscriptions download complete"); + break; + } } - std::this_thread::sleep_for (std::chrono::seconds (1)); // wait for 1 seconds - } - LogPrint (eLogError, "Addressbook: Subscription download timeout"); - m_IsDownloading = false; + } + if (!isDownloading) + m_Downloading.get (); + else + LogPrint (eLogError, "Addressbook: Subscription download timeout"); } if (m_Storage) { @@ -644,7 +652,6 @@ namespace client void AddressBook::DownloadComplete (bool success, const i2p::data::IdentHash& subscription, const std::string& etag, const std::string& lastModified) { - m_IsDownloading = false; m_NumRetries++; int nextUpdateTimeout = m_NumRetries*CONTINIOUS_SUBSCRIPTION_RETRY_TIMEOUT; if (m_NumRetries > CONTINIOUS_SUBSCRIPTION_MAX_NUM_RETRIES || nextUpdateTimeout > CONTINIOUS_SUBSCRIPTION_UPDATE_TIMEOUT) @@ -699,7 +706,13 @@ namespace client LogPrint(eLogWarning, "Addressbook: Missing local destination, skip subscription update"); return; } - if (!m_IsDownloading && dest->IsReady ()) + bool isDownloading = m_Downloading.valid (); + if (isDownloading && m_Downloading.wait_for(std::chrono::seconds(0)) == std::future_status::ready) // still active? + { + m_Downloading.get (); + isDownloading = false; + } + if (!isDownloading && dest->IsReady ()) { if (!m_IsLoaded) { @@ -708,17 +721,15 @@ namespace client std::string defaultSubURL; i2p::config::GetOption("addressbook.defaulturl", defaultSubURL); if (!m_DefaultSubscription) m_DefaultSubscription = std::make_shared(*this, defaultSubURL); - m_IsDownloading = true; - std::thread load_hosts(std::bind (&AddressBookSubscription::CheckUpdates, m_DefaultSubscription)); - load_hosts.detach(); // TODO: use join + m_Downloading = std::async (std::launch::async, + std::bind (&AddressBookSubscription::CheckUpdates, m_DefaultSubscription)); } else if (!m_Subscriptions.empty ()) { // pick random subscription auto ind = rand () % m_Subscriptions.size(); - m_IsDownloading = true; - std::thread load_hosts(std::bind (&AddressBookSubscription::CheckUpdates, m_Subscriptions[ind])); - load_hosts.detach(); // TODO: use join + m_Downloading = std::async (std::launch::async, + std::bind (&AddressBookSubscription::CheckUpdates, m_Subscriptions[ind])); } } else diff --git a/libi2pd_client/AddressBook.h b/libi2pd_client/AddressBook.h index fc4f19a7..553ae51b 100644 --- a/libi2pd_client/AddressBook.h +++ b/libi2pd_client/AddressBook.h @@ -11,10 +11,12 @@ #include #include +#include #include #include #include #include +#include #include #include #include "Base.h" @@ -124,7 +126,8 @@ namespace client std::mutex m_LookupsMutex; std::map m_Lookups; // nonce -> address AddressBookStorage * m_Storage; - volatile bool m_IsLoaded, m_IsDownloading; + volatile bool m_IsLoaded; + std::future m_Downloading; int m_NumRetries; std::vector > m_Subscriptions; std::shared_ptr m_DefaultSubscription; // in case if we don't know any addresses yet @@ -136,7 +139,7 @@ namespace client { public: - AddressBookSubscription (AddressBook& book, const std::string& link); + AddressBookSubscription (AddressBook& book, std::string_view link); void CheckUpdates (); private: From b2a10ac82b48864262a65fef492ac1165df2f013 Mon Sep 17 00:00:00 2001 From: orignal Date: Wed, 30 Oct 2024 22:00:12 -0400 Subject: [PATCH 13/18] don't update found router again in RequestComplete --- libi2pd/Transports.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libi2pd/Transports.cpp b/libi2pd/Transports.cpp index 178188f2..73c58ea2 100644 --- a/libi2pd/Transports.cpp +++ b/libi2pd/Transports.cpp @@ -491,9 +491,9 @@ namespace transport { auto r = netdb.FindRouter (ident); if (r && (r->IsUnreachable () || !r->IsReachableFrom (i2p::context.GetRouterInfo ()))) return; // router found but non-reachable - { - auto ts = i2p::util::GetSecondsSinceEpoch (); - peer = std::make_shared(r, ts); + + peer = std::make_shared(r, i2p::util::GetSecondsSinceEpoch ()); + { std::unique_lock l(m_PeersMutex); peer = m_Peers.emplace (ident, peer).first->second; } @@ -722,7 +722,7 @@ namespace transport void Transports::HandleRequestComplete (std::shared_ptr r, i2p::data::IdentHash ident) { auto it = m_Peers.find (ident); - if (it != m_Peers.end ()) + if (it != m_Peers.end () && !it->second->router) { if (r) { From 2419f52af4c6589d2c0936631edd97639a839624 Mon Sep 17 00:00:00 2001 From: orignal Date: Thu, 31 Oct 2024 15:01:19 -0400 Subject: [PATCH 14/18] fixed potential race condition --- libi2pd/Transports.cpp | 69 +++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/libi2pd/Transports.cpp b/libi2pd/Transports.cpp index 73c58ea2..98007620 100644 --- a/libi2pd/Transports.cpp +++ b/libi2pd/Transports.cpp @@ -480,8 +480,13 @@ namespace transport } if(RoutesRestricted() && !IsRestrictedPeer(ident)) return; std::shared_ptr peer; - auto it = m_Peers.find (ident); - if (it == m_Peers.end ()) + { + std::lock_guard l(m_PeersMutex); + auto it = m_Peers.find (ident); + if (it != m_Peers.end ()) + peer = it->second; + } + if (!peer) { // check if not banned if (i2p::data::IsRouterBanned (ident)) return; // don't create peer to unreachable router @@ -494,7 +499,7 @@ namespace transport peer = std::make_shared(r, i2p::util::GetSecondsSinceEpoch ()); { - std::unique_lock l(m_PeersMutex); + std::lock_guard l(m_PeersMutex); peer = m_Peers.emplace (ident, peer).first->second; } if (peer) @@ -506,8 +511,6 @@ namespace transport } if (!connected) return; } - else - peer = it->second; if (!peer) return; if (peer->IsConnected ()) @@ -522,7 +525,7 @@ namespace transport if (i2p::data::IsRouterBanned (ident)) { LogPrint (eLogWarning, "Transports: Router ", ident.ToBase64 (), " is banned. Peer dropped"); - std::unique_lock l(m_PeersMutex); + std::lock_guard l(m_PeersMutex); m_Peers.erase (ident); return; } @@ -542,7 +545,7 @@ namespace transport { LogPrint (eLogWarning, "Transports: Delayed messages queue size to ", ident.ToBase64 (), " exceeds ", MAX_NUM_DELAYED_MESSAGES); - std::unique_lock l(m_PeersMutex); + std::lock_guard l(m_PeersMutex); m_Peers.erase (ident); } } @@ -617,7 +620,7 @@ namespace transport if (!i2p::context.IsLimitedConnectivity () && peer->router->IsReachableFrom (i2p::context.GetRouterInfo ())) i2p::data::netdb.SetUnreachable (ident, true); // we are here because all connection attempts failed but router claimed them peer->Done (); - std::unique_lock l(m_PeersMutex); + std::lock_guard l(m_PeersMutex); m_Peers.erase (ident); return false; } @@ -625,7 +628,7 @@ namespace transport { LogPrint (eLogWarning, "Transports: Router ", ident.ToBase64 (), " is banned. Peer dropped"); peer->Done (); - std::unique_lock l(m_PeersMutex); + std::lock_guard l(m_PeersMutex); m_Peers.erase (ident); return false; } @@ -721,23 +724,29 @@ namespace transport void Transports::HandleRequestComplete (std::shared_ptr r, i2p::data::IdentHash ident) { - auto it = m_Peers.find (ident); - if (it != m_Peers.end () && !it->second->router) + std::shared_ptr peer; { - if (r) + std::lock_guard l(m_PeersMutex); + auto it = m_Peers.find (ident); + if (it != m_Peers.end ()) { - LogPrint (eLogDebug, "Transports: RouterInfo for ", ident.ToBase64 (), " found, trying to connect"); - it->second->SetRouter (r); - if (!it->second->IsConnected ()) - ConnectToPeer (ident, it->second); - } - else - { - LogPrint (eLogWarning, "Transports: RouterInfo not found, failed to send messages"); - std::unique_lock l(m_PeersMutex); - m_Peers.erase (it); - } + if (r) + peer = it->second; + else + m_Peers.erase (it); + } } + + if (peer && !peer->router && r) + { + LogPrint (eLogDebug, "Transports: RouterInfo for ", ident.ToBase64 (), " found, trying to connect"); + peer->SetRouter (r); + if (!peer->IsConnected ()) + ConnectToPeer (ident, peer); + } + else if (!r) + LogPrint (eLogInfo, "Transports: RouterInfo not found, failed to send messages"); + } void Transports::DetectExternalIP () @@ -911,7 +920,7 @@ namespace transport auto peer = std::make_shared(r, ts); peer->sessions.push_back (session); peer->router = nullptr; - std::unique_lock l(m_PeersMutex); + std::lock_guard l(m_PeersMutex); m_Peers.emplace (ident, peer); } }); @@ -940,7 +949,7 @@ namespace transport } else { - std::unique_lock l(m_PeersMutex); + std::lock_guard l(m_PeersMutex); m_Peers.erase (it); } } @@ -950,9 +959,13 @@ namespace transport bool Transports::IsConnected (const i2p::data::IdentHash& ident) const { - std::unique_lock l(m_PeersMutex); + std::lock_guard l(m_PeersMutex); +#if __cplusplus >= 202002L // C++20 + return m_Peers.contains (ident); +#else auto it = m_Peers.find (ident); return it != m_Peers.end (); +#endif } void Transports::HandlePeerCleanupTimer (const boost::system::error_code& ecode) @@ -976,7 +989,7 @@ namespace transport auto profile = i2p::data::GetRouterProfile (it->first); if (profile) profile->Unreachable (); } */ - std::unique_lock l(m_PeersMutex); + std::lock_guard l(m_PeersMutex); it = m_Peers.erase (it); } else @@ -1026,7 +1039,7 @@ namespace transport { uint16_t inds[3]; RAND_bytes ((uint8_t *)inds, sizeof (inds)); - std::unique_lock l(m_PeersMutex); + std::lock_guard l(m_PeersMutex); auto count = m_Peers.size (); if(count == 0) return nullptr; inds[0] %= count; From 4432c5a2c480db5f408e6e817801d065c98d6ea4 Mon Sep 17 00:00:00 2001 From: orignal Date: Thu, 31 Oct 2024 18:24:02 -0400 Subject: [PATCH 15/18] update last activity time after sending peer test or hole punch message --- libi2pd/SSU2OutOfSession.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libi2pd/SSU2OutOfSession.cpp b/libi2pd/SSU2OutOfSession.cpp index 262f93a9..651ed76c 100644 --- a/libi2pd/SSU2OutOfSession.cpp +++ b/libi2pd/SSU2OutOfSession.cpp @@ -111,7 +111,6 @@ namespace transport SendPeerTest (7, buf + offset, len - offset); else LogPrint (eLogWarning, "SSU2: Unknown address for peer test 6"); - GetServer ().AddConnectedRecently (GetRemoteEndpoint (), i2p::util::GetSecondsSinceEpoch ()); GetServer ().RequestRemoveSession (GetConnID ()); break; } @@ -141,7 +140,6 @@ namespace transport } } } - GetServer ().AddConnectedRecently (GetRemoteEndpoint (), i2p::util::GetSecondsSinceEpoch ()); GetServer ().RequestRemoveSession (GetConnID ()); break; } @@ -188,6 +186,7 @@ namespace transport i2p::crypto::ChaCha20 (h + 16, 16, addr->i, n, h + 16); // send GetServer ().Send (header.buf, 16, h + 16, 16, payload, payloadSize, GetRemoteEndpoint ()); + UpdateNumSentBytes (payloadSize + 32); } void SSU2PeerTestSession::SendPeerTest (uint8_t msg, const uint8_t * signedData, size_t signedDataLen, bool delayed) @@ -309,6 +308,7 @@ namespace transport i2p::crypto::ChaCha20 (h + 16, 16, addr->i, n, h + 16); // send GetServer ().Send (header.buf, 16, h + 16, 16, payload, payloadSize, ep); + UpdateNumSentBytes (payloadSize + 32); } void SSU2HolePunchSession::SendHolePunch (const uint8_t * relayResponseBlock, size_t relayResponseBlockLen) From b8d61e04f0ca5ce652e193198b5cd4f277fe2a18 Mon Sep 17 00:00:00 2001 From: orignal Date: Fri, 1 Nov 2024 14:46:13 -0400 Subject: [PATCH 16/18] generate x25519 keys more often --- libi2pd/Transports.cpp | 10 +++++----- libi2pd/Transports.h | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/libi2pd/Transports.cpp b/libi2pd/Transports.cpp index 98007620..6b695371 100644 --- a/libi2pd/Transports.cpp +++ b/libi2pd/Transports.cpp @@ -66,12 +66,12 @@ namespace transport while (m_IsRunning) { int num, total = 0; - while ((num = m_QueueSize - (int)m_Queue.size ()) > 0 && total < 10) + while ((num = m_QueueSize - (int)m_Queue.size ()) > 0 && total < m_QueueSize) { CreateEphemeralKeys (num); total += num; } - if (total >= 10) + if (total > m_QueueSize) { LogPrint (eLogWarning, "Transports: ", total, " ephemeral keys generated at the time"); std::this_thread::sleep_for (std::chrono::seconds(1)); // take a break @@ -124,12 +124,12 @@ namespace transport { if (pair) { - std::unique_lockl(m_AcquiredMutex); + std::unique_lock l(m_AcquiredMutex); if ((int)m_Queue.size () < 2*m_QueueSize) m_Queue.push (pair); } else - LogPrint(eLogError, "Transports: Return null DHKeys"); + LogPrint(eLogError, "Transports: Return null keys"); } void Peer::UpdateParams (std::shared_ptr router) @@ -149,7 +149,7 @@ namespace transport m_IsOnline (true), m_IsRunning (false), m_IsNAT (true), m_CheckReserved(true), m_Thread (nullptr), m_Service (nullptr), m_Work (nullptr), m_PeerCleanupTimer (nullptr), m_PeerTestTimer (nullptr), m_UpdateBandwidthTimer (nullptr), m_SSU2Server (nullptr), m_NTCP2Server (nullptr), - m_X25519KeysPairSupplier (15), // 15 pre-generated keys + m_X25519KeysPairSupplier (NUM_X25519_PRE_GENERATED_KEYS), m_TotalSentBytes (0), m_TotalReceivedBytes (0), m_TotalTransitTransmittedBytes (0), m_InBandwidth (0), m_OutBandwidth (0), m_TransitBandwidth (0), m_InBandwidth15s (0), m_OutBandwidth15s (0), m_TransitBandwidth15s (0), diff --git a/libi2pd/Transports.h b/libi2pd/Transports.h index 095cc81a..c7eb7677 100644 --- a/libi2pd/Transports.h +++ b/libi2pd/Transports.h @@ -108,7 +108,8 @@ namespace transport const int PEER_TEST_DELAY_INTERVAL_VARIANCE = 30; // in milliseconds const int MAX_NUM_DELAYED_MESSAGES = 150; const int CHECK_PROFILE_NUM_DELAYED_MESSAGES = 15; // check profile after - + const int NUM_X25519_PRE_GENERATED_KEYS = 25; // pre-generated x25519 keys pairs + const int TRAFFIC_SAMPLE_COUNT = 301; // seconds struct TrafficSample From 0d09a8be00b1b8ef3d29d82ed9ecc3fb1d7f64ae Mon Sep 17 00:00:00 2001 From: orignal Date: Fri, 1 Nov 2024 17:53:27 -0400 Subject: [PATCH 17/18] removed own implementation of x25519 --- libi2pd/Crypto.cpp | 36 ------------------- libi2pd/Crypto.h | 6 ---- libi2pd/Ed25519.cpp | 82 +------------------------------------------ libi2pd/Ed25519.h | 12 ++----- tests/CMakeLists.txt | 7 ---- tests/Makefile | 5 +-- tests/test-x25519.cpp | 38 -------------------- 7 files changed, 4 insertions(+), 182 deletions(-) delete mode 100644 tests/test-x25519.cpp diff --git a/libi2pd/Crypto.cpp b/libi2pd/Crypto.cpp index 8ce290f1..2371b529 100644 --- a/libi2pd/Crypto.cpp +++ b/libi2pd/Crypto.cpp @@ -240,17 +240,12 @@ namespace crypto // x25519 X25519Keys::X25519Keys () { -#if OPENSSL_X25519 m_Ctx = EVP_PKEY_CTX_new_id (NID_X25519, NULL); m_Pkey = nullptr; -#else - m_Ctx = BN_CTX_new (); -#endif } X25519Keys::X25519Keys (const uint8_t * priv, const uint8_t * pub) { -#if OPENSSL_X25519 m_Pkey = EVP_PKEY_new_raw_private_key (EVP_PKEY_X25519, NULL, priv, 32); m_Ctx = EVP_PKEY_CTX_new (m_Pkey, NULL); if (pub) @@ -260,29 +255,16 @@ namespace crypto size_t len = 32; EVP_PKEY_get_raw_public_key (m_Pkey, m_PublicKey, &len); } -#else - m_Ctx = BN_CTX_new (); - memcpy (m_PrivateKey, priv, 32); - if (pub) - memcpy (m_PublicKey, pub, 32); - else - GetEd25519 ()->ScalarMulB (m_PrivateKey, m_PublicKey, m_Ctx); -#endif } X25519Keys::~X25519Keys () { -#if OPENSSL_X25519 EVP_PKEY_CTX_free (m_Ctx); if (m_Pkey) EVP_PKEY_free (m_Pkey); -#else - BN_CTX_free (m_Ctx); -#endif } void X25519Keys::GenerateKeys () { -#if OPENSSL_X25519 if (m_Pkey) { EVP_PKEY_free (m_Pkey); @@ -294,16 +276,11 @@ namespace crypto m_Ctx = EVP_PKEY_CTX_new (m_Pkey, NULL); // TODO: do we really need to re-create m_Ctx? size_t len = 32; EVP_PKEY_get_raw_public_key (m_Pkey, m_PublicKey, &len); -#else - RAND_bytes (m_PrivateKey, 32); - GetEd25519 ()->ScalarMulB (m_PrivateKey, m_PublicKey, m_Ctx); -#endif } bool X25519Keys::Agree (const uint8_t * pub, uint8_t * shared) { if (!pub || (pub[31] & 0x80)) return false; // not x25519 key -#if OPENSSL_X25519 EVP_PKEY_derive_init (m_Ctx); auto pkey = EVP_PKEY_new_raw_public_key (EVP_PKEY_X25519, NULL, pub, 32); if (!pkey) return false; @@ -311,25 +288,17 @@ namespace crypto size_t len = 32; EVP_PKEY_derive (m_Ctx, shared, &len); EVP_PKEY_free (pkey); -#else - GetEd25519 ()->ScalarMul (pub, m_PrivateKey, shared, m_Ctx); -#endif return true; } void X25519Keys::GetPrivateKey (uint8_t * priv) const { -#if OPENSSL_X25519 size_t len = 32; EVP_PKEY_get_raw_private_key (m_Pkey, priv, &len); -#else - memcpy (priv, m_PrivateKey, 32); -#endif } void X25519Keys::SetPrivateKey (const uint8_t * priv, bool calculatePublic) { -#if OPENSSL_X25519 if (m_Ctx) EVP_PKEY_CTX_free (m_Ctx); if (m_Pkey) EVP_PKEY_free (m_Pkey); m_Pkey = EVP_PKEY_new_raw_private_key (EVP_PKEY_X25519, NULL, priv, 32); @@ -339,11 +308,6 @@ namespace crypto size_t len = 32; EVP_PKEY_get_raw_public_key (m_Pkey, m_PublicKey, &len); } -#else - memcpy (m_PrivateKey, priv, 32); - if (calculatePublic) - GetEd25519 ()->ScalarMulB (m_PrivateKey, m_PublicKey, m_Ctx); -#endif } // ElGamal diff --git a/libi2pd/Crypto.h b/libi2pd/Crypto.h index 13d331c8..75bd7bb2 100644 --- a/libi2pd/Crypto.h +++ b/libi2pd/Crypto.h @@ -31,7 +31,6 @@ #if (OPENSSL_VERSION_NUMBER >= 0x010101000) // 1.1.1 # define OPENSSL_HKDF 1 # define OPENSSL_EDDSA 1 -# define OPENSSL_X25519 1 # if (!defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER != 0x030000000)) // 3.0.0, regression in SipHash, not implemented in LibreSSL # define OPENSSL_SIPHASH 1 # endif @@ -70,13 +69,8 @@ namespace crypto private: uint8_t m_PublicKey[32]; -#if OPENSSL_X25519 EVP_PKEY_CTX * m_Ctx; EVP_PKEY * m_Pkey; -#else - BN_CTX * m_Ctx; - uint8_t m_PrivateKey[32]; -#endif bool m_IsElligatorIneligible = false; // true if definitely ineligible }; diff --git a/libi2pd/Ed25519.cpp b/libi2pd/Ed25519.cpp index 3e0795d5..47edb755 100644 --- a/libi2pd/Ed25519.cpp +++ b/libi2pd/Ed25519.cpp @@ -1,5 +1,5 @@ /* -* Copyright (c) 2013-2023, The PurpleI2P Project +* Copyright (c) 2013-2024, The PurpleI2P Project * * This file is part of Purple i2pd project and licensed under BSD3 * @@ -457,86 +457,6 @@ namespace crypto } } -#if !OPENSSL_X25519 - BIGNUM * Ed25519::ScalarMul (const BIGNUM * u, const BIGNUM * k, BN_CTX * ctx) const - { - BN_CTX_start (ctx); - auto x1 = BN_CTX_get (ctx); BN_copy (x1, u); - auto x2 = BN_CTX_get (ctx); BN_one (x2); - auto z2 = BN_CTX_get (ctx); BN_zero (z2); - auto x3 = BN_CTX_get (ctx); BN_copy (x3, u); - auto z3 = BN_CTX_get (ctx); BN_one (z3); - auto c121666 = BN_CTX_get (ctx); BN_set_word (c121666, 121666); - auto tmp0 = BN_CTX_get (ctx); auto tmp1 = BN_CTX_get (ctx); - unsigned int swap = 0; - auto bits = BN_num_bits (k); - while(bits) - { - --bits; - auto k_t = BN_is_bit_set(k, bits) ? 1 : 0; - swap ^= k_t; - if (swap) - { - std::swap (x2, x3); - std::swap (z2, z3); - } - swap = k_t; - BN_mod_sub(tmp0, x3, z3, q, ctx); - BN_mod_sub(tmp1, x2, z2, q, ctx); - BN_mod_add(x2, x2, z2, q, ctx); - BN_mod_add(z2, x3, z3, q, ctx); - BN_mod_mul(z3, tmp0, x2, q, ctx); - BN_mod_mul(z2, z2, tmp1, q, ctx); - BN_mod_sqr(tmp0, tmp1, q, ctx); - BN_mod_sqr(tmp1, x2, q, ctx); - BN_mod_add(x3, z3, z2, q, ctx); - BN_mod_sub(z2, z3, z2, q, ctx); - BN_mod_mul(x2, tmp1, tmp0, q, ctx); - BN_mod_sub(tmp1, tmp1, tmp0, q, ctx); - BN_mod_sqr(z2, z2, q, ctx); - BN_mod_mul(z3, tmp1, c121666, q, ctx); - BN_mod_sqr(x3, x3, q, ctx); - BN_mod_add(tmp0, tmp0, z3, q, ctx); - BN_mod_mul(z3, x1, z2, q, ctx); - BN_mod_mul(z2, tmp1, tmp0, q, ctx); - } - if (swap) - { - std::swap (x2, x3); - std::swap (z2, z3); - } - BN_mod_inverse (z2, z2, q, ctx); - BIGNUM * res = BN_new (); // not from ctx - BN_mod_mul(res, x2, z2, q, ctx); - BN_CTX_end (ctx); - return res; - } - - void Ed25519::ScalarMul (const uint8_t * p, const uint8_t * e, uint8_t * buf, BN_CTX * ctx) const - { - BIGNUM * p1 = DecodeBN<32> (p); - uint8_t k[32]; - memcpy (k, e, 32); - k[0] &= 248; k[31] &= 127; k[31] |= 64; - BIGNUM * n = DecodeBN<32> (k); - BIGNUM * q1 = ScalarMul (p1, n, ctx); - EncodeBN (q1, buf, 32); - BN_free (p1); BN_free (n); BN_free (q1); - } - - void Ed25519::ScalarMulB (const uint8_t * e, uint8_t * buf, BN_CTX * ctx) const - { - BIGNUM *p1 = BN_new (); BN_set_word (p1, 9); - uint8_t k[32]; - memcpy (k, e, 32); - k[0] &= 248; k[31] &= 127; k[31] |= 64; - BIGNUM * n = DecodeBN<32> (k); - BIGNUM * q1 = ScalarMul (p1, n, ctx); - EncodeBN (q1, buf, 32); - BN_free (p1); BN_free (n); BN_free (q1); - } -#endif - void Ed25519::BlindPublicKey (const uint8_t * pub, const uint8_t * seed, uint8_t * blinded) { BN_CTX * ctx = BN_CTX_new (); diff --git a/libi2pd/Ed25519.h b/libi2pd/Ed25519.h index 470d802f..9c0ad801 100644 --- a/libi2pd/Ed25519.h +++ b/libi2pd/Ed25519.h @@ -1,5 +1,5 @@ /* -* Copyright (c) 2013-2020, The PurpleI2P Project +* Copyright (c) 2013-2024, The PurpleI2P Project * * This file is part of Purple i2pd project and licensed under BSD3 * @@ -84,10 +84,7 @@ namespace crypto EDDSAPoint GeneratePublicKey (const uint8_t * expandedPrivateKey, BN_CTX * ctx) const; EDDSAPoint DecodePublicKey (const uint8_t * buf, BN_CTX * ctx) const; void EncodePublicKey (const EDDSAPoint& publicKey, uint8_t * buf, BN_CTX * ctx) const; -#if !OPENSSL_X25519 - void ScalarMul (const uint8_t * p, const uint8_t * e, uint8_t * buf, BN_CTX * ctx) const; // p is point, e is number for x25519 - void ScalarMulB (const uint8_t * e, uint8_t * buf, BN_CTX * ctx) const; -#endif + void BlindPublicKey (const uint8_t * pub, const uint8_t * seed, uint8_t * blinded); // for encrypted LeaseSet2, pub - 32, seed - 64, blinded - 32 void BlindPrivateKey (const uint8_t * priv, const uint8_t * seed, uint8_t * blindedPriv, uint8_t * blindedPub); // for encrypted LeaseSet2, pub - 32, seed - 64, blinded - 32 @@ -115,11 +112,6 @@ namespace crypto BIGNUM * DecodeBN (const uint8_t * buf) const; void EncodeBN (const BIGNUM * bn, uint8_t * buf, size_t len) const; -#if !OPENSSL_X25519 - // for x25519 - BIGNUM * ScalarMul (const BIGNUM * p, const BIGNUM * e, BN_CTX * ctx) const; -#endif - private: BIGNUM * q, * l, * d, * I; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index de319f5d..6353a683 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -49,10 +49,6 @@ set(test-gost-sig_SRCS test-gost-sig.cpp ) -set(test-x25519_SRCS - test-x25519.cpp -) - set(test-aeadchacha20poly1305_SRCS test-aeadchacha20poly1305.cpp ) @@ -77,7 +73,6 @@ add_executable(test-http-url ${test-http-url_SRCS}) add_executable(test-base-64 ${test-base-64_SRCS}) add_executable(test-gost ${test-gost_SRCS}) add_executable(test-gost-sig ${test-gost-sig_SRCS}) -add_executable(test-x25519 ${test-x25519_SRCS}) add_executable(test-aeadchacha20poly1305 ${test-aeadchacha20poly1305_SRCS}) add_executable(test-blinding ${test-blinding_SRCS}) add_executable(test-elligator ${test-elligator_SRCS}) @@ -102,7 +97,6 @@ target_link_libraries(test-http-url ${LIBS}) target_link_libraries(test-base-64 ${LIBS}) target_link_libraries(test-gost ${LIBS}) target_link_libraries(test-gost-sig ${LIBS}) -target_link_libraries(test-x25519 ${LIBS}) target_link_libraries(test-aeadchacha20poly1305 ${LIBS}) target_link_libraries(test-blinding ${LIBS}) target_link_libraries(test-elligator ${LIBS}) @@ -116,7 +110,6 @@ add_test(test-http-url ${TEST_PATH}/test-http-url) add_test(test-base-64 ${TEST_PATH}/test-base-64) add_test(test-gost ${TEST_PATH}/test-gost) add_test(test-gost-sig ${TEST_PATH}/test-gost-sig) -add_test(test-x25519 ${TEST_PATH}/test-x25519) add_test(test-aeadchacha20poly1305 ${TEST_PATH}/test-aeadchacha20poly1305) add_test(test-blinding ${TEST_PATH}/test-blinding) add_test(test-elligator ${TEST_PATH}/test-elligator) diff --git a/tests/Makefile b/tests/Makefile index 798fab42..51a11dfe 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -7,7 +7,7 @@ LIBI2PD = ../libi2pd.a TESTS = \ test-http-merge_chunked test-http-req test-http-res test-http-url test-http-url_decode \ - test-gost test-gost-sig test-base-64 test-x25519 test-aeadchacha20poly1305 test-blinding \ + test-gost test-gost-sig test-base-64 test-aeadchacha20poly1305 test-blinding \ test-elligator test-eddsa ifneq (, $(findstring mingw, $(SYS))$(findstring windows-gnu, $(SYS))$(findstring cygwin, $(SYS))) @@ -44,9 +44,6 @@ test-gost: test-gost.cpp $(LIBI2PD) test-gost-sig: test-gost-sig.cpp $(LIBI2PD) $(CXX) $(CXXFLAGS) $(NEEDED_CXXFLAGS) $(INCFLAGS) $(LDFLAGS) -o $@ $^ $(LDLIBS) -test-x25519: test-x25519.cpp $(LIBI2PD) - $(CXX) $(CXXFLAGS) $(NEEDED_CXXFLAGS) $(INCFLAGS) $(LDFLAGS) -o $@ $^ $(LDLIBS) - test-aeadchacha20poly1305: test-aeadchacha20poly1305.cpp $(LIBI2PD) $(CXX) $(CXXFLAGS) $(NEEDED_CXXFLAGS) $(INCFLAGS) $(LDFLAGS) -o $@ $^ $(LDLIBS) diff --git a/tests/test-x25519.cpp b/tests/test-x25519.cpp deleted file mode 100644 index a1f3f424..00000000 --- a/tests/test-x25519.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include -#include -#include - -#include "Ed25519.h" - -const uint8_t k[32] = -{ - 0xa5, 0x46, 0xe3, 0x6b, 0xf0, 0x52, 0x7c, 0x9d, 0x3b, 0x16, 0x15, - 0x4b, 0x82, 0x46, 0x5e, 0xdd, 0x62, 0x14, 0x4c, 0x0a, 0xc1, 0xfc, - 0x5a, 0x18, 0x50, 0x6a, 0x22, 0x44, 0xba, 0x44, 0x9a, 0xc4 -}; - -const uint8_t u[32] = -{ - 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb, 0x35, 0x94, 0xc1, - 0xa4, 0x24, 0xb1, 0x5f, 0x7c, 0x72, 0x66, 0x24, 0xec, 0x26, 0xb3, - 0x35, 0x3b, 0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab, 0x1c, 0x4c -}; - -uint8_t p[32] = -{ - 0xc3, 0xda, 0x55, 0x37, 0x9d, 0xe9, 0xc6, 0x90, 0x8e, 0x94, 0xea, - 0x4d, 0xf2, 0x8d, 0x08, 0x4f, 0x32, 0xec, 0xcf, 0x03, 0x49, 0x1c, - 0x71, 0xf7, 0x54, 0xb4, 0x07, 0x55, 0x77, 0xa2, 0x85, 0x52 -}; - -int main () -{ -#if !OPENSSL_X25519 -// we test it for openssl < 1.1.0 - uint8_t buf[32]; - BN_CTX * ctx = BN_CTX_new (); - i2p::crypto::GetEd25519 ()->ScalarMul (u, k, buf, ctx); - BN_CTX_free (ctx); - assert(memcmp (buf, p, 32) == 0); -#endif -} From 29d77113cc9bd5f12eb2b9b70357023466bdeb55 Mon Sep 17 00:00:00 2001 From: orignal Date: Sat, 2 Nov 2024 11:20:23 -0400 Subject: [PATCH 18/18] memory pool for x25519 keys --- libi2pd/Transports.cpp | 12 ++++++------ libi2pd/Transports.h | 4 +++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/libi2pd/Transports.cpp b/libi2pd/Transports.cpp index 6b695371..b07b1b0b 100644 --- a/libi2pd/Transports.cpp +++ b/libi2pd/Transports.cpp @@ -25,7 +25,7 @@ namespace transport { template EphemeralKeysSupplier::EphemeralKeysSupplier (int size): - m_QueueSize (size), m_IsRunning (false), m_Thread (nullptr) + m_QueueSize (size), m_IsRunning (false) { } @@ -39,7 +39,7 @@ namespace transport void EphemeralKeysSupplier::Start () { m_IsRunning = true; - m_Thread = new std::thread (std::bind (&EphemeralKeysSupplier::Run, this)); + m_Thread.reset (new std::thread (std::bind (&EphemeralKeysSupplier::Run, this))); } template @@ -53,8 +53,7 @@ namespace transport if (m_Thread) { m_Thread->join (); - delete m_Thread; - m_Thread = 0; + m_Thread = nullptr; } } @@ -78,6 +77,7 @@ namespace transport } else { + m_KeysPool.CleanUpMt (); std::unique_lock l(m_AcquiredMutex); if (!m_IsRunning) break; m_Acquired.wait (l); // wait for element gets acquired @@ -92,7 +92,7 @@ namespace transport { for (int i = 0; i < num; i++) { - auto pair = std::make_shared (); + auto pair = m_KeysPool.AcquireSharedMt (); pair->GenerateKeys (); std::unique_lock l(m_AcquiredMutex); m_Queue.push (pair); @@ -114,7 +114,7 @@ namespace transport } } // queue is empty, create new - auto pair = std::make_shared (); + auto pair = m_KeysPool.AcquireSharedMt (); pair->GenerateKeys (); return pair; } diff --git a/libi2pd/Transports.h b/libi2pd/Transports.h index c7eb7677..8271108d 100644 --- a/libi2pd/Transports.h +++ b/libi2pd/Transports.h @@ -26,6 +26,7 @@ #include "RouterInfo.h" #include "I2NPProtocol.h" #include "Identity.h" +#include "util.h" namespace i2p { @@ -53,9 +54,10 @@ namespace transport const int m_QueueSize; std::queue > m_Queue; + i2p::util::MemoryPoolMt m_KeysPool; bool m_IsRunning; - std::thread * m_Thread; + std::unique_ptr m_Thread; std::condition_variable m_Acquired; std::mutex m_AcquiredMutex; };