Debug: Detect which frame's state first diverged after a desync

src/crashlog.cpp

@@ -522,6 +522,9 @@ char *CrashLog::FillDesyncCrashLog(char *buffer, const char *last, const DesyncE
 				flag_check(DesyncExtraInfo::DEIF_STATE, "S"),
 				flag_check(DesyncExtraInfo::DEIF_DBL_RAND, "D"));
 	}
+	if (_network_server && (info.desync_frame_seed || info.desync_frame_state_checksum)) {
+		buffer += seprintf(buffer, last, "Desync frame: %08X (seed), %08X (state checksum)\n", info.desync_frame_seed, info.desync_frame_state_checksum);
+	}
 
 	extern uint32 _frame_counter;
 

src/crashlog.h

@@ -30,6 +30,8 @@ struct DesyncExtraInfo {
 	Flags flags = DEIF_NONE;
 	const char *client_name = nullptr;
 	int client_id = -1;
+	uint32 desync_frame_seed = 0;
+	uint32 desync_frame_state_checksum = 0;
 	FILE **log_file = nullptr; ///< save unclosed log file handle here
 	DesyncDeferredSaveInfo *defer_savegame_write = nullptr;
 };

src/network/core/tcp_game.cpp

@@ -74,6 +74,7 @@ static const char* _packet_game_type_names[] {
 	"CLIENT_DESYNC_LOG",
 	"SERVER_DESYNC_LOG",
 	"CLIENT_DESYNC_MSG",
+	"CLIENT_DESYNC_SYNC_DATA",
 };
 static_assert(lengthof(_packet_game_type_names) == PACKET_END);
 
@@ -175,6 +176,7 @@ NetworkRecvStatus NetworkGameSocketHandler::HandlePacket(Packet *p)
 		case PACKET_CLIENT_DESYNC_LOG:            return this->Receive_CLIENT_DESYNC_LOG(p);
 		case PACKET_SERVER_DESYNC_LOG:            return this->Receive_SERVER_DESYNC_LOG(p);
 		case PACKET_CLIENT_DESYNC_MSG:            return this->Receive_CLIENT_DESYNC_MSG(p);
+		case PACKET_CLIENT_DESYNC_SYNC_DATA:      return this->Receive_CLIENT_DESYNC_SYNC_DATA(p);
 		case PACKET_SERVER_QUIT:                  return this->Receive_SERVER_QUIT(p);
 		case PACKET_SERVER_ERROR_QUIT:            return this->Receive_SERVER_ERROR_QUIT(p);
 		case PACKET_SERVER_SHUTDOWN:              return this->Receive_SERVER_SHUTDOWN(p);
@@ -267,6 +269,7 @@ NetworkRecvStatus NetworkGameSocketHandler::Receive_CLIENT_ERROR(Packet *p) { re
 NetworkRecvStatus NetworkGameSocketHandler::Receive_CLIENT_DESYNC_LOG(Packet *p) { return this->ReceiveInvalidPacket(PACKET_CLIENT_DESYNC_LOG); }
 NetworkRecvStatus NetworkGameSocketHandler::Receive_SERVER_DESYNC_LOG(Packet *p) { return this->ReceiveInvalidPacket(PACKET_SERVER_DESYNC_LOG); }
 NetworkRecvStatus NetworkGameSocketHandler::Receive_CLIENT_DESYNC_MSG(Packet *p) { return this->ReceiveInvalidPacket(PACKET_SERVER_DESYNC_LOG); }
+NetworkRecvStatus NetworkGameSocketHandler::Receive_CLIENT_DESYNC_SYNC_DATA(Packet *p) { return this->ReceiveInvalidPacket(PACKET_CLIENT_DESYNC_SYNC_DATA); }
 NetworkRecvStatus NetworkGameSocketHandler::Receive_SERVER_QUIT(Packet *p) { return this->ReceiveInvalidPacket(PACKET_SERVER_QUIT); }
 NetworkRecvStatus NetworkGameSocketHandler::Receive_SERVER_ERROR_QUIT(Packet *p) { return this->ReceiveInvalidPacket(PACKET_SERVER_ERROR_QUIT); }
 NetworkRecvStatus NetworkGameSocketHandler::Receive_SERVER_SHUTDOWN(Packet *p) { return this->ReceiveInvalidPacket(PACKET_SERVER_SHUTDOWN); }

src/network/core/tcp_game.h

@@ -132,6 +132,7 @@ enum PacketGameType {
 	PACKET_CLIENT_DESYNC_LOG,            ///< A client reports a desync log
 	PACKET_SERVER_DESYNC_LOG,            ///< A server reports a desync log
 	PACKET_CLIENT_DESYNC_MSG,            ///< A client reports a desync message
+	PACKET_CLIENT_DESYNC_SYNC_DATA,      ///< A client reports desync sync data
 
 	PACKET_END,                          ///< Must ALWAYS be on the end of this list!! (period)
 };
@@ -454,6 +455,7 @@ protected:
 	virtual NetworkRecvStatus Receive_CLIENT_DESYNC_LOG(Packet *p);
 	virtual NetworkRecvStatus Receive_SERVER_DESYNC_LOG(Packet *p);
 	virtual NetworkRecvStatus Receive_CLIENT_DESYNC_MSG(Packet *p);
+	virtual NetworkRecvStatus Receive_CLIENT_DESYNC_SYNC_DATA(Packet *p);
 
 	/**
 	 * Notification that a client left the game:

src/network/network.cpp

@@ -91,6 +91,10 @@ uint32 _last_sync_frame_counter;      ///< "
 bool _network_first_time;             ///< Whether we have finished joining or not.
 CompanyMask _network_company_passworded; ///< Bitmask of the password status of all companies.
 
+std::vector<NetworkSyncRecord> _network_client_sync_records;
+std::unique_ptr<std::array<NetworkSyncRecord, 1024>> _network_server_sync_records;
+uint32 _network_server_sync_records_next;
+
 static_assert((int)NETWORK_COMPANY_NAME_LENGTH == MAX_LENGTH_COMPANY_NAME_CHARS * MAX_CHAR_LENGTH);
 
 /** The amount of clients connected */
@@ -647,6 +651,10 @@ void NetworkClose(bool close_admins)
 	_network_company_server_id.clear();
 
 	InitializeNetworkPools(close_admins);
+
+	_network_client_sync_records.clear();
+	_network_client_sync_records.shrink_to_fit();
+	_network_server_sync_records.reset();
 }
 
 /* Initializes the network (cleans sockets and stuff) */
@@ -938,6 +946,10 @@ bool NetworkServerStart()
 	_last_sync_frame = 0;
 	_network_own_client_id = CLIENT_ID_SERVER;
 
+	_network_server_sync_records.reset(new std::array<NetworkSyncRecord, 1024>());
+	_network_server_sync_records->fill({ 0, 0, 0 });
+	_network_server_sync_records_next = 0;
+
 	_network_clients_connected = 0;
 	_network_company_passworded = 0;
 
@@ -1227,6 +1239,9 @@ void NetworkGameLoop()
 #endif
 		_sync_state_checksum = _state_checksum.state;
 
+		(*_network_server_sync_records)[_network_server_sync_records_next] = { _frame_counter, _random.state[0], _state_checksum.state };
+		_network_server_sync_records_next = (_network_server_sync_records_next + 1) % _network_server_sync_records->size();
+
 		NetworkServer_Tick(send_frame);
 	} else {
 		/* Client */

src/network/network_client.cpp

@@ -329,6 +329,7 @@ void ClientNetworkGameSocketHandler::ClientError(NetworkRecvStatus res)
 				info.defer_savegame_write = &deferred_save;
 				CrashLog::DesyncCrashLog(nullptr, &desync_log, info);
 				my_client->SendDesyncLog(desync_log);
+				my_client->SendDesyncSyncData();
 				my_client->ClientError(NETWORK_RECV_STATUS_DESYNC);
 				CrashLog::WriteDesyncSavegame(desync_log.c_str(), deferred_save.name_buffer.c_str());
 				return false;
@@ -337,6 +338,7 @@ void ClientNetworkGameSocketHandler::ClientError(NetworkRecvStatus res)
 			_last_sync_date_fract = _date_fract;
 			_last_sync_tick_skip_counter = _tick_skip_counter;
 			_last_sync_frame_counter = _sync_frame;
+			_network_client_sync_records.clear();
 
 			/* If this is the first time we have a sync-frame, we
 			 *   need to let the server know that we are ready and at the same
@@ -353,6 +355,10 @@ void ClientNetworkGameSocketHandler::ClientError(NetworkRecvStatus res)
 		}
 	}
 
+	if (_network_client_sync_records.size() <= 256) {
+		_network_client_sync_records.push_back({ _frame_counter, _random.state[0], _state_checksum.state });
+	}
+
 	return true;
 }
 
@@ -570,6 +576,22 @@ NetworkRecvStatus ClientNetworkGameSocketHandler::SendDesyncMessage(const char *
 	return NETWORK_RECV_STATUS_OKAY;
 }
 
+/** Send an error-packet over the network */
+NetworkRecvStatus ClientNetworkGameSocketHandler::SendDesyncSyncData()
+{
+	if (_network_client_sync_records.empty()) return NETWORK_RECV_STATUS_OKAY;
+
+	Packet *p = new Packet(PACKET_CLIENT_DESYNC_SYNC_DATA, SHRT_MAX);
+	p->Send_uint32(_network_client_sync_records[0].frame);
+	p->Send_uint32((uint)_network_client_sync_records.size());
+	for (uint i = 0; i < (uint)_network_client_sync_records.size(); i++) {
+		p->Send_uint32(_network_client_sync_records[i].seed_1);
+		p->Send_uint64(_network_client_sync_records[i].state_checksum);
+	}
+	my_client->SendPacket(p);
+	return NETWORK_RECV_STATUS_OKAY;
+}
+
 /**
  * Tell the server that we like to change the password of the company.
  * @param password The new password.

src/network/network_client.h

@@ -95,6 +95,7 @@ public:
 	static NetworkRecvStatus SendError(NetworkErrorCode errorno, NetworkRecvStatus recvstatus = NETWORK_RECV_STATUS_OKAY);
 	static NetworkRecvStatus SendDesyncLog(const std::string &log);
 	static NetworkRecvStatus SendDesyncMessage(const char *msg);
+	static NetworkRecvStatus SendDesyncSyncData();
 	static NetworkRecvStatus SendQuit();
 	static NetworkRecvStatus SendAck();
 

src/network/network_internal.h

@@ -101,6 +101,16 @@ extern uint8 _network_reconnect;
 
 extern CompanyMask _network_company_passworded;
 
+/* Sync debugging */
+struct NetworkSyncRecord {
+	uint32 frame;
+	uint32 seed_1;
+	uint64 state_checksum;
+};
+extern std::vector<NetworkSyncRecord> _network_client_sync_records;
+extern std::unique_ptr<std::array<NetworkSyncRecord, 1024>> _network_server_sync_records;
+extern uint32 _network_server_sync_records_next;
+
 void NetworkQueryServer(const std::string &connection_string);
 
 void GetBindAddresses(NetworkAddressList *addresses, uint16 port);

src/network/network_server.cpp

@@ -1202,6 +1202,8 @@ NetworkRecvStatus ServerNetworkGameSocketHandler::Receive_CLIENT_ERROR(Packet *p
 		DesyncExtraInfo info;
 		info.client_name = client_name;
 		info.client_id = this->client_id;
+		info.desync_frame_seed = this->desync_frame_seed;
+		info.desync_frame_state_checksum = this->desync_frame_state_checksum;
 		CrashLog::DesyncCrashLog(&(this->desync_log), &server_desync_log, info);
 		this->SendDesyncLog(server_desync_log);
 
@@ -1243,6 +1245,39 @@ NetworkRecvStatus ServerNetworkGameSocketHandler::Receive_CLIENT_DESYNC_MSG(Pack
 	return NETWORK_RECV_STATUS_OKAY;
 }
 
+NetworkRecvStatus ServerNetworkGameSocketHandler::Receive_CLIENT_DESYNC_SYNC_DATA(Packet *p)
+{
+	uint32 frame = p->Recv_uint32();
+	uint32 count = p->Recv_uint32();
+
+	DEBUG(net, 2, "Received desync sync data: %u frames from %08X", count, frame);
+
+	uint server_idx = UINT32_MAX;
+	for (uint i = 0; i < _network_server_sync_records->size(); i++) {
+		if ((*_network_server_sync_records)[i].frame == frame) {
+			server_idx = i;
+			break;
+		}
+	}
+	if (server_idx == UINT32_MAX) return NETWORK_RECV_STATUS_OKAY;
+
+	for (uint i = 0; i < count; i++) {
+		uint32 seed_1 = p->Recv_uint32();
+		uint64 state_checksum = p->Recv_uint64();
+
+		const NetworkSyncRecord &record = (*_network_server_sync_records)[server_idx];
+
+		if (record.frame != frame) break;
+		if (record.seed_1 != seed_1 && this->desync_frame_seed == 0) this->desync_frame_seed = frame;
+		if (record.state_checksum != state_checksum && this->desync_frame_state_checksum == 0) this->desync_frame_state_checksum = frame;
+
+		frame++;
+		server_idx = (server_idx + 1) % _network_server_sync_records->size();
+	}
+
+	return NETWORK_RECV_STATUS_OKAY;
+}
+
 NetworkRecvStatus ServerNetworkGameSocketHandler::Receive_CLIENT_QUIT(Packet *p)
 {
 	/* The client wants to leave. Display this and report it to the other

src/network/network_server.h

@@ -39,6 +39,7 @@ protected:
 	NetworkRecvStatus Receive_CLIENT_ERROR(Packet *p) override;
 	NetworkRecvStatus Receive_CLIENT_DESYNC_LOG(Packet *p) override;
 	NetworkRecvStatus Receive_CLIENT_DESYNC_MSG(Packet *p) override;
+	NetworkRecvStatus Receive_CLIENT_DESYNC_SYNC_DATA(Packet *p) override;
 	NetworkRecvStatus Receive_CLIENT_RCON(Packet *p) override;
 	NetworkRecvStatus Receive_CLIENT_NEWGRFS_CHECKED(Packet *p) override;
 	NetworkRecvStatus Receive_CLIENT_MOVE(Packet *p) override;
@@ -86,6 +87,9 @@ public:
 
 	std::string desync_log;
 
+	uint desync_frame_seed = 0;
+	uint desync_frame_state_checksum = 0;
+
 	ServerNetworkGameSocketHandler(SOCKET s);
 	~ServerNetworkGameSocketHandler();