dave/openttd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Debug: Detect which frame's state first diverged after a desync

Browse Source
This commit is contained in:
Jonathan G Rennison
2022-11-13 01:07:53 +00:00
parent 7582993d00
commit 5e00d535bc
10 changed files with 97 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/crashlog.cpp
View File

@@ -522,6 +522,9 @@ char *CrashLog::FillDesyncCrashLog(char *buffer, const char *last, const DesyncE
flag_check(DesyncExtraInfo::DEIF_STATE, "S"),
flag_check(DesyncExtraInfo::DEIF_DBL_RAND, "D"));
}
if (_network_server && (info.desync_frame_seed || info.desync_frame_state_checksum)) {
buffer += seprintf(buffer, last, "Desync frame: %08X (seed), %08X (state checksum)\n", info.desync_frame_seed, info.desync_frame_state_checksum);
}
extern uint32 _frame_counter;

2
src/crashlog.h
View File

@@ -30,6 +30,8 @@ struct DesyncExtraInfo {
Flags flags = DEIF_NONE;
const char *client_name = nullptr;
int client_id = -1;
uint32 desync_frame_seed = 0;
uint32 desync_frame_state_checksum = 0;
FILE **log_file = nullptr; ///< save unclosed log file handle here
DesyncDeferredSaveInfo *defer_savegame_write = nullptr;
};

3
src/network/core/tcp_game.cpp
View File

@@ -74,6 +74,7 @@ static const char* _packet_game_type_names[] {
"CLIENT_DESYNC_LOG",
"SERVER_DESYNC_LOG",
"CLIENT_DESYNC_MSG",
"CLIENT_DESYNC_SYNC_DATA",
};
static_assert(lengthof(_packet_game_type_names) == PACKET_END);
@@ -175,6 +176,7 @@ NetworkRecvStatus NetworkGameSocketHandler::HandlePacket(Packet *p)
case PACKET_CLIENT_DESYNC_LOG: return this->Receive_CLIENT_DESYNC_LOG(p);
case PACKET_SERVER_DESYNC_LOG: return this->Receive_SERVER_DESYNC_LOG(p);
case PACKET_CLIENT_DESYNC_MSG: return this->Receive_CLIENT_DESYNC_MSG(p);
case PACKET_CLIENT_DESYNC_SYNC_DATA: return this->Receive_CLIENT_DESYNC_SYNC_DATA(p);
case PACKET_SERVER_QUIT: return this->Receive_SERVER_QUIT(p);
case PACKET_SERVER_ERROR_QUIT: return this->Receive_SERVER_ERROR_QUIT(p);
case PACKET_SERVER_SHUTDOWN: return this->Receive_SERVER_SHUTDOWN(p);
@@ -267,6 +269,7 @@ NetworkRecvStatus NetworkGameSocketHandler::Receive_CLIENT_ERROR(Packet *p) { re
NetworkRecvStatus NetworkGameSocketHandler::Receive_CLIENT_DESYNC_LOG(Packet *p) { return this->ReceiveInvalidPacket(PACKET_CLIENT_DESYNC_LOG); }
NetworkRecvStatus NetworkGameSocketHandler::Receive_SERVER_DESYNC_LOG(Packet *p) { return this->ReceiveInvalidPacket(PACKET_SERVER_DESYNC_LOG); }
NetworkRecvStatus NetworkGameSocketHandler::Receive_CLIENT_DESYNC_MSG(Packet *p) { return this->ReceiveInvalidPacket(PACKET_SERVER_DESYNC_LOG); }
NetworkRecvStatus NetworkGameSocketHandler::Receive_CLIENT_DESYNC_SYNC_DATA(Packet *p) { return this->ReceiveInvalidPacket(PACKET_CLIENT_DESYNC_SYNC_DATA); }
NetworkRecvStatus NetworkGameSocketHandler::Receive_SERVER_QUIT(Packet *p) { return this->ReceiveInvalidPacket(PACKET_SERVER_QUIT); }
NetworkRecvStatus NetworkGameSocketHandler::Receive_SERVER_ERROR_QUIT(Packet *p) { return this->ReceiveInvalidPacket(PACKET_SERVER_ERROR_QUIT); }
NetworkRecvStatus NetworkGameSocketHandler::Receive_SERVER_SHUTDOWN(Packet *p) { return this->ReceiveInvalidPacket(PACKET_SERVER_SHUTDOWN); }

2
src/network/core/tcp_game.h
View File

@@ -132,6 +132,7 @@ enum PacketGameType {
PACKET_CLIENT_DESYNC_LOG, ///< A client reports a desync log
PACKET_SERVER_DESYNC_LOG, ///< A server reports a desync log
PACKET_CLIENT_DESYNC_MSG, ///< A client reports a desync message
PACKET_CLIENT_DESYNC_SYNC_DATA, ///< A client reports desync sync data
PACKET_END, ///< Must ALWAYS be on the end of this list!! (period)
};
@@ -454,6 +455,7 @@ protected:
virtual NetworkRecvStatus Receive_CLIENT_DESYNC_LOG(Packet *p);
virtual NetworkRecvStatus Receive_SERVER_DESYNC_LOG(Packet *p);
virtual NetworkRecvStatus Receive_CLIENT_DESYNC_MSG(Packet *p);
virtual NetworkRecvStatus Receive_CLIENT_DESYNC_SYNC_DATA(Packet *p);
/**
* Notification that a client left the game:

15
src/network/network.cpp
View File

@@ -91,6 +91,10 @@ uint32 _last_sync_frame_counter; ///< "
bool _network_first_time; ///< Whether we have finished joining or not.
CompanyMask _network_company_passworded; ///< Bitmask of the password status of all companies.
std::vector<NetworkSyncRecord> _network_client_sync_records;
std::unique_ptr<std::array<NetworkSyncRecord, 1024>> _network_server_sync_records;
uint32 _network_server_sync_records_next;
static_assert((int)NETWORK_COMPANY_NAME_LENGTH == MAX_LENGTH_COMPANY_NAME_CHARS * MAX_CHAR_LENGTH);
/** The amount of clients connected */
@@ -647,6 +651,10 @@ void NetworkClose(bool close_admins)
_network_company_server_id.clear();
InitializeNetworkPools(close_admins);
_network_client_sync_records.clear();
_network_client_sync_records.shrink_to_fit();
_network_server_sync_records.reset();
}
/* Initializes the network (cleans sockets and stuff) */
@@ -938,6 +946,10 @@ bool NetworkServerStart()
_last_sync_frame = 0;
_network_own_client_id = CLIENT_ID_SERVER;
_network_server_sync_records.reset(new std::array<NetworkSyncRecord, 1024>());
_network_server_sync_records->fill({ 0, 0, 0 });
_network_server_sync_records_next = 0;
_network_clients_connected = 0;
_network_company_passworded = 0;
@@ -1227,6 +1239,9 @@ void NetworkGameLoop()
#endif
_sync_state_checksum = _state_checksum.state;
(*_network_server_sync_records)[_network_server_sync_records_next] = { _frame_counter, _random.state[0], _state_checksum.state };
_network_server_sync_records_next = (_network_server_sync_records_next + 1) % _network_server_sync_records->size();
NetworkServer_Tick(send_frame);
} else {
/* Client */

22
src/network/network_client.cpp
View File

@@ -329,6 +329,7 @@ void ClientNetworkGameSocketHandler::ClientError(NetworkRecvStatus res)
info.defer_savegame_write = &deferred_save;
CrashLog::DesyncCrashLog(nullptr, &desync_log, info);
my_client->SendDesyncLog(desync_log);
my_client->SendDesyncSyncData();
my_client->ClientError(NETWORK_RECV_STATUS_DESYNC);
CrashLog::WriteDesyncSavegame(desync_log.c_str(), deferred_save.name_buffer.c_str());
return false;
@@ -337,6 +338,7 @@ void ClientNetworkGameSocketHandler::ClientError(NetworkRecvStatus res)
_last_sync_date_fract = _date_fract;
_last_sync_tick_skip_counter = _tick_skip_counter;
_last_sync_frame_counter = _sync_frame;
_network_client_sync_records.clear();
/* If this is the first time we have a sync-frame, we
* need to let the server know that we are ready and at the same
@@ -353,6 +355,10 @@ void ClientNetworkGameSocketHandler::ClientError(NetworkRecvStatus res)
}
}
if (_network_client_sync_records.size() <= 256) {
_network_client_sync_records.push_back({ _frame_counter, _random.state[0], _state_checksum.state });
}
return true;
}
@@ -570,6 +576,22 @@ NetworkRecvStatus ClientNetworkGameSocketHandler::SendDesyncMessage(const char *
return NETWORK_RECV_STATUS_OKAY;
}
/** Send an error-packet over the network */
NetworkRecvStatus ClientNetworkGameSocketHandler::SendDesyncSyncData()
{
if (_network_client_sync_records.empty()) return NETWORK_RECV_STATUS_OKAY;
Packet *p = new Packet(PACKET_CLIENT_DESYNC_SYNC_DATA, SHRT_MAX);
p->Send_uint32(_network_client_sync_records[0].frame);
p->Send_uint32((uint)_network_client_sync_records.size());
for (uint i = 0; i < (uint)_network_client_sync_records.size(); i++) {
p->Send_uint32(_network_client_sync_records[i].seed_1);
p->Send_uint64(_network_client_sync_records[i].state_checksum);
}
my_client->SendPacket(p);
return NETWORK_RECV_STATUS_OKAY;
}
/**
* Tell the server that we like to change the password of the company.
* @param password The new password.

1
src/network/network_client.h
View File

@@ -95,6 +95,7 @@ public:
static NetworkRecvStatus SendError(NetworkErrorCode errorno, NetworkRecvStatus recvstatus = NETWORK_RECV_STATUS_OKAY);
static NetworkRecvStatus SendDesyncLog(const std::string &log);
static NetworkRecvStatus SendDesyncMessage(const char *msg);
static NetworkRecvStatus SendDesyncSyncData();
static NetworkRecvStatus SendQuit();
static NetworkRecvStatus SendAck();

10
src/network/network_internal.h
View File

@@ -101,6 +101,16 @@ extern uint8 _network_reconnect;
extern CompanyMask _network_company_passworded;
/* Sync debugging */
struct NetworkSyncRecord {
uint32 frame;
uint32 seed_1;
uint64 state_checksum;
};
extern std::vector<NetworkSyncRecord> _network_client_sync_records;
extern std::unique_ptr<std::array<NetworkSyncRecord, 1024>> _network_server_sync_records;
extern uint32 _network_server_sync_records_next;
void NetworkQueryServer(const std::string &connection_string);
void GetBindAddresses(NetworkAddressList *addresses, uint16 port);

35
src/network/network_server.cpp
View File

@@ -1202,6 +1202,8 @@ NetworkRecvStatus ServerNetworkGameSocketHandler::Receive_CLIENT_ERROR(Packet *p
DesyncExtraInfo info;
info.client_name = client_name;
info.client_id = this->client_id;
info.desync_frame_seed = this->desync_frame_seed;
info.desync_frame_state_checksum = this->desync_frame_state_checksum;
CrashLog::DesyncCrashLog(&(this->desync_log), &server_desync_log, info);
this->SendDesyncLog(server_desync_log);
@@ -1243,6 +1245,39 @@ NetworkRecvStatus ServerNetworkGameSocketHandler::Receive_CLIENT_DESYNC_MSG(Pack
return NETWORK_RECV_STATUS_OKAY;
}
NetworkRecvStatus ServerNetworkGameSocketHandler::Receive_CLIENT_DESYNC_SYNC_DATA(Packet *p)
{
uint32 frame = p->Recv_uint32();
uint32 count = p->Recv_uint32();
DEBUG(net, 2, "Received desync sync data: %u frames from %08X", count, frame);
uint server_idx = UINT32_MAX;
for (uint i = 0; i < _network_server_sync_records->size(); i++) {
if ((*_network_server_sync_records)[i].frame == frame) {
server_idx = i;
break;
}
}
if (server_idx == UINT32_MAX) return NETWORK_RECV_STATUS_OKAY;
for (uint i = 0; i < count; i++) {
uint32 seed_1 = p->Recv_uint32();
uint64 state_checksum = p->Recv_uint64();
const NetworkSyncRecord &record = (*_network_server_sync_records)[server_idx];
if (record.frame != frame) break;
if (record.seed_1 != seed_1 && this->desync_frame_seed == 0) this->desync_frame_seed = frame;
if (record.state_checksum != state_checksum && this->desync_frame_state_checksum == 0) this->desync_frame_state_checksum = frame;
frame++;
server_idx = (server_idx + 1) % _network_server_sync_records->size();
}
return NETWORK_RECV_STATUS_OKAY;
}
NetworkRecvStatus ServerNetworkGameSocketHandler::Receive_CLIENT_QUIT(Packet *p)
{
/* The client wants to leave. Display this and report it to the other

4
src/network/network_server.h
View File

@@ -39,6 +39,7 @@ protected:
NetworkRecvStatus Receive_CLIENT_ERROR(Packet *p) override;
NetworkRecvStatus Receive_CLIENT_DESYNC_LOG(Packet *p) override;
NetworkRecvStatus Receive_CLIENT_DESYNC_MSG(Packet *p) override;
NetworkRecvStatus Receive_CLIENT_DESYNC_SYNC_DATA(Packet *p) override;
NetworkRecvStatus Receive_CLIENT_RCON(Packet *p) override;
NetworkRecvStatus Receive_CLIENT_NEWGRFS_CHECKED(Packet *p) override;
NetworkRecvStatus Receive_CLIENT_MOVE(Packet *p) override;
@@ -86,6 +87,9 @@ public:
std::string desync_log;
uint desync_frame_seed = 0;
uint desync_frame_state_checksum = 0;
ServerNetworkGameSocketHandler(SOCKET s);
~ServerNetworkGameSocketHandler();