dave/openttd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add: Support Zstandard(zstd) savegame compression

Browse Source 
(cherry picked from commit 6f0aeaf2c5436550c93205e704624957e9abc969)
This commit is contained in:
dP
2021-03-01 01:05:50 +03:00
committed by Jonathan G Rennison
parent 0a31a3b923
commit b284d34277
8 changed files with 234 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/ci-build.yml vendored
View File

@@ -97,6 +97,7 @@ jobs:
libfontconfig-dev \
libicu-dev \
liblzma-dev \
libzstd-dev \
liblzo2-dev \
${{ matrix.libsdl }} \
zlib1g-dev \
@@ -161,7 +162,7 @@ jobs:
vcpkgDirectory: '/usr/local/share/vcpkg'
doNotUpdateVcpkg: false
vcpkgGitCommitId: 2a42024b53ebb512fb5dd63c523338bf26c8489c
vcpkgArguments: 'liblzma libpng lzo'
vcpkgArguments: 'liblzma libpng lzo zstd'
vcpkgTriplet: '${{ matrix.arch }}-osx'
- name: Install OpenGFX
@@ -232,7 +233,7 @@ jobs:
with:
vcpkgDirectory: 'c:/vcpkg'
doNotUpdateVcpkg: true
vcpkgArguments: 'liblzma libpng lzo zlib'
vcpkgArguments: 'liblzma libpng lzo zlib zstd'
vcpkgTriplet: '${{ matrix.arch }}-windows-static'
- name: Install OpenGFX

6
.github/workflows/release.yml vendored
View File

@@ -297,6 +297,7 @@ jobs:
SDL2-devel \
wget \
xz-devel \
libzstd-devel \
zlib-devel \
# EOF
echo "::endgroup::"
@@ -403,6 +404,7 @@ jobs:
libfluidsynth-dev \
libicu-dev \
liblzma-dev \
libzstd-dev \
liblzo2-dev \
libsdl2-dev \
lsb-release \
@@ -482,7 +484,7 @@ jobs:
vcpkgDirectory: '/usr/local/share/vcpkg'
doNotUpdateVcpkg: false
vcpkgGitCommitId: 2a42024b53ebb512fb5dd63c523338bf26c8489c
vcpkgArguments: 'liblzma:x64-osx libpng:x64-osx lzo:x64-osx liblzma:arm64-osx libpng:arm64-osx lzo:arm64-osx'
vcpkgArguments: 'liblzma:x64-osx libpng:x64-osx lzo:x64-osx zstd:x64-osx liblzma:arm64-osx libpng:arm64-osx lzo:arm64-osx zstd:arm64-osx'
- name: Build tools
run: |
@@ -667,7 +669,7 @@ jobs:
with:
vcpkgDirectory: 'c:/vcpkg'
doNotUpdateVcpkg: true
vcpkgArguments: 'liblzma libpng lzo zlib'
vcpkgArguments: 'liblzma libpng lzo zlib zstd'
vcpkgTriplet: '${{ matrix.arch }}-windows-static'
- name: Build tools

2
CMakeLists.txt
View File

@@ -97,6 +97,7 @@ find_package(Threads REQUIRED)
find_package(ZLIB)
find_package(LibLZMA)
find_package(LZO)
find_package(ZSTD)
find_package(PNG)
if(NOT WIN32)
@@ -291,6 +292,7 @@ link_package(PNG TARGET PNG::PNG ENCOURAGED)
link_package(ZLIB TARGET ZLIB::ZLIB ENCOURAGED)
link_package(LIBLZMA TARGET LibLZMA::LibLZMA ENCOURAGED)
link_package(LZO)
link_package(ZSTD TARGET ZSTD::ZSTD ENCOURAGED)
if(NOT OPTION_DEDICATED)
link_package(Fluidsynth)

6
COMPILING.md
View File

@@ -8,6 +8,7 @@ The following libraries are used by OpenTTD for:
heightmaps
- liblzo2: (de)compressing of old (pre 0.3.0) savegames
- liblzma: (de)compressing of savegames (1.1.0 and later)
- libzstd: (de)compressing of savegames (1.11.0 and later)
- libpng: making screenshots and loading heightmaps
- libfreetype: loading generic fonts and rendering them
- libfontconfig: searching for fonts, resolving font names to actual fonts
@@ -45,6 +46,7 @@ After this, you can install the dependencies OpenTTD needs. We advise to use
the `static` versions, and OpenTTD currently needs the following dependencies:
- liblzma
- libzstd
- libpng
- lzo
- zlib
@@ -52,8 +54,8 @@ the `static` versions, and OpenTTD currently needs the following dependencies:
To install both the x64 (64bit) and x86 (32bit) variants (though only one is necessary), you can use:
```ps
.\vcpkg install liblzma:x64-windows-static libpng:x64-windows-static lzo:x64-windows-static zlib:x64-windows-static
.\vcpkg install liblzma:x86-windows-static libpng:x86-windows-static lzo:x86-windows-static zlib:x86-windows-static
.\vcpkg install liblzma:x64-windows-static zstd:x64-windows-static libpng:x64-windows-static lzo:x64-windows-static zlib:x64-windows-static
.\vcpkg install liblzma:x86-windows-static zstd:x86-windows-static libpng:x86-windows-static lzo:x86-windows-static zlib:x86-windows-static
```
You can open the folder (as a CMake project). CMake will be detected, and you can compile from there.

1
Doxyfile.in
View File

@@ -290,6 +290,7 @@ INCLUDE_FILE_PATTERNS =
PREDEFINED = WITH_ZLIB \
WITH_LZO \
WITH_LIBLZMA \
WITH_ZSTD \
WITH_SDL \
WITH_PNG \
WITH_FONTCONFIG \

89
cmake/FindZSTD.cmake Normal file
View File

@@ -0,0 +1,89 @@
#[=======================================================================[.rst:
FindZSTD
-------
Finds the ZSTD library.
Result Variables
^^^^^^^^^^^^^^^^
This will define the following variables:
``ZSTD_FOUND``
True if the system has the ZSTD library.
``ZSTD_INCLUDE_DIRS``
Include directories needed to use ZSTD.
``ZSTD_LIBRARIES``
Libraries needed to link to ZSTD.
``ZSTD_VERSION``
The version of the ZSTD library which was found.
Cache Variables
^^^^^^^^^^^^^^^
The following cache variables may also be set:
``ZSTD_INCLUDE_DIR``
The directory containing ``zstd.h``.
``ZSTD_LIBRARY``
The path to the ZSTD library.
#]=======================================================================]
find_package(PkgConfig QUIET)
pkg_check_modules(PC_ZSTD QUIET libzstd)
find_path(ZSTD_INCLUDE_DIR
NAMES zstd.h
PATHS ${PC_ZSTD_INCLUDE_DIRS}
)
find_library(ZSTD_LIBRARY
NAMES zstd
PATHS ${PC_ZSTD_LIBRARY_DIRS}
)
# With vcpkg, the library path should contain both 'debug' and 'optimized'
# entries (see target_link_libraries() documentation for more information)
#
# NOTE: we only patch up when using vcpkg; the same issue might happen
# when not using vcpkg, but this is non-trivial to fix, as we have no idea
# what the paths are. With vcpkg we do. And we only official support vcpkg
# with Windows.
#
# NOTE: this is based on the assumption that the debug file has the same
# name as the optimized file. This is not always the case, but so far
# experiences has shown that in those case vcpkg CMake files do the right
# thing.
if(VCPKG_TOOLCHAIN AND ZSTD_LIBRARY)
if(ZSTD_LIBRARY MATCHES "/debug/")
set(ZSTD_LIBRARY_DEBUG ${ZSTD_LIBRARY})
string(REPLACE "/debug/lib/" "/lib/" ZSTD_LIBRARY_RELEASE ${ZSTD_LIBRARY})
else()
set(ZSTD_LIBRARY_RELEASE ${ZSTD_LIBRARY})
string(REPLACE "/lib/" "/debug/lib/" ZSTD_LIBRARY_DEBUG ${ZSTD_LIBRARY})
endif()
include(SelectLibraryConfigurations)
select_library_configurations(ZSTD)
endif()
set(ZSTD_VERSION ${PC_ZSTD_VERSION})
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(ZSTD
FOUND_VAR ZSTD_FOUND
REQUIRED_VARS
ZSTD_LIBRARY
ZSTD_INCLUDE_DIR
VERSION_VAR ZSTD_VERSION
)
if(ZSTD_FOUND)
set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
endif()
mark_as_advanced(
ZSTD_INCLUDE_DIR
ZSTD_LIBRARY
)

7
src/crashlog.cpp
View File

@@ -60,6 +60,9 @@
#ifdef WITH_LIBLZMA
# include <lzma.h>
#endif
#ifdef WITH_ZSTD
#include <zstd.h>
#endif
#ifdef WITH_LZO
#include <lzo/lzo1x.h>
#endif
@@ -292,6 +295,10 @@ char *CrashLog::LogLibraries(char *buffer, const char *last) const
buffer += seprintf(buffer, last, " LZMA: %s\n", lzma_version_string());
#endif
#ifdef WITH_ZSTD
buffer += seprintf(buffer, last, " ZSTD: %s\n", ZSTD_versionString());
#endif
#ifdef WITH_LZO
buffer += seprintf(buffer, last, " LZO: %s\n", lzo_version_string());
#endif

124
src/saveload/saveload.cpp
View File

@@ -2761,6 +2761,119 @@ struct LZMASaveFilter : SaveFilter {
#endif /* WITH_LIBLZMA */
/********************************************
********** START OF ZSTD CODE **************
********************************************/
#if defined(WITH_ZSTD)
#include <zstd.h>
/** Filter using ZSTD compression. */
struct ZSTDLoadFilter : LoadFilter {
ZSTD_DCtx *zstd; ///< ZSTD decompression context
byte fread_buf[MEMORY_CHUNK_SIZE]; ///< Buffer for reading from the file
ZSTD_inBuffer input; ///< ZSTD input buffer for fread_buf
/**
* Initialise this filter.
* @param chain The next filter in this chain.
*/
ZSTDLoadFilter(LoadFilter *chain) : LoadFilter(chain)
{
this->zstd = ZSTD_createDCtx();
if (!this->zstd) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor");
this->input = {this->fread_buf, 0, 0};
}
/** Clean everything up. */
~ZSTDLoadFilter()
{
ZSTD_freeDCtx(this->zstd);
}
size_t Read(byte *buf, size_t size) override
{
ZSTD_outBuffer output{buf, size, 0};
do {
/* read more bytes from the file? */
if (this->input.pos == this->input.size) {
this->input.size = this->chain->Read(this->fread_buf, sizeof(this->fread_buf));
this->input.pos = 0;
}
size_t ret = ZSTD_decompressStream(this->zstd, &output, &this->input);
if (ZSTD_isError(ret)) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "libzstd returned error code");
if (ret == 0) break;
} while (output.pos < output.size);
return output.pos;
}
};
/** Filter using ZSTD compression. */
struct ZSTDSaveFilter : SaveFilter {
ZSTD_CCtx *zstd; ///< ZSTD compression context
/**
* Initialise this filter.
* @param chain The next filter in this chain.
* @param compression_level The requested level of compression.
*/
ZSTDSaveFilter(SaveFilter *chain, byte compression_level) : SaveFilter(chain)
{
this->zstd = ZSTD_createCCtx();
if (!this->zstd) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor");
if (ZSTD_isError(ZSTD_CCtx_setParameter(this->zstd, ZSTD_c_compressionLevel, (int)compression_level - 100))) {
ZSTD_freeCCtx(this->zstd);
SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "invalid compresison level");
}
}
/** Clean up what we allocated. */
~ZSTDSaveFilter()
{
ZSTD_freeCCtx(this->zstd);
}
/**
* Helper loop for writing the data.
* @param p The bytes to write.
* @param len Amount of bytes to write.
* @param mode Mode for ZSTD_compressStream2.
*/
void WriteLoop(byte *p, size_t len, ZSTD_EndDirective mode)
{
byte buf[MEMORY_CHUNK_SIZE]; // output buffer
ZSTD_inBuffer input{p, len, 0};
bool finished;
do {
ZSTD_outBuffer output{buf, sizeof(buf), 0};
size_t remaining = ZSTD_compressStream2(this->zstd, &output, &input, mode);
if (ZSTD_isError(remaining)) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "libzstd returned error code");
if (output.pos != 0) this->chain->Write(buf, output.pos);
finished = (mode == ZSTD_e_end ? (remaining == 0) : (input.pos == input.size));
} while (!finished);
}
void Write(byte *buf, size_t size) override
{
this->WriteLoop(buf, size, ZSTD_e_continue);
}
void Finish() override
{
this->WriteLoop(nullptr, 0, ZSTD_e_end);
this->chain->Finish();
}
};
#endif /* WITH_LIBZSTD */
/*******************************************
************* END OF CODE *****************
*******************************************/
@@ -2797,6 +2910,17 @@ static const SaveLoadFormat _saveload_formats[] = {
#else
{"zlib", TO_BE32X('OTTZ'), nullptr, nullptr, 0, 0, 0, false},
#endif
#if defined(WITH_ZSTD)
/* Zstd provides a decent compression rate at a very high compression/decompression speed. Compared to lzma level 2
* zstd saves are about 40% larger (on level 1) but it has about 30x faster compression and 5x decompression making it
* a good choice for multiplayer servers. And zstd level 1 seems to be the optimal one for client connection speed
* (compress + 10 MB/s download + decompress time), about 3x faster than lzma:2 and 1.5x than zlib:2 and lzo.
* As zstd has negative compression levels the values were increased by 100 moving zstd level range -100..22 into
* openttd 0..122. Also note that value 100 mathes zstd level 0 which is a special value for default level 3 (openttd 103) */
{"zstd", TO_BE32X('OTTS'), CreateLoadFilter<ZSTDLoadFilter>, CreateSaveFilter<ZSTDSaveFilter>, 0, 101, 122, false},
#else
{"zstd", TO_BE32X('OTTS'), nullptr, nullptr, 0, 0, 0, false},
#endif
#if defined(WITH_LIBLZMA)
/* Level 2 compression is speed wise as fast as zlib level 6 compression (old default), but results in ~10% smaller saves.
* Higher compression levels are possible, and might improve savegame size by up to 25%, but are also up to 10 times slower.