diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d89374a..a538fa7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,10 +6,40 @@ on: - published jobs: + sde: + name: Fetch SDE + runs-on: windows-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Python2 + run: | + choco install python2 + C:\Python27\python.exe -m pip install --upgrade pip + C:\Python27\python.exe -m pip install requests + + - name: Download loaders + run: | + C:\Python27\python.exe download_sde/download_loaders.py + + - name: Execute loaders + run: | + C:\Python27\python.exe download_sde/execute_loaders.py + + - name: Publish artifact + uses: actions/upload-artifact@v4 + with: + name: sde + path: json + datafiles: name: Publish datafiles runs-on: ubuntu-latest + needs: [sde] + steps: - name: Checkout uses: actions/checkout@v4 @@ -51,33 +81,21 @@ jobs: echo "application/x-protobuf pb2" | sudo tee -a /etc/mime.types echo "80:application/x-protobuf:*.pb2" | sudo tee -a /usr/share/mime/globs2 - - name: Fetch SDE - run: | - wget -q https://eve-static-data-export.s3-eu-west-1.amazonaws.com/tranquility/sde.zip - unzip sde.zip - - - name: Validate SDE version - run: | - SDE_VERSION=$(date -r sde.zip "+%F" | sed 's/-//g') - RELEASE_SDE_VERSION=$(echo ${{ github.ref_name }} | cut -d. -f3) - - echo "SDK version: ${SDE_VERSION}" - echo "Release version: ${RELEASE_SDE_VERSION}" - - if [ "${SDE_VERSION}" != "${RELEASE_SDE_VERSION}" ]; then - echo "SDE version mismatch: ${SDE_VERSION} != ${RELEASE_SDE_VERSION}" - exit 1 - fi + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: sde + path: sde - name: Convert SDE YAML to Protobuf run: | protoc --python_out=. esf.proto - python convert.py sde/fsd - python list_shiptypes.py sde/fsd + python -m convert sde + python -m list_shiptypes sde - name: Fetch icons run: | - python download_icons.py sde/fsd + python -m download_icons sde - name: Build package run: | diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index b74f7b0..fae9c90 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -26,7 +26,89 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Install dependencies - run: npm install + run: | + npm install + pip install black flake8 + + - uses: TrueBrain/actions-flake8@v2 + with: + path: convert download_icons download_sde list_shiptypes + max_line_length: 120 - name: Run linter - run: npm run lint + run: | + npm run lint + black --check -l 120 convert download_icons download_sde list_shiptypes + + sde: + name: Fetch SDE + runs-on: windows-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Python2 + run: | + choco install python2 + C:\Python27\python.exe -m pip install --upgrade pip + C:\Python27\python.exe -m pip install requests + + - name: Download loaders + run: | + C:\Python27\python.exe download_sde/download_loaders.py + + - name: Execute loaders + run: | + C:\Python27\python.exe download_sde/execute_loaders.py + + - name: Publish artifact + uses: actions/upload-artifact@v4 + with: + name: sde + path: json + + datafiles: + name: Generate datafiles + runs-on: ubuntu-latest + + needs: [sde] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install NodeJS + uses: actions/setup-node@v4 + with: + registry-url: https://npm.pkg.github.com + scope: "@eveshipfit" + env: + NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y --no-install-recommends protobuf-compiler + pip install -r requirements.txt + npm install + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: sde + path: sde + + - name: Convert SDE YAML to Protobuf + run: | + protoc --python_out=. esf.proto + python -m convert sde + python -m list_shiptypes sde + + - name: Fetch icons + run: | + python -m download_icons sde + + - name: Build package + run: | + npm run build diff --git a/.gitignore b/.gitignore index 029d9ba..bf2bfae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,11 @@ -/__pycache__/ +__pycache__/ /.env/ +/data/ /dist/ /esf_pb2.js /esf_pb2.py +/json/ /node_modules/ /package-lock.json +/pyd/ /sde/ diff --git a/README.md b/README.md index cd23722..1d1ba9b 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,31 @@ Download the latest EVE SDE from [their website](https://developers.eveonline.co Now run the tool: ```bash -python convert.py +python -m convert ``` This will take a while to generate the protobuf files, but they will be outputed in the `dist` folder. +### SDE based on EVE client information + +The convert script also supports loading the information from data taken from the latest EVE client. +In general, this is more up-to-date than the published SDE. +The downside is, that it requires Python2 on Windows in order to extract. + +How it works: +- It downloads several `.pyd` files (which are actually DLLs, so this only works on Windows) from the installer. +- It downloads `.fsdbinary` files from the installer. +- It loads the `.pyd` files in a Python2 context. These files contain information how to load the `.fsdbinary` files. +- It exports the result as `.json` files in the `json/` folder. + +Use the `download_sde/download_loaders.py` to download the files, and `download_sde/execute_loaders.py` to convert them to JSON. + +Now convert them with: + +```bash +python -m convert +``` + ## Patches The EVE SDE has some quirks, that are easiest fixed in the conversion. diff --git a/convert.py b/convert/__main__.py similarity index 79% rename from convert.py rename to convert/__main__.py index ff3bef1..7b5da54 100644 --- a/convert.py +++ b/convert/__main__.py @@ -1,9 +1,12 @@ +import json import os import sys import yaml import esf_pb2 +from google.protobuf.json_format import MessageToJson + if len(sys.argv) < 2: print("Usage: python3 convert.py ") exit(1) @@ -11,13 +14,19 @@ if len(sys.argv) < 2: path = sys.argv[1] os.makedirs("dist/sde", exist_ok=True) +os.makedirs("dist/sde_json", exist_ok=True) def convert_type_dogma(path, ships): print("Converting typeDogma ...") - with open(f"{path}/typeDogma.yaml") as fp: - typeDogma = yaml.load(fp, Loader=yaml.CSafeLoader) + try: + with open(f"{path}/typeDogma.yaml") as fp: + typeDogma = yaml.load(fp, Loader=yaml.CSafeLoader) + except FileNotFoundError: + with open(f"{path}/typedogma.json") as fp: + typeDogma = json.load(fp) + typeDogma = {int(k): v for k, v in typeDogma.items()} pb2 = esf_pb2.TypeDogma() @@ -45,21 +54,34 @@ def convert_type_dogma(path, ships): with open("dist/sde/typeDogma.pb2", "wb") as fp: fp.write(pb2.SerializeToString()) + with open("dist/sde_json/typeDogma.json", "w") as fp: + fp.write(MessageToJson(pb2, sort_keys=True)) + def convert_type_ids(path): print("Converting typeIDs ...") - with open(f"{path}/groupIDs.yaml") as fp: - groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader) + try: + with open(f"{path}/groupIDs.yaml") as fp: + groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader) + except FileNotFoundError: + with open(f"{path}/groups.json") as fp: + groupIDs = json.load(fp) + groupIDs = {int(k): v for k, v in groupIDs.items()} - with open(f"{path}/typeIDs.yaml") as fp: - typeIDs = yaml.load(fp, Loader=yaml.CSafeLoader) + try: + with open(f"{path}/typeIDs.yaml") as fp: + typeIDs = yaml.load(fp, Loader=yaml.CSafeLoader) + except FileNotFoundError: + with open(f"{path}/types.json") as fp: + typeIDs = json.load(fp) + typeIDs = {int(k): v for k, v in typeIDs.items()} pb2 = esf_pb2.TypeIDs() ships = [] for id, entry in typeIDs.items(): - pb2.entries[id].name = entry["name"]["en"] + pb2.entries[id].name = entry["name"]["en"] if "name" in entry else entry["typeNameID"] pb2.entries[id].groupID = entry["groupID"] pb2.entries[id].categoryID = groupIDs[entry["groupID"]]["categoryID"] pb2.entries[id].published = entry["published"] @@ -73,48 +95,64 @@ def convert_type_ids(path): pb2.entries[id].marketGroupID = entry["marketGroupID"] if "metaGroupID" in entry: pb2.entries[id].metaGroupID = entry["metaGroupID"] - if "capacity" in entry: + if "capacity" in entry and entry["capacity"] != 0.0: pb2.entries[id].capacity = entry["capacity"] - if "mass" in entry: + if "mass" in entry and entry["mass"] != 0.0: pb2.entries[id].mass = entry["mass"] - if "radius" in entry: + if "radius" in entry and entry["radius"] != 1.0: pb2.entries[id].radius = entry["radius"] - if "volume" in entry: + if "volume" in entry and entry["volume"] != 0.0: pb2.entries[id].volume = entry["volume"] with open("dist/sde/typeIDs.pb2", "wb") as fp: fp.write(pb2.SerializeToString()) + with open("dist/sde_json/typeIDs.json", "w") as fp: + fp.write(MessageToJson(pb2, sort_keys=True)) + return ships def convert_group_ids(path): print("Converting groupIDs ...") - with open(f"{path}/groupIDs.yaml") as fp: - groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader) + try: + with open(f"{path}/groupIDs.yaml") as fp: + groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader) + except FileNotFoundError: + with open(f"{path}/groups.json") as fp: + groupIDs = json.load(fp) + groupIDs = {int(k): v for k, v in groupIDs.items()} pb2 = esf_pb2.GroupIDs() for id, entry in groupIDs.items(): - pb2.entries[id].name = entry["name"]["en"] + pb2.entries[id].name = entry["name"]["en"] if "name" in entry else entry["groupNameID"] pb2.entries[id].categoryID = entry["categoryID"] pb2.entries[id].published = entry["published"] with open("dist/sde/groupIDs.pb2", "wb") as fp: fp.write(pb2.SerializeToString()) + with open("dist/sde_json/groupIDs.json", "w") as fp: + fp.write(MessageToJson(pb2, sort_keys=True)) + def convert_market_groups(path): print("Converting marketGroups ...") - with open(f"{path}/marketGroups.yaml") as fp: - marketGroupIDs = yaml.load(fp, Loader=yaml.CSafeLoader) + try: + with open(f"{path}/marketGroups.yaml") as fp: + marketGroupIDs = yaml.load(fp, Loader=yaml.CSafeLoader) + except FileNotFoundError: + with open(f"{path}/marketgroups.json") as fp: + marketGroupIDs = json.load(fp) + marketGroupIDs = {int(k): v for k, v in marketGroupIDs.items()} pb2 = esf_pb2.MarketGroups() for id, entry in marketGroupIDs.items(): - pb2.entries[id].name = entry["nameID"]["en"] + pb2.entries[id].name = entry["nameID"] if isinstance(entry["nameID"], str) else entry["nameID"]["en"] if "parentGroupID" in entry: pb2.entries[id].parentGroupID = entry["parentGroupID"] @@ -124,12 +162,20 @@ def convert_market_groups(path): with open("dist/sde/marketGroups.pb2", "wb") as fp: fp.write(pb2.SerializeToString()) + with open("dist/sde_json/marketGroups.json", "w") as fp: + fp.write(MessageToJson(pb2, sort_keys=True)) + def convert_dogma_attributes(path): print("Converting dogmaAttributes ...") - with open(f"{path}/dogmaAttributes.yaml") as fp: - dogmaAttributes = yaml.load(fp, Loader=yaml.CSafeLoader) + try: + with open(f"{path}/dogmaAttributes.yaml") as fp: + dogmaAttributes = yaml.load(fp, Loader=yaml.CSafeLoader) + except FileNotFoundError: + with open(f"{path}/dogmaattributes.json") as fp: + dogmaAttributes = json.load(fp) + dogmaAttributes = {int(k): v for k, v in dogmaAttributes.items()} pb2 = esf_pb2.DogmaAttributes() @@ -187,12 +233,20 @@ def convert_dogma_attributes(path): with open("dist/sde/dogmaAttributes.pb2", "wb") as fp: fp.write(pb2.SerializeToString()) + with open("dist/sde_json/dogmaAttributes.json", "w") as fp: + fp.write(MessageToJson(pb2, sort_keys=True)) + def convert_dogma_effects(path): print("Converting dogmaEffects ...") - with open(f"{path}/dogmaEffects.yaml") as fp: - dogmaEffects = yaml.load(fp, Loader=yaml.CSafeLoader) + try: + with open(f"{path}/dogmaEffects.yaml") as fp: + dogmaEffects = yaml.load(fp, Loader=yaml.CSafeLoader) + except FileNotFoundError: + with open(f"{path}/dogmaeffects.json") as fp: + dogmaEffects = json.load(fp) + dogmaEffects = {int(k): v for k, v in dogmaEffects.items()} pb2 = esf_pb2.DogmaEffects() pbmi = pb2.DogmaEffect.ModifierInfo() @@ -337,7 +391,8 @@ def convert_dogma_effects(path): if entry["effectName"] == "moduleBonusAfterburner" or entry["effectName"] == "moduleBonusMicrowarpdrive": add_modifier(id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, 4, 2, 796) # mass massAddition - # Velocity change is calculated like this: velocityBoost = item.speedFactor * item.speedBoostFactor / ship.mass + # Velocity change is calculated like this: + # velocityBoost = item.speedFactor * item.speedBoostFactor / ship.mass # First, calculate the multiplication on the item. add_modifier( id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, -7, -1, 567 @@ -346,7 +401,8 @@ def convert_dogma_effects(path): id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, -7, 4, 20 ) # velocityBoost speedFactor - # Next, "applyVelocityBoost" is applied on all ships which takes care of the final calculation (as mass is an attribute of the ship). + # Next, "applyVelocityBoost" is applied on all ships which takes care of + # the final calculation (as mass is an attribute of the ship). # missileEMDmgBonus, missileExplosiveDmgBonus, missileKineticDmgBonus, missileThermalDmgBonus don't apply # any effect direct, but this is handled internally in EVE. For us, @@ -379,6 +435,9 @@ def convert_dogma_effects(path): with open("dist/sde/dogmaEffects.pb2", "wb") as fp: fp.write(pb2.SerializeToString()) + with open("dist/sde_json/dogmaEffects.json", "w") as fp: + fp.write(MessageToJson(pb2, sort_keys=True)) + convert_group_ids(path) convert_market_groups(path) diff --git a/download_icons.py b/download_icons/__main__.py similarity index 71% rename from download_icons.py rename to download_icons/__main__.py index df8d659..b2a6dcb 100644 --- a/download_icons.py +++ b/download_icons/__main__.py @@ -1,3 +1,4 @@ +import json import os import requests import sys @@ -17,14 +18,29 @@ folders = [ ] files = {} -with open(f"{path}/marketGroups.yaml") as fp: - marketGroups = yaml.load(fp, Loader=yaml.CSafeLoader) +try: + with open(f"{path}/marketGroups.yaml") as fp: + marketGroups = yaml.load(fp, Loader=yaml.CSafeLoader) +except FileNotFoundError: + with open(f"{path}/marketgroups.json") as fp: + marketGroups = json.load(fp) + marketGroups = {int(k): v for k, v in marketGroups.items()} -with open(f"{path}/metaGroups.yaml") as fp: - metaGroups = yaml.load(fp, Loader=yaml.CSafeLoader) +try: + with open(f"{path}/metaGroups.yaml") as fp: + metaGroups = yaml.load(fp, Loader=yaml.CSafeLoader) +except FileNotFoundError: + with open(f"{path}/metagroups.json") as fp: + metaGroups = json.load(fp) + metaGroups = {int(k): v for k, v in metaGroups.items()} -with open(f"{path}/iconIDs.yaml") as fp: - iconIDs = yaml.load(fp, Loader=yaml.CSafeLoader) +try: + with open(f"{path}/iconIDs.yaml") as fp: + iconIDs = yaml.load(fp, Loader=yaml.CSafeLoader) +except FileNotFoundError: + with open(f"{path}/iconids.json") as fp: + iconIDs = json.load(fp) + iconIDs = {int(k): v for k, v in iconIDs.items()} for marketGroupID, marketGroup in marketGroups.items(): if "iconID" not in marketGroup or marketGroup["iconID"] == 0: diff --git a/download_sde/download_loaders.py b/download_sde/download_loaders.py new file mode 100644 index 0000000..7f23fbb --- /dev/null +++ b/download_sde/download_loaders.py @@ -0,0 +1,73 @@ +import os +import requests + +# Only download these loaders and their data. +LOADER_LIST = [ + "categories", + "dogmaattributes", + "dogmaeffects", + "iconids", + "groups", + "marketgroups", + "metagroups", + "typedogma", + "types", +] + +os.makedirs("pyd") +os.makedirs("data") + +session = requests.Session() + +# Find the latest installer listing. +latest = session.get("https://binaries.eveonline.com/eveclient_TQ.json").json() +build = latest["build"] +installer = session.get("https://binaries.eveonline.com/eveonline_" + build + ".txt").text + +# Download all the loaders. +resfileindex = None +for line in installer.split("\n"): + if not line: + continue + + res, path, _, _, _, _ = line.split(",") + if res == "app:/resfileindex.txt": + resfileindex = line.split(",")[1] + + if not res.startswith("app:/bin64") or not res.endswith("Loader.pyd"): + continue + loader = res.split("/")[-1][:-10].lower() + if loader not in LOADER_LIST: + continue + + local_path = "pyd/" + res.split("/")[-1] + + print("Downloading " + local_path + " ...") + + with open(local_path, "wb") as f: + f.write(session.get("https://binaries.eveonline.com/" + path).content) + +if resfileindex is None: + raise Exception("resfileindex not found") + +# Download all the fsdbinary files. +resfile = requests.get("https://binaries.eveonline.com/" + resfileindex).text +for line in resfile.split("\n"): + if not line: + continue + + res, path, _, _, _ = line.split(",") + if ( + not res.startswith("res:/staticdata/") or not res.endswith(".fsdbinary") + ) and res != "res:/localizationfsd/localization_fsd_en-us.pickle": + continue + loader = res.split("/")[-1][:-10] + if res != "res:/localizationfsd/localization_fsd_en-us.pickle" and loader not in LOADER_LIST: + continue + + local_path = "data/" + res.split("/")[-1] + + print("Downloading " + local_path + " ...") + + with open(local_path, "wb") as f: + f.write(session.get("https://resources.eveonline.com/" + path).content) diff --git a/download_sde/execute_loaders.py b/download_sde/execute_loaders.py new file mode 100644 index 0000000..8c07080 --- /dev/null +++ b/download_sde/execute_loaders.py @@ -0,0 +1,60 @@ +import pickle +import glob +import importlib +import json +import os +import sys + +sys.path.append("pyd") +os.makedirs("json") + + +def decode_cfsd(key, data, strings): + data_type = type(data) + + if data_type.__module__ == "cfsd" and data_type.__name__ == "dict": + return {k: decode_cfsd(k, v, strings) for k, v in data.items()} + if data_type.__module__.endswith("Loader"): + return {x: decode_cfsd(x, getattr(data, x), strings) for x in dir(data) if not x.startswith("__")} + + if data_type.__module__ == "cfsd" and data_type.__name__ == "list": + return [decode_cfsd(None, v, strings) for v in data] + if isinstance(data, tuple): + return tuple([decode_cfsd(None, v, strings) for v in data]) + + if data_type.__name__.endswith("_vector"): + # TODO + return None + + if isinstance(data, int) or data_type.__name__ == "long": + # In case it is a NameID, look up the name. + if key is not None and isinstance(key, str) and key.lower().endswith("nameid") and key != "dungeonNameID": + return strings[data][0] + return data + if isinstance(data, float): + return data + if isinstance(data, str): + return data + + raise ValueError("Unknown type: " + str(type(data))) + + +# Load all the english strings. +print("Loading 'localization_fsd_en-us.pickle' ...") +with open("data/localization_fsd_en-us.pickle", "rb") as f: + strings = pickle.load(f)[1] + + +# Convert all available fsdbinary files via their Loader to JSON. +for loader in glob.glob("pyd/*Loader.pyd"): + loader_name = os.path.splitext(os.path.basename(loader))[0] + data_name = loader_name.replace("Loader", "").lower() + ".fsdbinary" + + print("Loading '" + data_name + "' with '" + loader_name + "' ...") + + lib = importlib.import_module(loader_name) + data = lib.load("data/" + data_name) + data = decode_cfsd(None, data, strings) + + with open("json/" + data_name.replace(".fsdbinary", ".json"), "w") as f: + json.dump(data, f, indent=4) diff --git a/list_shiptypes.py b/list_shiptypes.py deleted file mode 100644 index d7a7fc4..0000000 --- a/list_shiptypes.py +++ /dev/null @@ -1,35 +0,0 @@ -import json -import os -import sys -import yaml - -if len(sys.argv) < 2: - print("Usage: python3 convert.py ") - exit(1) - -path = sys.argv[1] - -os.makedirs("dist", exist_ok=True) - -with open(f"{path}/groupIDs.yaml") as fp: - groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader) - -with open(f"{path}/typeIDs.yaml") as fp: - typeIDs = yaml.load(fp, Loader=yaml.CSafeLoader) - -ships = [] - -for id, entry in typeIDs.items(): - group = groupIDs[entry["groupID"]] - - if group["categoryID"] == 6 and entry["published"]: - ships.append( - { - "id": id, - "name": entry["name"]["en"], - "group": group["name"]["en"], - } - ) - -with open("dist/shiptypes.json", "w") as fp: - json.dump(ships, fp) diff --git a/list_shiptypes/__main__.py b/list_shiptypes/__main__.py new file mode 100644 index 0000000..3fd3fff --- /dev/null +++ b/list_shiptypes/__main__.py @@ -0,0 +1,45 @@ +import json +import os +import sys +import yaml + +if len(sys.argv) < 2: + print("Usage: python3 convert.py ") + exit(1) + +path = sys.argv[1] + +os.makedirs("dist", exist_ok=True) + +try: + with open(f"{path}/groupIDs.yaml") as fp: + groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader) +except FileNotFoundError: + with open(f"{path}/groups.json") as fp: + groupIDs = json.load(fp) + groupIDs = {int(k): v for k, v in groupIDs.items()} + +try: + with open(f"{path}/typeIDs.yaml") as fp: + typeIDs = yaml.load(fp, Loader=yaml.CSafeLoader) +except FileNotFoundError: + with open(f"{path}/types.json") as fp: + typeIDs = json.load(fp) + typeIDs = {int(k): v for k, v in typeIDs.items()} + +ships = [] + +for id, entry in typeIDs.items(): + group = groupIDs[entry["groupID"]] + + if group["categoryID"] == 6 and entry["published"]: + ships.append( + { + "id": id, + "name": entry["name"]["en"] if "name" in entry else entry["typeNameID"], + "group": group["name"]["en"] if "name" in group else group["groupNameID"], + } + ) + +with open("dist/shiptypes.json", "w") as fp: + json.dump(ships, fp)