dave/eveship.fit.data
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: use EVE client data to construct the latest SDE (#48)

Browse Source
This commit is contained in:
Patric Stout
2024-05-17 19:58:04 +02:00
committed by GitHub
parent 141844e72b
commit e0d53bf81b
10 changed files with 429 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
.github/workflows/release.yml vendored
View File

@@ -6,10 +6,40 @@ on:
- published
jobs:
sde:
name: Fetch SDE
runs-on: windows-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Python2
run: |
choco install python2
C:\Python27\python.exe -m pip install --upgrade pip
C:\Python27\python.exe -m pip install requests
- name: Download loaders
run: |
C:\Python27\python.exe download_sde/download_loaders.py
- name: Execute loaders
run: |
C:\Python27\python.exe download_sde/execute_loaders.py
- name: Publish artifact
uses: actions/upload-artifact@v4
with:
name: sde
path: json
datafiles:
name: Publish datafiles
runs-on: ubuntu-latest
needs: [sde]
steps:
- name: Checkout
uses: actions/checkout@v4
@@ -51,33 +81,21 @@ jobs:
echo "application/x-protobuf pb2" | sudo tee -a /etc/mime.types
echo "80:application/x-protobuf:*.pb2" | sudo tee -a /usr/share/mime/globs2
- name: Fetch SDE
run: |
wget -q https://eve-static-data-export.s3-eu-west-1.amazonaws.com/tranquility/sde.zip
unzip sde.zip
- name: Validate SDE version
run: |
SDE_VERSION=$(date -r sde.zip "+%F" | sed 's/-//g')
RELEASE_SDE_VERSION=$(echo ${{ github.ref_name }} | cut -d. -f3)
echo "SDK version: ${SDE_VERSION}"
echo "Release version: ${RELEASE_SDE_VERSION}"
if [ "${SDE_VERSION}" != "${RELEASE_SDE_VERSION}" ]; then
echo "SDE version mismatch: ${SDE_VERSION} != ${RELEASE_SDE_VERSION}"
exit 1
fi
- name: Download artifact
uses: actions/download-artifact@v4
with:
name: sde
path: sde
- name: Convert SDE YAML to Protobuf
run: |
protoc --python_out=. esf.proto
python convert.py sde/fsd
python list_shiptypes.py sde/fsd
python -m convert sde
python -m list_shiptypes sde
- name: Fetch icons
run: |
python download_icons.py sde/fsd
python -m download_icons sde
- name: Build package
run: |

86
.github/workflows/testing.yml vendored
View File

@@ -26,7 +26,89 @@ jobs:
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Install dependencies
run: npm install
run: |
npm install
pip install black flake8
- uses: TrueBrain/actions-flake8@v2
with:
path: convert download_icons download_sde list_shiptypes
max_line_length: 120
- name: Run linter
run: npm run lint
run: |
npm run lint
black --check -l 120 convert download_icons download_sde list_shiptypes
sde:
name: Fetch SDE
runs-on: windows-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Python2
run: |
choco install python2
C:\Python27\python.exe -m pip install --upgrade pip
C:\Python27\python.exe -m pip install requests
- name: Download loaders
run: |
C:\Python27\python.exe download_sde/download_loaders.py
- name: Execute loaders
run: |
C:\Python27\python.exe download_sde/execute_loaders.py
- name: Publish artifact
uses: actions/upload-artifact@v4
with:
name: sde
path: json
datafiles:
name: Generate datafiles
runs-on: ubuntu-latest
needs: [sde]
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install NodeJS
uses: actions/setup-node@v4
with:
registry-url: https://npm.pkg.github.com
scope: "@eveshipfit"
env:
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y --no-install-recommends protobuf-compiler
pip install -r requirements.txt
npm install
- name: Download artifact
uses: actions/download-artifact@v4
with:
name: sde
path: sde
- name: Convert SDE YAML to Protobuf
run: |
protoc --python_out=. esf.proto
python -m convert sde
python -m list_shiptypes sde
- name: Fetch icons
run: |
python -m download_icons sde
- name: Build package
run: |
npm run build

5
.gitignore vendored
View File

@@ -1,8 +1,11 @@
/__pycache__/
__pycache__/
/.env/
/data/
/dist/
/esf_pb2.js
/esf_pb2.py
/json/
/node_modules/
/package-lock.json
/pyd/
/sde/

22
README.md
View File

@@ -23,11 +23,31 @@ Download the latest EVE SDE from [their website](https://developers.eveonline.co
Now run the tool:
```bash
python convert.py <path to fsd folder inside the sde>
python -m convert <path to fsd folder inside the sde>
```
This will take a while to generate the protobuf files, but they will be outputed in the `dist` folder.
### SDE based on EVE client information
The convert script also supports loading the information from data taken from the latest EVE client.
In general, this is more up-to-date than the published SDE.
The downside is, that it requires Python2 on Windows in order to extract.
How it works:
- It downloads several `.pyd` files (which are actually DLLs, so this only works on Windows) from the installer.
- It downloads `.fsdbinary` files from the installer.
- It loads the `.pyd` files in a Python2 context. These files contain information how to load the `.fsdbinary` files.
- It exports the result as `.json` files in the `json/` folder.
Use the `download_sde/download_loaders.py` to download the files, and `download_sde/execute_loaders.py` to convert them to JSON.
Now convert them with:
```bash
python -m convert <path to json>
```
## Patches
The EVE SDE has some quirks, that are easiest fixed in the conversion.

105
convert.py → convert/__main__.py
View File

@@ -1,9 +1,12 @@
import json
import os
import sys
import yaml
import esf_pb2
from google.protobuf.json_format import MessageToJson
if len(sys.argv) < 2:
print("Usage: python3 convert.py <path/to/eve-sde/fsd>")
exit(1)
@@ -11,13 +14,19 @@ if len(sys.argv) < 2:
path = sys.argv[1]
os.makedirs("dist/sde", exist_ok=True)
os.makedirs("dist/sde_json", exist_ok=True)
def convert_type_dogma(path, ships):
print("Converting typeDogma ...")
with open(f"{path}/typeDogma.yaml") as fp:
typeDogma = yaml.load(fp, Loader=yaml.CSafeLoader)
try:
with open(f"{path}/typeDogma.yaml") as fp:
typeDogma = yaml.load(fp, Loader=yaml.CSafeLoader)
except FileNotFoundError:
with open(f"{path}/typedogma.json") as fp:
typeDogma = json.load(fp)
typeDogma = {int(k): v for k, v in typeDogma.items()}
pb2 = esf_pb2.TypeDogma()
@@ -45,21 +54,34 @@ def convert_type_dogma(path, ships):
with open("dist/sde/typeDogma.pb2", "wb") as fp:
fp.write(pb2.SerializeToString())
with open("dist/sde_json/typeDogma.json", "w") as fp:
fp.write(MessageToJson(pb2, sort_keys=True))
def convert_type_ids(path):
print("Converting typeIDs ...")
with open(f"{path}/groupIDs.yaml") as fp:
groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
try:
with open(f"{path}/groupIDs.yaml") as fp:
groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
except FileNotFoundError:
with open(f"{path}/groups.json") as fp:
groupIDs = json.load(fp)
groupIDs = {int(k): v for k, v in groupIDs.items()}
with open(f"{path}/typeIDs.yaml") as fp:
typeIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
try:
with open(f"{path}/typeIDs.yaml") as fp:
typeIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
except FileNotFoundError:
with open(f"{path}/types.json") as fp:
typeIDs = json.load(fp)
typeIDs = {int(k): v for k, v in typeIDs.items()}
pb2 = esf_pb2.TypeIDs()
ships = []
for id, entry in typeIDs.items():
pb2.entries[id].name = entry["name"]["en"]
pb2.entries[id].name = entry["name"]["en"] if "name" in entry else entry["typeNameID"]
pb2.entries[id].groupID = entry["groupID"]
pb2.entries[id].categoryID = groupIDs[entry["groupID"]]["categoryID"]
pb2.entries[id].published = entry["published"]
@@ -73,48 +95,64 @@ def convert_type_ids(path):
pb2.entries[id].marketGroupID = entry["marketGroupID"]
if "metaGroupID" in entry:
pb2.entries[id].metaGroupID = entry["metaGroupID"]
if "capacity" in entry:
if "capacity" in entry and entry["capacity"] != 0.0:
pb2.entries[id].capacity = entry["capacity"]
if "mass" in entry:
if "mass" in entry and entry["mass"] != 0.0:
pb2.entries[id].mass = entry["mass"]
if "radius" in entry:
if "radius" in entry and entry["radius"] != 1.0:
pb2.entries[id].radius = entry["radius"]
if "volume" in entry:
if "volume" in entry and entry["volume"] != 0.0:
pb2.entries[id].volume = entry["volume"]
with open("dist/sde/typeIDs.pb2", "wb") as fp:
fp.write(pb2.SerializeToString())
with open("dist/sde_json/typeIDs.json", "w") as fp:
fp.write(MessageToJson(pb2, sort_keys=True))
return ships
def convert_group_ids(path):
print("Converting groupIDs ...")
with open(f"{path}/groupIDs.yaml") as fp:
groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
try:
with open(f"{path}/groupIDs.yaml") as fp:
groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
except FileNotFoundError:
with open(f"{path}/groups.json") as fp:
groupIDs = json.load(fp)
groupIDs = {int(k): v for k, v in groupIDs.items()}
pb2 = esf_pb2.GroupIDs()
for id, entry in groupIDs.items():
pb2.entries[id].name = entry["name"]["en"]
pb2.entries[id].name = entry["name"]["en"] if "name" in entry else entry["groupNameID"]
pb2.entries[id].categoryID = entry["categoryID"]
pb2.entries[id].published = entry["published"]
with open("dist/sde/groupIDs.pb2", "wb") as fp:
fp.write(pb2.SerializeToString())
with open("dist/sde_json/groupIDs.json", "w") as fp:
fp.write(MessageToJson(pb2, sort_keys=True))
def convert_market_groups(path):
print("Converting marketGroups ...")
with open(f"{path}/marketGroups.yaml") as fp:
marketGroupIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
try:
with open(f"{path}/marketGroups.yaml") as fp:
marketGroupIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
except FileNotFoundError:
with open(f"{path}/marketgroups.json") as fp:
marketGroupIDs = json.load(fp)
marketGroupIDs = {int(k): v for k, v in marketGroupIDs.items()}
pb2 = esf_pb2.MarketGroups()
for id, entry in marketGroupIDs.items():
pb2.entries[id].name = entry["nameID"]["en"]
pb2.entries[id].name = entry["nameID"] if isinstance(entry["nameID"], str) else entry["nameID"]["en"]
if "parentGroupID" in entry:
pb2.entries[id].parentGroupID = entry["parentGroupID"]
@@ -124,12 +162,20 @@ def convert_market_groups(path):
with open("dist/sde/marketGroups.pb2", "wb") as fp:
fp.write(pb2.SerializeToString())
with open("dist/sde_json/marketGroups.json", "w") as fp:
fp.write(MessageToJson(pb2, sort_keys=True))
def convert_dogma_attributes(path):
print("Converting dogmaAttributes ...")
with open(f"{path}/dogmaAttributes.yaml") as fp:
dogmaAttributes = yaml.load(fp, Loader=yaml.CSafeLoader)
try:
with open(f"{path}/dogmaAttributes.yaml") as fp:
dogmaAttributes = yaml.load(fp, Loader=yaml.CSafeLoader)
except FileNotFoundError:
with open(f"{path}/dogmaattributes.json") as fp:
dogmaAttributes = json.load(fp)
dogmaAttributes = {int(k): v for k, v in dogmaAttributes.items()}
pb2 = esf_pb2.DogmaAttributes()
@@ -187,12 +233,20 @@ def convert_dogma_attributes(path):
with open("dist/sde/dogmaAttributes.pb2", "wb") as fp:
fp.write(pb2.SerializeToString())
with open("dist/sde_json/dogmaAttributes.json", "w") as fp:
fp.write(MessageToJson(pb2, sort_keys=True))
def convert_dogma_effects(path):
print("Converting dogmaEffects ...")
with open(f"{path}/dogmaEffects.yaml") as fp:
dogmaEffects = yaml.load(fp, Loader=yaml.CSafeLoader)
try:
with open(f"{path}/dogmaEffects.yaml") as fp:
dogmaEffects = yaml.load(fp, Loader=yaml.CSafeLoader)
except FileNotFoundError:
with open(f"{path}/dogmaeffects.json") as fp:
dogmaEffects = json.load(fp)
dogmaEffects = {int(k): v for k, v in dogmaEffects.items()}
pb2 = esf_pb2.DogmaEffects()
pbmi = pb2.DogmaEffect.ModifierInfo()
@@ -337,7 +391,8 @@ def convert_dogma_effects(path):
if entry["effectName"] == "moduleBonusAfterburner" or entry["effectName"] == "moduleBonusMicrowarpdrive":
add_modifier(id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, 4, 2, 796) # mass <modAdd> massAddition
# Velocity change is calculated like this: velocityBoost = item.speedFactor * item.speedBoostFactor / ship.mass
# Velocity change is calculated like this:
# velocityBoost = item.speedFactor * item.speedBoostFactor / ship.mass
# First, calculate the multiplication on the item.
add_modifier(
id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, -7, -1, 567
@@ -346,7 +401,8 @@ def convert_dogma_effects(path):
id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, -7, 4, 20
) # velocityBoost <postMul> speedFactor
# Next, "applyVelocityBoost" is applied on all ships which takes care of the final calculation (as mass is an attribute of the ship).
# Next, "applyVelocityBoost" is applied on all ships which takes care of
# the final calculation (as mass is an attribute of the ship).
# missileEMDmgBonus, missileExplosiveDmgBonus, missileKineticDmgBonus, missileThermalDmgBonus don't apply
# any effect direct, but this is handled internally in EVE. For us,
@@ -379,6 +435,9 @@ def convert_dogma_effects(path):
with open("dist/sde/dogmaEffects.pb2", "wb") as fp:
fp.write(pb2.SerializeToString())
with open("dist/sde_json/dogmaEffects.json", "w") as fp:
fp.write(MessageToJson(pb2, sort_keys=True))
convert_group_ids(path)
convert_market_groups(path)

28
download_icons.py → download_icons/__main__.py
View File

@@ -1,3 +1,4 @@
import json
import os
import requests
import sys
@@ -17,14 +18,29 @@ folders = [
]
files = {}
with open(f"{path}/marketGroups.yaml") as fp:
marketGroups = yaml.load(fp, Loader=yaml.CSafeLoader)
try:
with open(f"{path}/marketGroups.yaml") as fp:
marketGroups = yaml.load(fp, Loader=yaml.CSafeLoader)
except FileNotFoundError:
with open(f"{path}/marketgroups.json") as fp:
marketGroups = json.load(fp)
marketGroups = {int(k): v for k, v in marketGroups.items()}
with open(f"{path}/metaGroups.yaml") as fp:
metaGroups = yaml.load(fp, Loader=yaml.CSafeLoader)
try:
with open(f"{path}/metaGroups.yaml") as fp:
metaGroups = yaml.load(fp, Loader=yaml.CSafeLoader)
except FileNotFoundError:
with open(f"{path}/metagroups.json") as fp:
metaGroups = json.load(fp)
metaGroups = {int(k): v for k, v in metaGroups.items()}
with open(f"{path}/iconIDs.yaml") as fp:
iconIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
try:
with open(f"{path}/iconIDs.yaml") as fp:
iconIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
except FileNotFoundError:
with open(f"{path}/iconids.json") as fp:
iconIDs = json.load(fp)
iconIDs = {int(k): v for k, v in iconIDs.items()}
for marketGroupID, marketGroup in marketGroups.items():
if "iconID" not in marketGroup or marketGroup["iconID"] == 0:

73
download_sde/download_loaders.py Normal file
View File

@@ -0,0 +1,73 @@
import os
import requests
# Only download these loaders and their data.
LOADER_LIST = [
"categories",
"dogmaattributes",
"dogmaeffects",
"iconids",
"groups",
"marketgroups",
"metagroups",
"typedogma",
"types",
]
os.makedirs("pyd")
os.makedirs("data")
session = requests.Session()
# Find the latest installer listing.
latest = session.get("https://binaries.eveonline.com/eveclient_TQ.json").json()
build = latest["build"]
installer = session.get("https://binaries.eveonline.com/eveonline_" + build + ".txt").text
# Download all the loaders.
resfileindex = None
for line in installer.split("\n"):
if not line:
continue
res, path, _, _, _, _ = line.split(",")
if res == "app:/resfileindex.txt":
resfileindex = line.split(",")[1]
if not res.startswith("app:/bin64") or not res.endswith("Loader.pyd"):
continue
loader = res.split("/")[-1][:-10].lower()
if loader not in LOADER_LIST:
continue
local_path = "pyd/" + res.split("/")[-1]
print("Downloading " + local_path + " ...")
with open(local_path, "wb") as f:
f.write(session.get("https://binaries.eveonline.com/" + path).content)
if resfileindex is None:
raise Exception("resfileindex not found")
# Download all the fsdbinary files.
resfile = requests.get("https://binaries.eveonline.com/" + resfileindex).text
for line in resfile.split("\n"):
if not line:
continue
res, path, _, _, _ = line.split(",")
if (
not res.startswith("res:/staticdata/") or not res.endswith(".fsdbinary")
) and res != "res:/localizationfsd/localization_fsd_en-us.pickle":
continue
loader = res.split("/")[-1][:-10]
if res != "res:/localizationfsd/localization_fsd_en-us.pickle" and loader not in LOADER_LIST:
continue
local_path = "data/" + res.split("/")[-1]
print("Downloading " + local_path + " ...")
with open(local_path, "wb") as f:
f.write(session.get("https://resources.eveonline.com/" + path).content)

60
download_sde/execute_loaders.py Normal file
View File

@@ -0,0 +1,60 @@
import pickle
import glob
import importlib
import json
import os
import sys
sys.path.append("pyd")
os.makedirs("json")
def decode_cfsd(key, data, strings):
data_type = type(data)
if data_type.__module__ == "cfsd" and data_type.__name__ == "dict":
return {k: decode_cfsd(k, v, strings) for k, v in data.items()}
if data_type.__module__.endswith("Loader"):
return {x: decode_cfsd(x, getattr(data, x), strings) for x in dir(data) if not x.startswith("__")}
if data_type.__module__ == "cfsd" and data_type.__name__ == "list":
return [decode_cfsd(None, v, strings) for v in data]
if isinstance(data, tuple):
return tuple([decode_cfsd(None, v, strings) for v in data])
if data_type.__name__.endswith("_vector"):
# TODO
return None
if isinstance(data, int) or data_type.__name__ == "long":
# In case it is a NameID, look up the name.
if key is not None and isinstance(key, str) and key.lower().endswith("nameid") and key != "dungeonNameID":
return strings[data][0]
return data
if isinstance(data, float):
return data
if isinstance(data, str):
return data
raise ValueError("Unknown type: " + str(type(data)))
# Load all the english strings.
print("Loading 'localization_fsd_en-us.pickle' ...")
with open("data/localization_fsd_en-us.pickle", "rb") as f:
strings = pickle.load(f)[1]
# Convert all available fsdbinary files via their Loader to JSON.
for loader in glob.glob("pyd/*Loader.pyd"):
loader_name = os.path.splitext(os.path.basename(loader))[0]
data_name = loader_name.replace("Loader", "").lower() + ".fsdbinary"
print("Loading '" + data_name + "' with '" + loader_name + "' ...")
lib = importlib.import_module(loader_name)
data = lib.load("data/" + data_name)
data = decode_cfsd(None, data, strings)
with open("json/" + data_name.replace(".fsdbinary", ".json"), "w") as f:
json.dump(data, f, indent=4)

35
list_shiptypes.py
View File

@@ -1,35 +0,0 @@
import json
import os
import sys
import yaml
if len(sys.argv) < 2:
print("Usage: python3 convert.py <path/to/eve-sde/fsd>")
exit(1)
path = sys.argv[1]
os.makedirs("dist", exist_ok=True)
with open(f"{path}/groupIDs.yaml") as fp:
groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
with open(f"{path}/typeIDs.yaml") as fp:
typeIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
ships = []
for id, entry in typeIDs.items():
group = groupIDs[entry["groupID"]]
if group["categoryID"] == 6 and entry["published"]:
ships.append(
{
"id": id,
"name": entry["name"]["en"],
"group": group["name"]["en"],
}
)
with open("dist/shiptypes.json", "w") as fp:
json.dump(ships, fp)

45
list_shiptypes/__main__.py Normal file
View File

@@ -0,0 +1,45 @@
import json
import os
import sys
import yaml
if len(sys.argv) < 2:
print("Usage: python3 convert.py <path/to/eve-sde/fsd>")
exit(1)
path = sys.argv[1]
os.makedirs("dist", exist_ok=True)
try:
with open(f"{path}/groupIDs.yaml") as fp:
groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
except FileNotFoundError:
with open(f"{path}/groups.json") as fp:
groupIDs = json.load(fp)
groupIDs = {int(k): v for k, v in groupIDs.items()}
try:
with open(f"{path}/typeIDs.yaml") as fp:
typeIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
except FileNotFoundError:
with open(f"{path}/types.json") as fp:
typeIDs = json.load(fp)
typeIDs = {int(k): v for k, v in typeIDs.items()}
ships = []
for id, entry in typeIDs.items():
group = groupIDs[entry["groupID"]]
if group["categoryID"] == 6 and entry["published"]:
ships.append(
{
"id": id,
"name": entry["name"]["en"] if "name" in entry else entry["typeNameID"],
"group": group["name"]["en"] if "name" in group else group["groupNameID"],
}
)
with open("dist/shiptypes.json", "w") as fp:
json.dump(ships, fp)