From 2a5d4021983d1eef15881376beb3b33c8eb5fbb6 Mon Sep 17 00:00:00 2001 From: Patric Stout Date: Sun, 12 Nov 2023 10:42:45 +0100 Subject: [PATCH] feat: initial version of the SDE data importer (#1) --- .gitignore | 5 ++ README.md | 29 ++++++++ convert.py | 173 ++++++++++++++++++++++++++++++++++++++++++++++ esf.proto | 105 ++++++++++++++++++++++++++++ requirements.base | 2 + requirements.txt | 2 + 6 files changed, 316 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 convert.py create mode 100644 esf.proto create mode 100644 requirements.base create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..60edfa7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/__pycache__/ +/.env/ +/dist/ +/sde/ +/esf_pb2.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..e233240 --- /dev/null +++ b/README.md @@ -0,0 +1,29 @@ +# Data used by EVEShip.fit + +To have the best experience possible, we convert the EVE SDE dataset into a format that is as small as possible and readable as fast as possible. + +For this we use Google's Protobuf, and we strip out a lot of fields we don't actually need. + +## Protobuf definition + +In this folder is a tool (`convert.py`), which converts the YAML files from the SDE into Protobuf (v2) binary files. + +In `esf.proto` is the Protobuf definition. +This is exported to Python and Javascript with the following commands: + +```bash +protoc --python_out=. esf.proto +web/node_modules/.bin/pbjs -t static-module -w es6 -o esf_pb2.js esf.proto --no-create --no-encode --no-verify --no-convert --no-delimited --no-typeurl --no-beautify --no-comments --no-service +``` + +## Converting + +Download the latest EVE SDE from [their website](https://developers.eveonline.com/resource/resources). + +Now run the tool: + +```bash +python convert.py +``` + +This will take a while to generate the protobuf files, but they will be outputed in the `dist` folder. diff --git a/convert.py b/convert.py new file mode 100644 index 0000000..8dee3c4 --- /dev/null +++ b/convert.py @@ -0,0 +1,173 @@ +import os +import sys +import yaml + +import esf_pb2 + +if len(sys.argv) < 2: + print("Usage: python3 convert.py ") + exit(1) + +path = sys.argv[1] + +os.makedirs("dist", exist_ok=True) + +def convert_type_dogma(path): + with open(f"{path}/typeDogma.yaml") as fp: + typeDogma = yaml.load(fp, Loader=yaml.CSafeLoader) + + pb2 = esf_pb2.TypeDogma() + + for id, entry in typeDogma.items(): + for attribute in entry["dogmaAttributes"]: + pbea = pb2.TypeDogmaEntry.DogmaAttributes( + attributeID=attribute["attributeID"], + value=attribute["value"] + ) + + pb2.entries[id].dogmaAttributes.append(pbea) + + for effect in entry["dogmaEffects"]: + pbee = pb2.TypeDogmaEntry.DogmaEffects( + effectID=effect["effectID"], + isDefault=effect["isDefault"] + ) + + pb2.entries[id].dogmaEffects.append(pbee) + + with open("dist/typeDogma.pb2", "wb") as fp: + fp.write(pb2.SerializeToString()) + + +def convert_type_ids(path): + with open(f"{path}/groupIDs.yaml") as fp: + groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader) + + with open(f"{path}/typeIDs.yaml") as fp: + typeIDs = yaml.load(fp, Loader=yaml.CSafeLoader) + + pb2 = esf_pb2.TypeIDs() + + for id, entry in typeIDs.items(): + pb2.entries[id].name = entry["name"]["en"] + pb2.entries[id].groupID = entry["groupID"] + pb2.entries[id].categoryID = groupIDs[entry["groupID"]]["categoryID"] + pb2.entries[id].published = entry["published"] + + if "marketGroupID" in entry: + pb2.entries[id].marketGroupID = entry["marketGroupID"] + if "capacity" in entry: + pb2.entries[id].capacity = entry["capacity"] + if "mass" in entry: + pb2.entries[id].mass = entry["mass"] + if "radius" in entry: + pb2.entries[id].radius = entry["radius"] + if "volume" in entry: + pb2.entries[id].volume = entry["volume"] + + with open("dist/typeIDs.pb2", "wb") as fp: + fp.write(pb2.SerializeToString()) + + +def convert_dogma_attributes(path): + with open(f"{path}/dogmaAttributes.yaml") as fp: + dogmaAttributes = yaml.load(fp, Loader=yaml.CSafeLoader) + + pb2 = esf_pb2.DogmaAttributes() + + for id, entry in dogmaAttributes.items(): + pb2.entries[id].name = entry["name"] + pb2.entries[id].published = entry["published"] + pb2.entries[id].defaultValue = entry["defaultValue"] + pb2.entries[id].highIsGood = entry["highIsGood"] + pb2.entries[id].stackable = entry["stackable"] + + # Entries that don't exist in the SDE, but are calculated by the library. + def add_esf_attribute(id, name): + pb2.entries[id].name = name + pb2.entries[id].published = True + pb2.entries[id].defaultValue = 0 + pb2.entries[id].highIsGood = True + pb2.entries[id].stackable = False + + add_esf_attribute(-1, "alignTime") + add_esf_attribute(-2, "scanStrength") + add_esf_attribute(-3, "cpuUsage") + add_esf_attribute(-4, "powerUsage") + + with open("dist/dogmaAttributes.pb2", "wb") as fp: + fp.write(pb2.SerializeToString()) + + +def convert_dogma_effects(path): + with open(f"{path}/dogmaEffects.yaml") as fp: + dogmaEffects = yaml.load(fp, Loader=yaml.CSafeLoader) + + pb2 = esf_pb2.DogmaEffects() + + for id, entry in dogmaEffects.items(): + pb2.entries[id].name = entry["effectName"] + pb2.entries[id].effectCategory = entry["effectCategory"] + pb2.entries[id].electronicChance = entry["electronicChance"] + pb2.entries[id].isAssistance = entry["isAssistance"] + pb2.entries[id].isOffensive = entry["isOffensive"] + pb2.entries[id].isWarpSafe = entry["isWarpSafe"] + pb2.entries[id].propulsionChance = entry["propulsionChance"] + pb2.entries[id].rangeChance = entry["rangeChance"] + + if "dischargeAttributeID" in entry: + pb2.entries[id].dischargeAttributeID = entry["dischargeAttributeID"] + if "durationAttributeID" in entry: + pb2.entries[id].durationAttributeID = entry["durationAttributeID"] + if "rangeAttributeID" in entry: + pb2.entries[id].rangeAttributeID = entry["rangeAttributeID"] + if "falloffAttributeID" in entry: + pb2.entries[id].falloffAttributeID = entry["falloffAttributeID"] + if "trackingSpeedAttributeID" in entry: + pb2.entries[id].trackingSpeedAttributeID = entry["trackingSpeedAttributeID"] + if "fittingUsageChanceAttributeID" in entry: + pb2.entries[id].fittingUsageChanceAttributeID = entry["fittingUsageChanceAttributeID"] + if "resistanceAttributeID" in entry: + pb2.entries[id].resistanceAttributeID = entry["resistanceAttributeID"] + + if "modifierInfo" in entry: + for modifier_info in entry["modifierInfo"]: + pbmi = pb2.DogmaEffect.ModifierInfo() + + match modifier_info["domain"]: + case "itemID": pbmi.domain = pbmi.Domain.itemID + case "shipID": pbmi.domain = pbmi.Domain.shipID + case "charID": pbmi.domain = pbmi.Domain.charID + case "otherID": pbmi.domain = pbmi.Domain.otherID + case "structureID": pbmi.domain = pbmi.Domain.structureID + case "target": pbmi.domain = pbmi.Domain.target + case "targetID": pbmi.domain = pbmi.Domain.targetID + + match modifier_info["func"]: + case "ItemModifier": pbmi.func = pbmi.Func.ItemModifier + case "LocationGroupModifier": pbmi.func = pbmi.Func.LocationGroupModifier + case "LocationModifier": pbmi.func = pbmi.Func.LocationModifier + case "LocationRequiredSkillModifier": pbmi.func = pbmi.Func.LocationRequiredSkillModifier + case "OwnerRequiredSkillModifier": pbmi.func = pbmi.Func.OwnerRequiredSkillModifier + case "EffectStopper": pbmi.func = pbmi.Func.EffectStopper + + if "modifiedAttributeID" in modifier_info: + pbmi.modifiedAttributeID = modifier_info["modifiedAttributeID"] + if "modifyingAttributeID" in modifier_info: + pbmi.modifyingAttributeID = modifier_info["modifyingAttributeID"] + if "operation" in modifier_info: + pbmi.operation = modifier_info["operation"] + if "groupID" in modifier_info: + pbmi.groupID = modifier_info["groupID"] + if "skillTypeID" in modifier_info: + pbmi.skillTypeID = modifier_info["skillTypeID"] + + pb2.entries[id].modifierInfo.append(pbmi) + + with open("dist/dogmaEffects.pb2", "wb") as fp: + fp.write(pb2.SerializeToString()) + +convert_type_dogma(path) +convert_type_ids(path) +convert_dogma_attributes(path) +convert_dogma_effects(path) diff --git a/esf.proto b/esf.proto new file mode 100644 index 0000000..b9657f0 --- /dev/null +++ b/esf.proto @@ -0,0 +1,105 @@ +syntax = "proto2"; + +package esf; + +message TypeDogma { + message TypeDogmaEntry { + message DogmaAttributes { + required int32 attributeID = 1; + required float value = 2; + } + + message DogmaEffects { + required int32 effectID = 1; + required bool isDefault = 2; + } + + repeated DogmaAttributes dogmaAttributes = 1; + repeated DogmaEffects dogmaEffects = 2; + } + + map entries = 1; +} + +message TypeIDs { + message TypeID { + required string name = 1; + required int32 groupID = 2; + required int32 categoryID = 3; + required bool published = 4; + + optional int32 marketGroupID = 5; + optional float capacity = 6; + optional float mass = 7; + optional float radius = 8; + optional float volume = 9; + } + + map entries = 1; +} + +message DogmaAttributes { + message DogmaAttribute { + required string name = 1; + required bool published = 2; + required float defaultValue = 3; + required bool highIsGood = 4; + required bool stackable = 5; + } + + map entries = 1; +} + +message DogmaEffects { + message DogmaEffect { + message ModifierInfo { + enum Domain { + itemID = 0; + shipID = 1; + charID = 2; + otherID = 3; + structureID = 4; + target = 5; + targetID = 6; + } + + enum Func { + ItemModifier = 0; + LocationGroupModifier = 1; + LocationModifier = 2; + LocationRequiredSkillModifier = 3; + OwnerRequiredSkillModifier = 4; + EffectStopper = 5; + } + + required Domain domain = 1; + required Func func = 2; + optional int32 modifiedAttributeID = 3; + optional int32 modifyingAttributeID = 4; + optional int32 operation = 5; + optional int32 groupID = 6; + optional int32 skillTypeID = 7; + } + + required string name = 1; + required int32 effectCategory = 2; + required bool electronicChance = 3; + required bool isAssistance = 4; + required bool isOffensive = 5; + required bool isWarpSafe = 6; + required bool propulsionChance = 7; + required bool rangeChance = 8; + + optional int32 dischargeAttributeID = 9; + optional int32 durationAttributeID = 10; + optional int32 rangeAttributeID = 11; + optional int32 falloffAttributeID = 12; + optional int32 trackingSpeedAttributeID = 13; + optional int32 fittingUsageChanceAttributeID = 14; + optional int32 resistanceAttributeID = 15; + + repeated ModifierInfo modifierInfo = 16; + } + + map entries = 1; +} diff --git a/requirements.base b/requirements.base new file mode 100644 index 0000000..3c6b7e1 --- /dev/null +++ b/requirements.base @@ -0,0 +1,2 @@ +pyyaml +protobuf diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d906e16 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +protobuf==3.12.4 +PyYAML==6.0.1