feat: initial version of the SDE data importer (#1)
This commit is contained in:
Patric Stout
5
.gitignore
vendored
Normal file
5
.gitignore
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
/__pycache__/
|
||||
/.env/
|
||||
/dist/
|
||||
/sde/
|
||||
/esf_pb2.py
|
||||
29
README.md
Normal file
29
README.md
Normal file
@@ -0,0 +1,29 @@
|
||||
# Data used by EVEShip.fit
|
||||
|
||||
To have the best experience possible, we convert the EVE SDE dataset into a format that is as small as possible and readable as fast as possible.
|
||||
|
||||
For this we use Google's Protobuf, and we strip out a lot of fields we don't actually need.
|
||||
|
||||
## Protobuf definition
|
||||
|
||||
In this folder is a tool (`convert.py`), which converts the YAML files from the SDE into Protobuf (v2) binary files.
|
||||
|
||||
In `esf.proto` is the Protobuf definition.
|
||||
This is exported to Python and Javascript with the following commands:
|
||||
|
||||
```bash
|
||||
protoc --python_out=. esf.proto
|
||||
web/node_modules/.bin/pbjs -t static-module -w es6 -o esf_pb2.js esf.proto --no-create --no-encode --no-verify --no-convert --no-delimited --no-typeurl --no-beautify --no-comments --no-service
|
||||
```
|
||||
|
||||
## Converting
|
||||
|
||||
Download the latest EVE SDE from [their website](https://developers.eveonline.com/resource/resources).
|
||||
|
||||
Now run the tool:
|
||||
|
||||
```bash
|
||||
python convert.py <path to fsd folder inside the sde>
|
||||
```
|
||||
|
||||
This will take a while to generate the protobuf files, but they will be outputed in the `dist` folder.
|
||||
173
convert.py
Normal file
173
convert.py
Normal file
@@ -0,0 +1,173 @@
|
||||
import os
|
||||
import sys
|
||||
import yaml
|
||||
|
||||
import esf_pb2
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python3 convert.py <path/to/eve-sde/fsd>")
|
||||
exit(1)
|
||||
|
||||
path = sys.argv[1]
|
||||
|
||||
os.makedirs("dist", exist_ok=True)
|
||||
|
||||
def convert_type_dogma(path):
|
||||
with open(f"{path}/typeDogma.yaml") as fp:
|
||||
typeDogma = yaml.load(fp, Loader=yaml.CSafeLoader)
|
||||
|
||||
pb2 = esf_pb2.TypeDogma()
|
||||
|
||||
for id, entry in typeDogma.items():
|
||||
for attribute in entry["dogmaAttributes"]:
|
||||
pbea = pb2.TypeDogmaEntry.DogmaAttributes(
|
||||
attributeID=attribute["attributeID"],
|
||||
value=attribute["value"]
|
||||
)
|
||||
|
||||
pb2.entries[id].dogmaAttributes.append(pbea)
|
||||
|
||||
for effect in entry["dogmaEffects"]:
|
||||
pbee = pb2.TypeDogmaEntry.DogmaEffects(
|
||||
effectID=effect["effectID"],
|
||||
isDefault=effect["isDefault"]
|
||||
)
|
||||
|
||||
pb2.entries[id].dogmaEffects.append(pbee)
|
||||
|
||||
with open("dist/typeDogma.pb2", "wb") as fp:
|
||||
fp.write(pb2.SerializeToString())
|
||||
|
||||
|
||||
def convert_type_ids(path):
|
||||
with open(f"{path}/groupIDs.yaml") as fp:
|
||||
groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
|
||||
|
||||
with open(f"{path}/typeIDs.yaml") as fp:
|
||||
typeIDs = yaml.load(fp, Loader=yaml.CSafeLoader)
|
||||
|
||||
pb2 = esf_pb2.TypeIDs()
|
||||
|
||||
for id, entry in typeIDs.items():
|
||||
pb2.entries[id].name = entry["name"]["en"]
|
||||
pb2.entries[id].groupID = entry["groupID"]
|
||||
pb2.entries[id].categoryID = groupIDs[entry["groupID"]]["categoryID"]
|
||||
pb2.entries[id].published = entry["published"]
|
||||
|
||||
if "marketGroupID" in entry:
|
||||
pb2.entries[id].marketGroupID = entry["marketGroupID"]
|
||||
if "capacity" in entry:
|
||||
pb2.entries[id].capacity = entry["capacity"]
|
||||
if "mass" in entry:
|
||||
pb2.entries[id].mass = entry["mass"]
|
||||
if "radius" in entry:
|
||||
pb2.entries[id].radius = entry["radius"]
|
||||
if "volume" in entry:
|
||||
pb2.entries[id].volume = entry["volume"]
|
||||
|
||||
with open("dist/typeIDs.pb2", "wb") as fp:
|
||||
fp.write(pb2.SerializeToString())
|
||||
|
||||
|
||||
def convert_dogma_attributes(path):
|
||||
with open(f"{path}/dogmaAttributes.yaml") as fp:
|
||||
dogmaAttributes = yaml.load(fp, Loader=yaml.CSafeLoader)
|
||||
|
||||
pb2 = esf_pb2.DogmaAttributes()
|
||||
|
||||
for id, entry in dogmaAttributes.items():
|
||||
pb2.entries[id].name = entry["name"]
|
||||
pb2.entries[id].published = entry["published"]
|
||||
pb2.entries[id].defaultValue = entry["defaultValue"]
|
||||
pb2.entries[id].highIsGood = entry["highIsGood"]
|
||||
pb2.entries[id].stackable = entry["stackable"]
|
||||
|
||||
# Entries that don't exist in the SDE, but are calculated by the library.
|
||||
def add_esf_attribute(id, name):
|
||||
pb2.entries[id].name = name
|
||||
pb2.entries[id].published = True
|
||||
pb2.entries[id].defaultValue = 0
|
||||
pb2.entries[id].highIsGood = True
|
||||
pb2.entries[id].stackable = False
|
||||
|
||||
add_esf_attribute(-1, "alignTime")
|
||||
add_esf_attribute(-2, "scanStrength")
|
||||
add_esf_attribute(-3, "cpuUsage")
|
||||
add_esf_attribute(-4, "powerUsage")
|
||||
|
||||
with open("dist/dogmaAttributes.pb2", "wb") as fp:
|
||||
fp.write(pb2.SerializeToString())
|
||||
|
||||
|
||||
def convert_dogma_effects(path):
|
||||
with open(f"{path}/dogmaEffects.yaml") as fp:
|
||||
dogmaEffects = yaml.load(fp, Loader=yaml.CSafeLoader)
|
||||
|
||||
pb2 = esf_pb2.DogmaEffects()
|
||||
|
||||
for id, entry in dogmaEffects.items():
|
||||
pb2.entries[id].name = entry["effectName"]
|
||||
pb2.entries[id].effectCategory = entry["effectCategory"]
|
||||
pb2.entries[id].electronicChance = entry["electronicChance"]
|
||||
pb2.entries[id].isAssistance = entry["isAssistance"]
|
||||
pb2.entries[id].isOffensive = entry["isOffensive"]
|
||||
pb2.entries[id].isWarpSafe = entry["isWarpSafe"]
|
||||
pb2.entries[id].propulsionChance = entry["propulsionChance"]
|
||||
pb2.entries[id].rangeChance = entry["rangeChance"]
|
||||
|
||||
if "dischargeAttributeID" in entry:
|
||||
pb2.entries[id].dischargeAttributeID = entry["dischargeAttributeID"]
|
||||
if "durationAttributeID" in entry:
|
||||
pb2.entries[id].durationAttributeID = entry["durationAttributeID"]
|
||||
if "rangeAttributeID" in entry:
|
||||
pb2.entries[id].rangeAttributeID = entry["rangeAttributeID"]
|
||||
if "falloffAttributeID" in entry:
|
||||
pb2.entries[id].falloffAttributeID = entry["falloffAttributeID"]
|
||||
if "trackingSpeedAttributeID" in entry:
|
||||
pb2.entries[id].trackingSpeedAttributeID = entry["trackingSpeedAttributeID"]
|
||||
if "fittingUsageChanceAttributeID" in entry:
|
||||
pb2.entries[id].fittingUsageChanceAttributeID = entry["fittingUsageChanceAttributeID"]
|
||||
if "resistanceAttributeID" in entry:
|
||||
pb2.entries[id].resistanceAttributeID = entry["resistanceAttributeID"]
|
||||
|
||||
if "modifierInfo" in entry:
|
||||
for modifier_info in entry["modifierInfo"]:
|
||||
pbmi = pb2.DogmaEffect.ModifierInfo()
|
||||
|
||||
match modifier_info["domain"]:
|
||||
case "itemID": pbmi.domain = pbmi.Domain.itemID
|
||||
case "shipID": pbmi.domain = pbmi.Domain.shipID
|
||||
case "charID": pbmi.domain = pbmi.Domain.charID
|
||||
case "otherID": pbmi.domain = pbmi.Domain.otherID
|
||||
case "structureID": pbmi.domain = pbmi.Domain.structureID
|
||||
case "target": pbmi.domain = pbmi.Domain.target
|
||||
case "targetID": pbmi.domain = pbmi.Domain.targetID
|
||||
|
||||
match modifier_info["func"]:
|
||||
case "ItemModifier": pbmi.func = pbmi.Func.ItemModifier
|
||||
case "LocationGroupModifier": pbmi.func = pbmi.Func.LocationGroupModifier
|
||||
case "LocationModifier": pbmi.func = pbmi.Func.LocationModifier
|
||||
case "LocationRequiredSkillModifier": pbmi.func = pbmi.Func.LocationRequiredSkillModifier
|
||||
case "OwnerRequiredSkillModifier": pbmi.func = pbmi.Func.OwnerRequiredSkillModifier
|
||||
case "EffectStopper": pbmi.func = pbmi.Func.EffectStopper
|
||||
|
||||
if "modifiedAttributeID" in modifier_info:
|
||||
pbmi.modifiedAttributeID = modifier_info["modifiedAttributeID"]
|
||||
if "modifyingAttributeID" in modifier_info:
|
||||
pbmi.modifyingAttributeID = modifier_info["modifyingAttributeID"]
|
||||
if "operation" in modifier_info:
|
||||
pbmi.operation = modifier_info["operation"]
|
||||
if "groupID" in modifier_info:
|
||||
pbmi.groupID = modifier_info["groupID"]
|
||||
if "skillTypeID" in modifier_info:
|
||||
pbmi.skillTypeID = modifier_info["skillTypeID"]
|
||||
|
||||
pb2.entries[id].modifierInfo.append(pbmi)
|
||||
|
||||
with open("dist/dogmaEffects.pb2", "wb") as fp:
|
||||
fp.write(pb2.SerializeToString())
|
||||
|
||||
convert_type_dogma(path)
|
||||
convert_type_ids(path)
|
||||
convert_dogma_attributes(path)
|
||||
convert_dogma_effects(path)
|
||||
105
esf.proto
Normal file
105
esf.proto
Normal file
@@ -0,0 +1,105 @@
|
||||
syntax = "proto2";
|
||||
|
||||
package esf;
|
||||
|
||||
message TypeDogma {
|
||||
message TypeDogmaEntry {
|
||||
message DogmaAttributes {
|
||||
required int32 attributeID = 1;
|
||||
required float value = 2;
|
||||
}
|
||||
|
||||
message DogmaEffects {
|
||||
required int32 effectID = 1;
|
||||
required bool isDefault = 2;
|
||||
}
|
||||
|
||||
repeated DogmaAttributes dogmaAttributes = 1;
|
||||
repeated DogmaEffects dogmaEffects = 2;
|
||||
}
|
||||
|
||||
map<int32, TypeDogmaEntry> entries = 1;
|
||||
}
|
||||
|
||||
message TypeIDs {
|
||||
message TypeID {
|
||||
required string name = 1;
|
||||
required int32 groupID = 2;
|
||||
required int32 categoryID = 3;
|
||||
required bool published = 4;
|
||||
|
||||
optional int32 marketGroupID = 5;
|
||||
optional float capacity = 6;
|
||||
optional float mass = 7;
|
||||
optional float radius = 8;
|
||||
optional float volume = 9;
|
||||
}
|
||||
|
||||
map<int32, TypeID> entries = 1;
|
||||
}
|
||||
|
||||
message DogmaAttributes {
|
||||
message DogmaAttribute {
|
||||
required string name = 1;
|
||||
required bool published = 2;
|
||||
required float defaultValue = 3;
|
||||
required bool highIsGood = 4;
|
||||
required bool stackable = 5;
|
||||
}
|
||||
|
||||
map<int32, DogmaAttribute> entries = 1;
|
||||
}
|
||||
|
||||
message DogmaEffects {
|
||||
message DogmaEffect {
|
||||
message ModifierInfo {
|
||||
enum Domain {
|
||||
itemID = 0;
|
||||
shipID = 1;
|
||||
charID = 2;
|
||||
otherID = 3;
|
||||
structureID = 4;
|
||||
target = 5;
|
||||
targetID = 6;
|
||||
}
|
||||
|
||||
enum Func {
|
||||
ItemModifier = 0;
|
||||
LocationGroupModifier = 1;
|
||||
LocationModifier = 2;
|
||||
LocationRequiredSkillModifier = 3;
|
||||
OwnerRequiredSkillModifier = 4;
|
||||
EffectStopper = 5;
|
||||
}
|
||||
|
||||
required Domain domain = 1;
|
||||
required Func func = 2;
|
||||
optional int32 modifiedAttributeID = 3;
|
||||
optional int32 modifyingAttributeID = 4;
|
||||
optional int32 operation = 5;
|
||||
optional int32 groupID = 6;
|
||||
optional int32 skillTypeID = 7;
|
||||
}
|
||||
|
||||
required string name = 1;
|
||||
required int32 effectCategory = 2;
|
||||
required bool electronicChance = 3;
|
||||
required bool isAssistance = 4;
|
||||
required bool isOffensive = 5;
|
||||
required bool isWarpSafe = 6;
|
||||
required bool propulsionChance = 7;
|
||||
required bool rangeChance = 8;
|
||||
|
||||
optional int32 dischargeAttributeID = 9;
|
||||
optional int32 durationAttributeID = 10;
|
||||
optional int32 rangeAttributeID = 11;
|
||||
optional int32 falloffAttributeID = 12;
|
||||
optional int32 trackingSpeedAttributeID = 13;
|
||||
optional int32 fittingUsageChanceAttributeID = 14;
|
||||
optional int32 resistanceAttributeID = 15;
|
||||
|
||||
repeated ModifierInfo modifierInfo = 16;
|
||||
}
|
||||
|
||||
map<int32, DogmaEffect> entries = 1;
|
||||
}
|
||||
2
requirements.base
Normal file
2
requirements.base
Normal file
@@ -0,0 +1,2 @@
|
||||
pyyaml
|
||||
protobuf
|
||||
2
requirements.txt
Normal file
2
requirements.txt
Normal file
@@ -0,0 +1,2 @@
|
||||
protobuf==3.12.4
|
||||
PyYAML==6.0.1
|
||||
Reference in New Issue
Block a user