From ec6c67fcf41d6865676062c911f007502cde9d9f Mon Sep 17 00:00:00 2001 From: Patric Stout Date: Mon, 20 Nov 2023 20:53:02 +0100 Subject: [PATCH] feat: automate releasing of new data-files (#11) --- .github/workflows/release.yml | 66 +++++++++++++++++++++++++ convert.py | 90 +++++++++++++++++++++-------------- download_icons.py | 29 +++++++++++ requirements.base | 1 + requirements.txt | 6 +++ 5 files changed, 155 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 download_icons.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..6dc9892 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,66 @@ +name: Release + +on: + release: + types: + - published + +jobs: + datafiles: + name: Publish datafiles + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y --no-install-recommends rclone protobuf-compiler + pip install -r requirements.txt + + - name: Setup rclone + run: | + mkdir -p ~/.config/rclone + + echo "[eveshipfit]" > ~/.config/rclone/rclone.conf + echo "type = s3" >> ~/.config/rclone/rclone.conf + echo "provider = Cloudflare" >> ~/.config/rclone/rclone.conf + echo "acl = private" >> ~/.config/rclone/rclone.conf + echo "no_check_bucket = true" >> ~/.config/rclone/rclone.conf + + - name: Fetch SDE + run: | + wget -q https://eve-static-data-export.s3-eu-west-1.amazonaws.com/tranquility/sde.zip + unzip sde.zip + + - name: Validate SDE version + run: | + SDE_VERSION=$(date -r sde.zip "+%F" | sed 's/-//g') + RELEASE_SDE_VERSION=$(echo ${{ github.event.release.tag_name }} | cut -d- -f2) + + echo "SDK version: ${SDE_VERSION}" + echo "Release version: ${RELEASE_SDE_VERSION}" + + if [ "${SDE_VERSION}" != "${RELEASE_SDE_VERSION}" ]; then + echo "SDE version mismatch: ${SDE_VERSION} != ${RELEASE_SDE_VERSION}" + exit 1 + fi + + - name: Convert SDE YAML to Protobuf + run: | + protoc --python_out=. esf.proto + python convert.py sde/fsd + + - name: Fetch icons + run: | + python download_icons.py + + - name: Publish to R2 bucket + run: | + rclone copy dist eveshipfit:eveshipfit/${{ github.event.release.tag_name }}/ --progress + env: + RCLONE_CONFIG_EVESHIPFIT_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + RCLONE_CONFIG_EVESHIPFIT_ENDPOINT: ${{ secrets.R2_ENDPOINT }} + RCLONE_CONFIG_EVESHIPFIT_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} diff --git a/convert.py b/convert.py index 456b21e..4302192 100644 --- a/convert.py +++ b/convert.py @@ -10,9 +10,12 @@ if len(sys.argv) < 2: path = sys.argv[1] -os.makedirs("dist", exist_ok=True) +os.makedirs("dist/sde", exist_ok=True) + def convert_type_dogma(path, ships): + print("Converting typeDogma ...") + with open(f"{path}/typeDogma.yaml") as fp: typeDogma = yaml.load(fp, Loader=yaml.CSafeLoader) @@ -20,35 +23,27 @@ def convert_type_dogma(path, ships): for id, entry in typeDogma.items(): for attribute in entry["dogmaAttributes"]: - pbea = pb2.TypeDogmaEntry.DogmaAttributes( - attributeID=attribute["attributeID"], - value=attribute["value"] - ) + pbea = pb2.TypeDogmaEntry.DogmaAttributes(attributeID=attribute["attributeID"], value=attribute["value"]) pb2.entries[id].dogmaAttributes.append(pbea) for effect in entry["dogmaEffects"]: - pbee = pb2.TypeDogmaEntry.DogmaEffects( - effectID=effect["effectID"], - isDefault=effect["isDefault"] - ) + pbee = pb2.TypeDogmaEntry.DogmaEffects(effectID=effect["effectID"], isDefault=effect["isDefault"]) pb2.entries[id].dogmaEffects.append(pbee) # Add the "applyVelocityBoost" effect for all ships. if id in ships: - pbee = pb2.TypeDogmaEntry.DogmaEffects( - effectID=-1, - isDefault=False - ) + pbee = pb2.TypeDogmaEntry.DogmaEffects(effectID=-1, isDefault=False) pb2.entries[id].dogmaEffects.append(pbee) - - with open("dist/typeDogma.pb2", "wb") as fp: + with open("dist/sde/typeDogma.pb2", "wb") as fp: fp.write(pb2.SerializeToString()) def convert_type_ids(path): + print("Converting typeIDs ...") + with open(f"{path}/groupIDs.yaml") as fp: groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader) @@ -78,13 +73,15 @@ def convert_type_ids(path): if "volume" in entry: pb2.entries[id].volume = entry["volume"] - with open("dist/typeIDs.pb2", "wb") as fp: + with open("dist/sde/typeIDs.pb2", "wb") as fp: fp.write(pb2.SerializeToString()) return ships def convert_dogma_attributes(path): + print("Converting dogmaAttributes ...") + with open(f"{path}/dogmaAttributes.yaml") as fp: dogmaAttributes = yaml.load(fp, Loader=yaml.CSafeLoader) @@ -128,11 +125,13 @@ def convert_dogma_attributes(path): add_esf_attribute(-21, "armorRepairRateEhp") add_esf_attribute(-22, "hullRepairRateEhp") - with open("dist/dogmaAttributes.pb2", "wb") as fp: + with open("dist/sde/dogmaAttributes.pb2", "wb") as fp: fp.write(pb2.SerializeToString()) def convert_dogma_effects(path): + print("Converting dogmaEffects ...") + with open(f"{path}/dogmaEffects.yaml") as fp: dogmaEffects = yaml.load(fp, Loader=yaml.CSafeLoader) @@ -150,7 +149,6 @@ def convert_dogma_effects(path): pb2.entries[id].modifierInfo.append(pbmi) - # Add the "applyVelocityBoost" effect. pb2.entries[-1].name = "applyVelocityBoost" pb2.entries[-1].effectCategory = 0 @@ -165,7 +163,6 @@ def convert_dogma_effects(path): add_modifier(-1, pbmi.Domain.itemID, pbmi.Func.ItemModifier, -7, 5, 4) # velocityBoost mass add_modifier(-1, pbmi.Domain.itemID, pbmi.Func.ItemModifier, 37, 6, -7) # maxVelocity velocityBoost - for id, entry in dogmaEffects.items(): pb2.entries[id].name = entry["effectName"] # In the SDE, the "online" effect is in the "active" category. @@ -202,21 +199,34 @@ def convert_dogma_effects(path): pbmi = pb2.DogmaEffect.ModifierInfo() match modifier_info["domain"]: - case "itemID": pbmi.domain = pbmi.Domain.itemID - case "shipID": pbmi.domain = pbmi.Domain.shipID - case "charID": pbmi.domain = pbmi.Domain.charID - case "otherID": pbmi.domain = pbmi.Domain.otherID - case "structureID": pbmi.domain = pbmi.Domain.structureID - case "target": pbmi.domain = pbmi.Domain.target - case "targetID": pbmi.domain = pbmi.Domain.targetID + case "itemID": + pbmi.domain = pbmi.Domain.itemID + case "shipID": + pbmi.domain = pbmi.Domain.shipID + case "charID": + pbmi.domain = pbmi.Domain.charID + case "otherID": + pbmi.domain = pbmi.Domain.otherID + case "structureID": + pbmi.domain = pbmi.Domain.structureID + case "target": + pbmi.domain = pbmi.Domain.target + case "targetID": + pbmi.domain = pbmi.Domain.targetID match modifier_info["func"]: - case "ItemModifier": pbmi.func = pbmi.Func.ItemModifier - case "LocationGroupModifier": pbmi.func = pbmi.Func.LocationGroupModifier - case "LocationModifier": pbmi.func = pbmi.Func.LocationModifier - case "LocationRequiredSkillModifier": pbmi.func = pbmi.Func.LocationRequiredSkillModifier - case "OwnerRequiredSkillModifier": pbmi.func = pbmi.Func.OwnerRequiredSkillModifier - case "EffectStopper": pbmi.func = pbmi.Func.EffectStopper + case "ItemModifier": + pbmi.func = pbmi.Func.ItemModifier + case "LocationGroupModifier": + pbmi.func = pbmi.Func.LocationGroupModifier + case "LocationModifier": + pbmi.func = pbmi.Func.LocationModifier + case "LocationRequiredSkillModifier": + pbmi.func = pbmi.Func.LocationRequiredSkillModifier + case "OwnerRequiredSkillModifier": + pbmi.func = pbmi.Func.OwnerRequiredSkillModifier + case "EffectStopper": + pbmi.func = pbmi.Func.EffectStopper if "modifiedAttributeID" in modifier_info: pbmi.modifiedAttributeID = modifier_info["modifiedAttributeID"] @@ -231,26 +241,32 @@ def convert_dogma_effects(path): pb2.entries[id].modifierInfo.append(pbmi) - # In the SDE, the ABs and MWDs don't have an active modifier effect. # Internally EVE does some magic here; but in our case, these # modifiers can just be assigned to the effects. if entry["effectName"] == "moduleBonusMicrowarpdrive": - add_modifier(id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, 552, 6, 554) # signatureRadius signatureRadiusBonus + add_modifier( + id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, 552, 6, 554 + ) # signatureRadius signatureRadiusBonus if entry["effectName"] == "moduleBonusAfterburner" or entry["effectName"] == "moduleBonusMicrowarpdrive": add_modifier(id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, 4, 2, 796) # mass massAddition # Velocity change is calculated like this: velocityBoost = item.speedFactor * item.speedBoostFactor / ship.mass # First, calculate the multiplication on the item. - add_modifier(id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, -7, -1, 567) # velocityBoost speedBoostFactor - add_modifier(id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, -7, 4, 20) # velocityBoost speedFactor + add_modifier( + id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, -7, -1, 567 + ) # velocityBoost speedBoostFactor + add_modifier( + id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, -7, 4, 20 + ) # velocityBoost speedFactor # Next, "applyVelocityBoost" is applied on all ships which takes care of the final calculation (as mass is an attribute of the ship). - with open("dist/dogmaEffects.pb2", "wb") as fp: + with open("dist/sde/dogmaEffects.pb2", "wb") as fp: fp.write(pb2.SerializeToString()) + ships = convert_type_ids(path) convert_type_dogma(path, ships) convert_dogma_attributes(path) diff --git a/download_icons.py b/download_icons.py new file mode 100644 index 0000000..754677f --- /dev/null +++ b/download_icons.py @@ -0,0 +1,29 @@ +import os +import requests +import shutil +import sys + +latest = requests.get("https://binaries.eveonline.com/eveclient_TQ.json").json() +build = latest["build"] + +installer = requests.get(f"https://binaries.eveonline.com/eveonline_{build}.txt").text +for line in installer.split("\n"): + if line.startswith("app:/resfileindex.txt"): + resfileindex = line.split(",")[1] + +resfile = requests.get(f"https://binaries.eveonline.com/{resfileindex}").text + +for line in resfile.split("\n"): + if not line: + continue + + res, path, _, _, _ = line.split(",") + + if res.startswith("res:/ui/texture/classes/fitting/"): + filename = res.split(":")[1][1:] + print(f"Downloading {filename} ...") + local_path = "dist/" + filename + + os.makedirs(os.path.dirname(local_path), exist_ok=True) + with open(local_path, "wb") as f: + f.write(requests.get(f"https://binaries.eveonline.com/{path}").content) diff --git a/requirements.base b/requirements.base index 3c6b7e1..6be77c7 100644 --- a/requirements.base +++ b/requirements.base @@ -1,2 +1,3 @@ pyyaml protobuf +requests diff --git a/requirements.txt b/requirements.txt index d906e16..195fbbf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,8 @@ +certifi==2023.11.17 +charset-normalizer==3.3.2 +idna==3.4 protobuf==3.12.4 PyYAML==6.0.1 +requests==2.31.0 +six==1.16.0 +urllib3==2.1.0