feat: automate releasing of new data-files (#11)

2023-11-20 20:53:02 +01:00
parent 02a05a4e68
commit ec6c67fcf4
5 changed files with 155 additions and 37 deletions
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -0,0 +1,66 @@
+name: Release
+
+on:
+  release:
+    types:
+    - published
+
+jobs:
+  datafiles:
+    name: Publish datafiles
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y --no-install-recommends rclone protobuf-compiler
+        pip install -r requirements.txt
+
+    - name: Setup rclone
+      run: |
+        mkdir -p ~/.config/rclone
+
+        echo "[eveshipfit]" > ~/.config/rclone/rclone.conf
+        echo "type = s3" >> ~/.config/rclone/rclone.conf
+        echo "provider = Cloudflare" >> ~/.config/rclone/rclone.conf
+        echo "acl = private" >> ~/.config/rclone/rclone.conf
+        echo "no_check_bucket = true" >> ~/.config/rclone/rclone.conf
+
+    - name: Fetch SDE
+      run: |
+        wget -q https://eve-static-data-export.s3-eu-west-1.amazonaws.com/tranquility/sde.zip
+        unzip sde.zip
+
+    - name: Validate SDE version
+      run: |
+        SDE_VERSION=$(date -r sde.zip "+%F" | sed 's/-//g')
+        RELEASE_SDE_VERSION=$(echo ${{ github.event.release.tag_name }} | cut -d- -f2)
+
+        echo "SDK version: ${SDE_VERSION}"
+        echo "Release version: ${RELEASE_SDE_VERSION}"
+
+        if [ "${SDE_VERSION}" != "${RELEASE_SDE_VERSION}" ]; then
+          echo "SDE version mismatch: ${SDE_VERSION} != ${RELEASE_SDE_VERSION}"
+          exit 1
+        fi
+
+    - name: Convert SDE YAML to Protobuf
+      run: |
+        protoc --python_out=. esf.proto
+        python convert.py sde/fsd
+
+    - name: Fetch icons
+      run: |
+        python download_icons.py
+
+    - name: Publish to R2 bucket
+      run: |
+        rclone copy dist eveshipfit:eveshipfit/${{ github.event.release.tag_name }}/ --progress
+      env:
+        RCLONE_CONFIG_EVESHIPFIT_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+        RCLONE_CONFIG_EVESHIPFIT_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+        RCLONE_CONFIG_EVESHIPFIT_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
--- a/convert.py
+++ b/convert.py
@@ -10,9 +10,12 @@ if len(sys.argv) < 2:

 path = sys.argv[1]

-os.makedirs("dist", exist_ok=True)
+os.makedirs("dist/sde", exist_ok=True)
+

 def convert_type_dogma(path, ships):
+    print("Converting typeDogma ...")
+
    with open(f"{path}/typeDogma.yaml") as fp:
        typeDogma = yaml.load(fp, Loader=yaml.CSafeLoader)

@@ -20,35 +23,27 @@ def convert_type_dogma(path, ships):

    for id, entry in typeDogma.items():
        for attribute in entry["dogmaAttributes"]:
-            pbea = pb2.TypeDogmaEntry.DogmaAttributes(
-                attributeID=attribute["attributeID"],
-                value=attribute["value"]
-            )
+            pbea = pb2.TypeDogmaEntry.DogmaAttributes(attributeID=attribute["attributeID"], value=attribute["value"])

            pb2.entries[id].dogmaAttributes.append(pbea)

        for effect in entry["dogmaEffects"]:
-            pbee = pb2.TypeDogmaEntry.DogmaEffects(
-                effectID=effect["effectID"],
-                isDefault=effect["isDefault"]
-            )
+            pbee = pb2.TypeDogmaEntry.DogmaEffects(effectID=effect["effectID"], isDefault=effect["isDefault"])

            pb2.entries[id].dogmaEffects.append(pbee)

        # Add the "applyVelocityBoost" effect for all ships.
        if id in ships:
-            pbee = pb2.TypeDogmaEntry.DogmaEffects(
-                effectID=-1,
-                isDefault=False
-            )
+            pbee = pb2.TypeDogmaEntry.DogmaEffects(effectID=-1, isDefault=False)
            pb2.entries[id].dogmaEffects.append(pbee)

-
-    with open("dist/typeDogma.pb2", "wb") as fp:
+    with open("dist/sde/typeDogma.pb2", "wb") as fp:
        fp.write(pb2.SerializeToString())


 def convert_type_ids(path):
+    print("Converting typeIDs ...")
+
    with open(f"{path}/groupIDs.yaml") as fp:
        groupIDs = yaml.load(fp, Loader=yaml.CSafeLoader)

@@ -78,13 +73,15 @@ def convert_type_ids(path):
        if "volume" in entry:
            pb2.entries[id].volume = entry["volume"]

-    with open("dist/typeIDs.pb2", "wb") as fp:
+    with open("dist/sde/typeIDs.pb2", "wb") as fp:
        fp.write(pb2.SerializeToString())

    return ships


 def convert_dogma_attributes(path):
+    print("Converting dogmaAttributes ...")
+
    with open(f"{path}/dogmaAttributes.yaml") as fp:
        dogmaAttributes = yaml.load(fp, Loader=yaml.CSafeLoader)

@@ -128,11 +125,13 @@ def convert_dogma_attributes(path):
    add_esf_attribute(-21, "armorRepairRateEhp")
    add_esf_attribute(-22, "hullRepairRateEhp")

-    with open("dist/dogmaAttributes.pb2", "wb") as fp:
+    with open("dist/sde/dogmaAttributes.pb2", "wb") as fp:
        fp.write(pb2.SerializeToString())


 def convert_dogma_effects(path):
+    print("Converting dogmaEffects ...")
+
    with open(f"{path}/dogmaEffects.yaml") as fp:
        dogmaEffects = yaml.load(fp, Loader=yaml.CSafeLoader)

@@ -150,7 +149,6 @@ def convert_dogma_effects(path):

        pb2.entries[id].modifierInfo.append(pbmi)

-
    # Add the "applyVelocityBoost" effect.
    pb2.entries[-1].name = "applyVelocityBoost"
    pb2.entries[-1].effectCategory = 0
@@ -165,7 +163,6 @@ def convert_dogma_effects(path):
    add_modifier(-1, pbmi.Domain.itemID, pbmi.Func.ItemModifier, -7, 5, 4)  # velocityBoost <postDiv> mass
    add_modifier(-1, pbmi.Domain.itemID, pbmi.Func.ItemModifier, 37, 6, -7)  # maxVelocity <postPercent> velocityBoost

-
    for id, entry in dogmaEffects.items():
        pb2.entries[id].name = entry["effectName"]
        # In the SDE, the "online" effect is in the "active" category.
@@ -202,21 +199,34 @@ def convert_dogma_effects(path):
                pbmi = pb2.DogmaEffect.ModifierInfo()

                match modifier_info["domain"]:
-                    case "itemID": pbmi.domain = pbmi.Domain.itemID
-                    case "shipID": pbmi.domain = pbmi.Domain.shipID
-                    case "charID": pbmi.domain = pbmi.Domain.charID
-                    case "otherID": pbmi.domain = pbmi.Domain.otherID
-                    case "structureID": pbmi.domain = pbmi.Domain.structureID
-                    case "target": pbmi.domain = pbmi.Domain.target
-                    case "targetID": pbmi.domain = pbmi.Domain.targetID
+                    case "itemID":
+                        pbmi.domain = pbmi.Domain.itemID
+                    case "shipID":
+                        pbmi.domain = pbmi.Domain.shipID
+                    case "charID":
+                        pbmi.domain = pbmi.Domain.charID
+                    case "otherID":
+                        pbmi.domain = pbmi.Domain.otherID
+                    case "structureID":
+                        pbmi.domain = pbmi.Domain.structureID
+                    case "target":
+                        pbmi.domain = pbmi.Domain.target
+                    case "targetID":
+                        pbmi.domain = pbmi.Domain.targetID

                match modifier_info["func"]:
-                    case "ItemModifier": pbmi.func = pbmi.Func.ItemModifier
-                    case "LocationGroupModifier": pbmi.func = pbmi.Func.LocationGroupModifier
-                    case "LocationModifier": pbmi.func = pbmi.Func.LocationModifier
-                    case "LocationRequiredSkillModifier": pbmi.func = pbmi.Func.LocationRequiredSkillModifier
-                    case "OwnerRequiredSkillModifier": pbmi.func = pbmi.Func.OwnerRequiredSkillModifier
-                    case "EffectStopper": pbmi.func = pbmi.Func.EffectStopper
+                    case "ItemModifier":
+                        pbmi.func = pbmi.Func.ItemModifier
+                    case "LocationGroupModifier":
+                        pbmi.func = pbmi.Func.LocationGroupModifier
+                    case "LocationModifier":
+                        pbmi.func = pbmi.Func.LocationModifier
+                    case "LocationRequiredSkillModifier":
+                        pbmi.func = pbmi.Func.LocationRequiredSkillModifier
+                    case "OwnerRequiredSkillModifier":
+                        pbmi.func = pbmi.Func.OwnerRequiredSkillModifier
+                    case "EffectStopper":
+                        pbmi.func = pbmi.Func.EffectStopper

                if "modifiedAttributeID" in modifier_info:
                    pbmi.modifiedAttributeID = modifier_info["modifiedAttributeID"]
@@ -231,26 +241,32 @@ def convert_dogma_effects(path):

                pb2.entries[id].modifierInfo.append(pbmi)

-
        # In the SDE, the ABs and MWDs don't have an active modifier effect.
        # Internally EVE does some magic here; but in our case, these
        # modifiers can just be assigned to the effects.
        if entry["effectName"] == "moduleBonusMicrowarpdrive":
-            add_modifier(id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, 552, 6, 554)  # signatureRadius <postPercent> signatureRadiusBonus
+            add_modifier(
+                id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, 552, 6, 554
+            )  # signatureRadius <postPercent> signatureRadiusBonus

        if entry["effectName"] == "moduleBonusAfterburner" or entry["effectName"] == "moduleBonusMicrowarpdrive":
            add_modifier(id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, 4, 2, 796)  # mass <modAdd> massAddition

            # Velocity change is calculated like this: velocityBoost = item.speedFactor * item.speedBoostFactor / ship.mass
            # First, calculate the multiplication on the item.
-            add_modifier(id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, -7, -1, 567)  # velocityBoost <preAssign> speedBoostFactor
-            add_modifier(id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, -7, 4, 20)  # velocityBoost <postMul> speedFactor
+            add_modifier(
+                id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, -7, -1, 567
+            )  # velocityBoost <preAssign> speedBoostFactor
+            add_modifier(
+                id, pbmi.Domain.shipID, pbmi.Func.ItemModifier, -7, 4, 20
+            )  # velocityBoost <postMul> speedFactor

            # Next, "applyVelocityBoost" is applied on all ships which takes care of the final calculation (as mass is an attribute of the ship).

-    with open("dist/dogmaEffects.pb2", "wb") as fp:
+    with open("dist/sde/dogmaEffects.pb2", "wb") as fp:
        fp.write(pb2.SerializeToString())

+
 ships = convert_type_ids(path)
 convert_type_dogma(path, ships)
 convert_dogma_attributes(path)
--- a/download_icons.py
+++ b/download_icons.py
@@ -0,0 +1,29 @@
+import os
+import requests
+import shutil
+import sys
+
+latest = requests.get("https://binaries.eveonline.com/eveclient_TQ.json").json()
+build = latest["build"]
+
+installer = requests.get(f"https://binaries.eveonline.com/eveonline_{build}.txt").text
+for line in installer.split("\n"):
+    if line.startswith("app:/resfileindex.txt"):
+        resfileindex = line.split(",")[1]
+
+resfile = requests.get(f"https://binaries.eveonline.com/{resfileindex}").text
+
+for line in resfile.split("\n"):
+    if not line:
+        continue
+
+    res, path, _, _, _ = line.split(",")
+
+    if res.startswith("res:/ui/texture/classes/fitting/"):
+        filename = res.split(":")[1][1:]
+        print(f"Downloading {filename} ...")
+        local_path = "dist/" + filename
+
+        os.makedirs(os.path.dirname(local_path), exist_ok=True)
+        with open(local_path, "wb") as f:
+            f.write(requests.get(f"https://binaries.eveonline.com/{path}").content)
--- a/requirements.base
+++ b/requirements.base
@@ -1,2 +1,3 @@
 pyyaml
 protobuf
+requests
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,8 @@
+certifi==2023.11.17
+charset-normalizer==3.3.2
+idna==3.4
 protobuf==3.12.4
 PyYAML==6.0.1
+requests==2.31.0
+six==1.16.0
+urllib3==2.1.0