Use new table as source for skill requirement data

db_update.py

@@ -33,7 +33,7 @@ DB_PATH = os.path.join(ROOT_DIR, 'eve.db')
 JSON_DIR = os.path.join(ROOT_DIR, 'staticdata')
 if ROOT_DIR not in sys.path:
     sys.path.insert(0, ROOT_DIR)
-GAMEDATA_SCHEMA_VERSION = 1
+GAMEDATA_SCHEMA_VERSION = 2
 
 
 def db_needs_update():
@@ -319,6 +319,32 @@ def update_db():
         data = _readData('phobos', 'metadata')
         _addRows(data, eos.gamedata.MetaData)
 
+    def processReqSkills(eveTypesData):
+        print('processing requiredskillsfortypes')
+
+        def composeReqSkills(raw):
+            reqSkills = {}
+            for skillTypeID, skillLevels in raw.items():
+                reqSkills[int(skillTypeID)] = skillLevels[0]
+            return reqSkills
+
+        eveTypeIds = set(r['typeID'] for r in eveTypesData)
+        data = _readData('fsd_binary', 'requiredskillsfortypes')
+        reqsByItem = {}
+        itemsByReq = {}
+        for typeID, skillreqData in data.items():
+            typeID = int(typeID)
+            if typeID not in eveTypeIds:
+                continue
+            for skillTypeID, skillLevel in composeReqSkills(skillreqData).items():
+                reqsByItem.setdefault(typeID, {})[skillTypeID] = skillLevel
+                itemsByReq.setdefault(skillTypeID, {})[typeID] = skillLevel
+        for item in eos.db.gamedata_session.query(eos.gamedata.Item).all():
+            if item.typeID in reqsByItem:
+                item.reqskills = json.dumps(reqsByItem[item.typeID])
+            if item.typeID in itemsByReq:
+                item.requiredfor = json.dumps(itemsByReq[item.typeID])
+
     def processReplacements(eveTypesData, eveGroupsData, dogmaTypeAttributesData, dogmaTypeEffectsData):
         print('finding item replacements')
 
@@ -438,7 +464,9 @@ def update_db():
                     replacements.setdefault(type2[0], set()).add(type1[0])
         # Update DB session with data we generated
         for item in eos.db.gamedata_session.query(eos.gamedata.Item).all():
-            item.replacements = ','.join('{}'.format(tid) for tid in sorted(replacements.get(item.typeID, ())))
+            itemReplacements = replacements.get(item.typeID)
+            if itemReplacements is not None:
+                item.replacements = ','.join('{}'.format(tid) for tid in sorted(itemReplacements))
 
     eveTypesData = processEveTypes()
     eveGroupsData = processEveGroups()
@@ -456,6 +484,7 @@ def update_db():
     processMetadata()
 
     eos.db.gamedata_session.flush()
+    processReqSkills(eveTypesData)
     processReplacements(eveTypesData, eveGroupsData, dogmaTypeAttributesData, dogmaTypeEffectsData)
 
     # Add schema version to prevent further updates

eos/db/gamedata/item.py

@@ -44,7 +44,9 @@ items_table = Table("invtypes", gamedata_meta,
                     Column("metaLevel", Integer),
                     Column("metaGroupID", Integer, ForeignKey("invmetagroups.metaGroupID"), index=True),
                     Column("variationParentTypeID", Integer, ForeignKey("invtypes.typeID"), index=True),
-                    Column("replacements", String))
+                    Column("replacements", String),
+                    Column("reqskills", String),
+                    Column("requiredfor", String))
 
 from .traits import traits_table  # noqa
 

eos/db/gamedata/queries.py

@@ -424,25 +424,3 @@ def getDynamicItem(itemID, eager=None):
     except exc.NoResultFound:
         result = None
     return result
-
-
-def getRequiredFor(itemID, attrMapping):
-    Attribute1 = aliased(Attribute)
-    Attribute2 = aliased(Attribute)
-
-    skillToLevelClauses = []
-
-    for attrSkill, attrLevel in attrMapping.items():
-        skillToLevelClauses.append(and_(Attribute1.attributeID == attrSkill, Attribute2.attributeID == attrLevel))
-
-    queryOr = or_(*skillToLevelClauses)
-
-    q = select((Attribute2.typeID, Attribute2.value),
-               and_(Attribute1.value == itemID, queryOr),
-               from_obj=[
-                   join(Attribute1, Attribute2, Attribute1.typeID == Attribute2.typeID)
-               ])
-
-    result = gamedata_session.execute(q).fetchall()
-
-    return result

eos/gamedata.py

@@ -18,6 +18,7 @@
 # ===============================================================================
 
 
+import json
 from collections import OrderedDict
 
 from logbook import Logger
@@ -314,50 +315,26 @@ class Item(EqBase):
         eos.db.saveddata_session.delete(override)
         eos.db.commit()
 
-    srqIDMap = {182: 277, 183: 278, 184: 279, 1285: 1286, 1289: 1287, 1290: 1288}
-
     @property
     def requiredSkills(self):
         if self.__requiredSkills is None:
-            requiredSkills = OrderedDict()
-            self.__requiredSkills = requiredSkills
-            # Map containing attribute IDs we may need for required skills
-            # { requiredSkillX : requiredSkillXLevel }
-            combinedAttrIDs = set(self.srqIDMap.keys()).union(set(self.srqIDMap.values()))
-            # Map containing result of the request
-            # { attributeID : attributeValue }
-            skillAttrs = {}
-            # Get relevant attribute values from db (required skill IDs and levels) for our item
-            for attrInfo in eos.db.directAttributeRequest((self.ID,), tuple(combinedAttrIDs)):
-                attrID = attrInfo[1]
-                attrVal = attrInfo[2]
-                skillAttrs[attrID] = attrVal
-            # Go through all attributeID pairs
-            for srqIDAtrr, srqLvlAttr in self.srqIDMap.items():
-                # Check if we have both in returned result
-                if srqIDAtrr in skillAttrs and srqLvlAttr in skillAttrs:
-                    skillID = int(skillAttrs[srqIDAtrr])
-                    skillLvl = skillAttrs[srqLvlAttr]
-                    # Fetch item from database and fill map
-                    item = eos.db.getItem(skillID)
-                    requiredSkills[item] = skillLvl
+            self.__requiredSkills = {}
+            if self.reqskills:
+                for skillTypeID, skillLevel in json.loads(self.reqskills).items():
+                    skillItem = eos.db.getItem(int(skillTypeID))
+                    if skillItem:
+                        self.__requiredSkills[skillItem] = skillLevel
         return self.__requiredSkills
 
     @property
     def requiredFor(self):
         if self.__requiredFor is None:
-            self.__requiredFor = dict()
-
-            # Map containing attribute IDs we may need for required skills
-
-            # Get relevant attribute values from db (required skill IDs and levels) for our item
-            q = eos.db.getRequiredFor(self.ID, self.srqIDMap)
-
-            for itemID, lvl in q:
-                # Fetch item from database and fill map
-                item = eos.db.getItem(itemID)
-                self.__requiredFor[item] = lvl
-
+            self.__requiredFor = {}
+            if self.requiredfor:
+                for typeID, skillLevel in json.loads(self.requiredfor).items():
+                    requiredForItem = eos.db.getItem(int(typeID))
+                    if requiredForItem:
+                        self.__requiredFor[requiredForItem] = skillLevel
         return self.__requiredFor
 
     factionMap = {

service/market.py

@@ -802,7 +802,7 @@ class Market:
     def getReplacements(self, identity):
         item = self.getItem(identity)
         # We already store needed type IDs in database
-        replTypeIDs = {int(i) for i in item.replacements.split(",") if i}
+        replTypeIDs = {int(i) for i in item.replacements.split(",") if i} if item.replacements is not None else {}
         if not replTypeIDs:
             return ()
         # As replacements were generated without keeping track which items were published,