From c45abbdbcf0ac02a8cbe3d0cbfada25d15173e38 Mon Sep 17 00:00:00 2001 From: DarkPhoenix Date: Thu, 19 Dec 2019 20:44:17 +0300 Subject: [PATCH] Use new table as source for skill requirement data --- db_update.py | 33 +++++++++++++++++++++++-- eos/db/gamedata/item.py | 4 +++- eos/db/gamedata/queries.py | 22 ----------------- eos/gamedata.py | 49 ++++++++++---------------------------- service/market.py | 2 +- 5 files changed, 48 insertions(+), 62 deletions(-) diff --git a/db_update.py b/db_update.py index 17e6eb637..098312079 100644 --- a/db_update.py +++ b/db_update.py @@ -33,7 +33,7 @@ DB_PATH = os.path.join(ROOT_DIR, 'eve.db') JSON_DIR = os.path.join(ROOT_DIR, 'staticdata') if ROOT_DIR not in sys.path: sys.path.insert(0, ROOT_DIR) -GAMEDATA_SCHEMA_VERSION = 1 +GAMEDATA_SCHEMA_VERSION = 2 def db_needs_update(): @@ -319,6 +319,32 @@ def update_db(): data = _readData('phobos', 'metadata') _addRows(data, eos.gamedata.MetaData) + def processReqSkills(eveTypesData): + print('processing requiredskillsfortypes') + + def composeReqSkills(raw): + reqSkills = {} + for skillTypeID, skillLevels in raw.items(): + reqSkills[int(skillTypeID)] = skillLevels[0] + return reqSkills + + eveTypeIds = set(r['typeID'] for r in eveTypesData) + data = _readData('fsd_binary', 'requiredskillsfortypes') + reqsByItem = {} + itemsByReq = {} + for typeID, skillreqData in data.items(): + typeID = int(typeID) + if typeID not in eveTypeIds: + continue + for skillTypeID, skillLevel in composeReqSkills(skillreqData).items(): + reqsByItem.setdefault(typeID, {})[skillTypeID] = skillLevel + itemsByReq.setdefault(skillTypeID, {})[typeID] = skillLevel + for item in eos.db.gamedata_session.query(eos.gamedata.Item).all(): + if item.typeID in reqsByItem: + item.reqskills = json.dumps(reqsByItem[item.typeID]) + if item.typeID in itemsByReq: + item.requiredfor = json.dumps(itemsByReq[item.typeID]) + def processReplacements(eveTypesData, eveGroupsData, dogmaTypeAttributesData, dogmaTypeEffectsData): print('finding item replacements') @@ -438,7 +464,9 @@ def update_db(): replacements.setdefault(type2[0], set()).add(type1[0]) # Update DB session with data we generated for item in eos.db.gamedata_session.query(eos.gamedata.Item).all(): - item.replacements = ','.join('{}'.format(tid) for tid in sorted(replacements.get(item.typeID, ()))) + itemReplacements = replacements.get(item.typeID) + if itemReplacements is not None: + item.replacements = ','.join('{}'.format(tid) for tid in sorted(itemReplacements)) eveTypesData = processEveTypes() eveGroupsData = processEveGroups() @@ -456,6 +484,7 @@ def update_db(): processMetadata() eos.db.gamedata_session.flush() + processReqSkills(eveTypesData) processReplacements(eveTypesData, eveGroupsData, dogmaTypeAttributesData, dogmaTypeEffectsData) # Add schema version to prevent further updates diff --git a/eos/db/gamedata/item.py b/eos/db/gamedata/item.py index 16f1fd134..d4603b8a1 100644 --- a/eos/db/gamedata/item.py +++ b/eos/db/gamedata/item.py @@ -44,7 +44,9 @@ items_table = Table("invtypes", gamedata_meta, Column("metaLevel", Integer), Column("metaGroupID", Integer, ForeignKey("invmetagroups.metaGroupID"), index=True), Column("variationParentTypeID", Integer, ForeignKey("invtypes.typeID"), index=True), - Column("replacements", String)) + Column("replacements", String), + Column("reqskills", String), + Column("requiredfor", String)) from .traits import traits_table # noqa diff --git a/eos/db/gamedata/queries.py b/eos/db/gamedata/queries.py index 9d48db1a2..19c5321b4 100644 --- a/eos/db/gamedata/queries.py +++ b/eos/db/gamedata/queries.py @@ -424,25 +424,3 @@ def getDynamicItem(itemID, eager=None): except exc.NoResultFound: result = None return result - - -def getRequiredFor(itemID, attrMapping): - Attribute1 = aliased(Attribute) - Attribute2 = aliased(Attribute) - - skillToLevelClauses = [] - - for attrSkill, attrLevel in attrMapping.items(): - skillToLevelClauses.append(and_(Attribute1.attributeID == attrSkill, Attribute2.attributeID == attrLevel)) - - queryOr = or_(*skillToLevelClauses) - - q = select((Attribute2.typeID, Attribute2.value), - and_(Attribute1.value == itemID, queryOr), - from_obj=[ - join(Attribute1, Attribute2, Attribute1.typeID == Attribute2.typeID) - ]) - - result = gamedata_session.execute(q).fetchall() - - return result diff --git a/eos/gamedata.py b/eos/gamedata.py index 7eb2bd160..6a4b90def 100644 --- a/eos/gamedata.py +++ b/eos/gamedata.py @@ -18,6 +18,7 @@ # =============================================================================== +import json from collections import OrderedDict from logbook import Logger @@ -314,50 +315,26 @@ class Item(EqBase): eos.db.saveddata_session.delete(override) eos.db.commit() - srqIDMap = {182: 277, 183: 278, 184: 279, 1285: 1286, 1289: 1287, 1290: 1288} - @property def requiredSkills(self): if self.__requiredSkills is None: - requiredSkills = OrderedDict() - self.__requiredSkills = requiredSkills - # Map containing attribute IDs we may need for required skills - # { requiredSkillX : requiredSkillXLevel } - combinedAttrIDs = set(self.srqIDMap.keys()).union(set(self.srqIDMap.values())) - # Map containing result of the request - # { attributeID : attributeValue } - skillAttrs = {} - # Get relevant attribute values from db (required skill IDs and levels) for our item - for attrInfo in eos.db.directAttributeRequest((self.ID,), tuple(combinedAttrIDs)): - attrID = attrInfo[1] - attrVal = attrInfo[2] - skillAttrs[attrID] = attrVal - # Go through all attributeID pairs - for srqIDAtrr, srqLvlAttr in self.srqIDMap.items(): - # Check if we have both in returned result - if srqIDAtrr in skillAttrs and srqLvlAttr in skillAttrs: - skillID = int(skillAttrs[srqIDAtrr]) - skillLvl = skillAttrs[srqLvlAttr] - # Fetch item from database and fill map - item = eos.db.getItem(skillID) - requiredSkills[item] = skillLvl + self.__requiredSkills = {} + if self.reqskills: + for skillTypeID, skillLevel in json.loads(self.reqskills).items(): + skillItem = eos.db.getItem(int(skillTypeID)) + if skillItem: + self.__requiredSkills[skillItem] = skillLevel return self.__requiredSkills @property def requiredFor(self): if self.__requiredFor is None: - self.__requiredFor = dict() - - # Map containing attribute IDs we may need for required skills - - # Get relevant attribute values from db (required skill IDs and levels) for our item - q = eos.db.getRequiredFor(self.ID, self.srqIDMap) - - for itemID, lvl in q: - # Fetch item from database and fill map - item = eos.db.getItem(itemID) - self.__requiredFor[item] = lvl - + self.__requiredFor = {} + if self.requiredfor: + for typeID, skillLevel in json.loads(self.requiredfor).items(): + requiredForItem = eos.db.getItem(int(typeID)) + if requiredForItem: + self.__requiredFor[requiredForItem] = skillLevel return self.__requiredFor factionMap = { diff --git a/service/market.py b/service/market.py index d55148800..cf6406a9d 100644 --- a/service/market.py +++ b/service/market.py @@ -802,7 +802,7 @@ class Market: def getReplacements(self, identity): item = self.getItem(identity) # We already store needed type IDs in database - replTypeIDs = {int(i) for i in item.replacements.split(",") if i} + replTypeIDs = {int(i) for i in item.replacements.split(",") if i} if item.replacements is not None else {} if not replTypeIDs: return () # As replacements were generated without keeping track which items were published,