diff --git a/db_update.py b/db_update.py index 0a556acc4..ef31fb90d 100644 --- a/db_update.py +++ b/db_update.py @@ -88,52 +88,160 @@ def update_db(): # Create the database tables eos.db.gamedata_meta.create_all() - # Config dict - tables = { - 'clonegrades': ('fsd_lite', eos.gamedata.AlphaCloneSkill), - 'dogmaattributes': ('bulkdata', eos.gamedata.AttributeInfo), - 'dogmaeffects': ('bulkdata', eos.gamedata.Effect), - 'dogmatypeattributes': ('bulkdata', eos.gamedata.Attribute), - 'dogmatypeeffects': ('bulkdata', eos.gamedata.ItemEffect), - 'dogmaunits': ('bulkdata', eos.gamedata.Unit), - 'evecategories': ('fsd_lite', eos.gamedata.Category), - 'evegroups': ('fsd_lite', eos.gamedata.Group), - 'metagroups': ('fsd_binary', eos.gamedata.MetaGroup), - 'evetypes': ('fsd_lite', eos.gamedata.Item), - 'traits': ('phobos', eos.gamedata.Traits), - 'metadata': ('phobos', eos.gamedata.MetaData), - 'marketgroups': ('fsd_binary', eos.gamedata.MarketGroup)} + def _readData(minerName, jsonName, keyIdName=None): + with open(os.path.join(JSON_DIR, minerName, '{}.json'.format(jsonName)), encoding='utf-8') as f: + rawData = json.load(f) + if not keyIdName: + return rawData + # IDs in keys, rows in values + data = [] + for k, v in rawData.items(): + row = {} + row.update(v) + if keyIdName not in row: + row[keyIdName] = int(k) + data.append(row) + return data - fieldMapping = { - 'marketgroups': { - 'id': 'marketGroupID', - 'name': 'marketGroupName'}, - 'metagroups': { - 'id': 'metaGroupID'}} + def _addRows(data, cls, fieldMap=None): + if fieldMap is None: + fieldMap = {} + for row in data: + instance = cls() + for k, v in row.items(): + if isinstance(v, str): + v = v.strip() + setattr(instance, fieldMap.get(k, k), v) + eos.db.gamedata_session.add(instance) - rowsInValues = ( - 'evetypes', - 'evegroups', - 'evecategories', - 'marketgroups', - 'metagroups') + def processEveTypes(): + print('processing evetypes') + data = _readData('fsd_lite', 'evetypes', keyIdName='typeID') + for row in data: + if ( + # Apparently people really want Civilian modules available + (row['typeName'].startswith('Civilian') and "Shuttle" not in row['typeName']) or + row['typeName'] in ('Capsule', 'Dark Blood Tracking Disruptor') + ): + row['published'] = True - def convertIcons(data): - new = [] - for k, v in list(data.items()): - v['iconID'] = k - new.append(v) - return new - - def convertClones(data): newData = [] + for row in data: + if ( + row['published'] or + # group Ship Modifiers, for items like tactical t3 ship modes + row['groupID'] == 1306 or + # Micro Bombs (Fighters) + row['typeID'] in (41549, 41548, 41551, 41550) or + # Abyssal weather (environment) + row['groupID'] in ( + 1882, + 1975, + 1971, + # the "container" for the abyssal environments + 1983) + ): + newData.append(row) + _addRows(newData, eos.gamedata.Item) + return newData + + def processEveGroups(): + print('processing evegroups') + data = _readData('fsd_lite', 'evegroups', keyIdName='groupID') + _addRows(data, eos.gamedata.Group) + return data + + def processEveCategories(): + print('processing evecategories') + data = _readData('fsd_lite', 'evecategories', keyIdName='categoryID') + _addRows(data, eos.gamedata.Category) + + def processDogmaAttributes(): + print('processing dogmaattributes') + data = _readData('fsd_binary', 'dogmaattributes', keyIdName='attributeID') + _addRows(data, eos.gamedata.AttributeInfo) + + def processDogmaTypeAttributes(eveTypesData): + print('processing dogmatypeattributes') + data = _readData('fsd_binary', 'typedogma', keyIdName='typeID') + eveTypeIds = set(r['typeID'] for r in eveTypesData) + newData = [] + for typeData in data: + if typeData['typeID'] not in eveTypeIds: + continue + for row in typeData.get('dogmaAttributes', ()): + row['typeID'] = typeData['typeID'] + newData.append(row) + _addRows(newData, eos.gamedata.Attribute) + return newData + + def processDynamicItemAttributes(): + print('processing dynamicitemattributes') + data = _readData('fsd_binary', 'dynamicitemattributes') + for mutaID, mutaData in data.items(): + muta = eos.gamedata.DynamicItem() + muta.typeID = mutaID + muta.resultingTypeID = mutaData['inputOutputMapping'][0]['resultingType'] + eos.db.gamedata_session.add(muta) + + for x in mutaData['inputOutputMapping'][0]['applicableTypes']: + item = eos.gamedata.DynamicItemItem() + item.typeID = mutaID + item.applicableTypeID = x + eos.db.gamedata_session.add(item) + + for attrID, attrData in mutaData['attributeIDs'].items(): + attr = eos.gamedata.DynamicItemAttribute() + attr.typeID = mutaID + attr.attributeID = attrID + attr.min = attrData['min'] + attr.max = attrData['max'] + eos.db.gamedata_session.add(attr) + + def processDogmaEffects(): + print('processing dogmaeffects') + data = _readData('fsd_binary', 'dogmaeffects', keyIdName='effectID') + _addRows(data, eos.gamedata.Effect) + + def processDogmaTypeEffects(eveTypesData): + print('processing dogmatypeeffects') + data = _readData('fsd_binary', 'typedogma', keyIdName='typeID') + eveTypeIds = set(r['typeID'] for r in eveTypesData) + newData = [] + for typeData in data: + if typeData['typeID'] not in eveTypeIds: + continue + for row in typeData.get('dogmaEffects', ()): + row['typeID'] = typeData['typeID'] + newData.append(row) + _addRows(newData, eos.gamedata.ItemEffect) + return newData + + def processDogmaUnits(): + print('processing dogmaunits') + data = _readData('fsd_binary', 'dogmaunits', keyIdName='unitID') + _addRows(data, eos.gamedata.Unit, fieldMap={'name': 'marketGroupName'}) + + def processMarketGroups(): + print('processing marketgroups') + data = _readData('fsd_binary', 'marketgroups', keyIdName='marketGroupID') + _addRows(data, eos.gamedata.MarketGroup) + + def processMetaGroups(): + print('processing metagroups') + data = _readData('fsd_binary', 'metagroups', keyIdName='metaGroupID') + _addRows(data, eos.gamedata.MetaGroup) + + def processCloneGrades(): + print('processing clonegrades') + data = _readData('fsd_lite', 'clonegrades') + + newData = [] # December, 2017 - CCP decided to use only one set of skill levels for alpha clones. However, this is still # represented in the data as a skillset per race. To ensure that all skills are the same, we store them in a way # that we can check to make sure all races have the same skills, as well as skill levels - check = {} - for ID in data: for skill in data[ID]['skills']: newData.append({ @@ -144,18 +252,25 @@ def update_db(): if ID not in check: check[ID] = {} check[ID][int(skill['typeID'])] = int(skill['level']) - if not functools.reduce(lambda a, b: a if a == b else False, [v for _, v in check.items()]): raise Exception('Alpha Clones not all equal') - newData = [x for x in newData if x['alphaCloneID'] == 1] - if len(newData) == 0: raise Exception('Alpha Clone processing failed') - return newData + tmp = [] + for row in newData: + if row['alphaCloneID'] not in tmp: + cloneParent = eos.gamedata.AlphaClone() + setattr(cloneParent, 'alphaCloneID', row['alphaCloneID']) + setattr(cloneParent, 'alphaCloneName', row['alphaCloneName']) + eos.db.gamedata_session.add(cloneParent) + tmp.append(row['alphaCloneID']) + _addRows(newData, eos.gamedata.AlphaCloneSkill) - def convertTraits(data): + def processTraits(): + print('processing traits') + data = _readData('phobos', 'traits') def convertSection(sectionData): sectionLines = [] @@ -182,9 +297,16 @@ def update_db(): traitLine = '
\n
\n'.join(typeLines) newRow = {'typeID': typeId, 'traitText': traitLine} newData.append(newRow) - return newData - def fillReplacements(tables): + _addRows(newData, eos.gamedata.Traits) + + def processMetadata(): + print('processing metadata') + data = _readData('phobos', 'metadata') + _addRows(data, eos.gamedata.MetaData) + + def processReplacements(eveTypesData, eveGroupsData, dogmaTypeAttributesData, dogmaTypeEffectsData): + print('finding item replacements') def compareAttrs(attrs1, attrs2): # Consider items as different if they have no attrs @@ -196,7 +318,6 @@ def update_db(): return True return False - print('finding replacements') skillReqAttribs = { 182: 277, 183: 278, @@ -208,18 +329,18 @@ def update_db(): # Get data on type groups # Format: {type ID: group ID} typesGroups = {} - for row in tables['evetypes']: + for row in eveTypesData: typesGroups[row['typeID']] = row['groupID'] # Get data on item effects # Format: {type ID: set(effect, IDs)} typesEffects = {} - for row in tables['dogmatypeeffects']: + for row in dogmaTypeEffectsData: typesEffects.setdefault(row['typeID'], set()).add(row['effectID']) # Get data on type attributes # Format: {type ID: {attribute ID: attribute value}} typesNormalAttribs = {} typesSkillAttribs = {} - for row in tables['dogmatypeattributes']: + for row in dogmaTypeAttributesData: attributeID = row['attributeID'] if attributeID in skillReqAttribsFlat: typeSkillAttribs = typesSkillAttribs.setdefault(row['typeID'], {}) @@ -258,13 +379,13 @@ def update_db(): typeSkillReqs[skillType] = skillLevel # Format: {group ID: category ID} groupCategories = {} - for row in tables['evegroups']: + for row in eveGroupsData: groupCategories[row['groupID']] = row['categoryID'] # As EVE affects various types mostly depending on their group or skill requirements, # we're going to group various types up this way # Format: {(group ID, frozenset(skillreq, type, IDs), frozenset(type, effect, IDs): [type ID, {attribute ID: attribute value}]} groupedData = {} - for row in tables['evetypes']: + for row in eveTypesData: typeID = row['typeID'] # Ignore items outside of categories we need if groupCategories[typesGroups[typeID]] not in ( @@ -301,134 +422,25 @@ def update_db(): if compareAttrs(type1[1], type2[1]): replacements.setdefault(type1[0], set()).add(type2[0]) replacements.setdefault(type2[0], set()).add(type1[0]) - # Put this data into types table so that normal process hooks it up - for row in tables['evetypes']: - row['replacements'] = ','.join('{}'.format(tid) for tid in sorted(replacements.get(row['typeID'], ()))) + # Update DB session with data we generated + for item in eos.db.gamedata_session.query(eos.gamedata.Item).all(): + item.replacements = ','.join('{}'.format(tid) for tid in sorted(replacements.get(item.typeID, ()))) - data = {} - - # Dump all data to memory so we can easely cross check ignored rows - for jsonName, (minerName, cls) in tables.items(): - with open(os.path.join(JSON_DIR, minerName, '{}.json'.format(jsonName)), encoding='utf-8') as f: - tableData = json.load(f) - if jsonName in rowsInValues: - newTableData = [] - for k, v in tableData.items(): - row = {} - row.update(v) - if 'id' not in row: - row['id'] = int(k) - newTableData.append(row) - tableData = newTableData - if jsonName == 'icons': - tableData = convertIcons(tableData) - if jsonName == 'traits': - tableData = convertTraits(tableData) - if jsonName == 'clonegrades': - tableData = convertClones(tableData) - data[jsonName] = tableData - - fillReplacements(data) - - # Set with typeIDs which we will have in our database - # Sometimes CCP unpublishes some items we want to have published, we - # can do it here - just add them to initial set - eveTypes = set() - for row in data['evetypes']: - if ( - row['published'] or - row['typeName'] == 'Capsule' or - # group Ship Modifiers, for items like tactical t3 ship modes - row['groupID'] == 1306 or - # Civilian weapons - (row['typeName'].startswith('Civilian') and "Shuttle" not in row['typeName']) or - # Micro Bombs (Fighters) - row['typeID'] in (41549, 41548, 41551, 41550) or - # Abyssal weather (environment) - row['groupID'] in ( - 1882, - 1975, - 1971, - # the "container" for the abyssal environments - 1983) or - # Dark Blood Tracking Disruptor (drops, but rarely) - row['typeID'] == 32416 - ): - eveTypes.add(row['typeID']) - - # ignore checker - def isIgnored(file, row): - if file in ('evetypes', 'dogmatypeeffects', 'dogmatypeattributes') and row['typeID'] not in eveTypes: - return True - return False - - # Loop through each json file and write it away, checking ignored rows - for jsonName, table in data.items(): - fieldMap = fieldMapping.get(jsonName, {}) - tmp = [] - - print('processing {}'.format(jsonName)) - - for row in table: - # We don't care about some kind of rows, filter it out if so - if not isIgnored(jsonName, row): - if ( - jsonName == 'evetypes' and ( - # Apparently people really want Civilian modules available - (row['typeName'].startswith('Civilian') and "Shuttle" not in row['typeName']) or - row['typeName'] in ('Capsule', 'Dark Blood Tracking Disruptor')) - ): - row['published'] = True - - instance = tables[jsonName][1]() - # fix for issue 80 - if jsonName is 'icons' and 'res:/ui/texture/icons/' in str(row['iconFile']).lower(): - row['iconFile'] = row['iconFile'].lower().replace('res:/ui/texture/icons/', '').replace('.png', '') - # with res:/ui... references, it points to the actual icon file (including it's size variation of #_size_#) - # strip this info out and get the identifying info - split = row['iconFile'].split('_') - if len(split) == 3: - row['iconFile'] = '{}_{}'.format(split[0], split[2]) - if jsonName is 'icons' and 'modules/' in str(row['iconFile']).lower(): - row['iconFile'] = row['iconFile'].lower().replace('modules/', '').replace('.png', '') - - if jsonName is 'clonegrades': - if row['alphaCloneID'] not in tmp: - cloneParent = eos.gamedata.AlphaClone() - setattr(cloneParent, 'alphaCloneID', row['alphaCloneID']) - setattr(cloneParent, 'alphaCloneName', row['alphaCloneName']) - eos.db.gamedata_session.add(cloneParent) - tmp.append(row['alphaCloneID']) - - for k, v in row.items(): - if isinstance(v, str): - v = v.strip() - setattr(instance, fieldMap.get(k, k), v) - - eos.db.gamedata_session.add(instance) - - # quick and dirty hack to get this data in - with open(os.path.join(JSON_DIR, 'fsd_binary', 'dynamicitemattributes.json'), encoding='utf-8') as f: - bulkdata = json.load(f) - for mutaID, data in bulkdata.items(): - muta = eos.gamedata.DynamicItem() - muta.typeID = mutaID - muta.resultingTypeID = data['inputOutputMapping'][0]['resultingType'] - eos.db.gamedata_session.add(muta) - - for x in data['inputOutputMapping'][0]['applicableTypes']: - item = eos.gamedata.DynamicItemItem() - item.typeID = mutaID - item.applicableTypeID = x - eos.db.gamedata_session.add(item) - - for attrID, attrData in data['attributeIDs'].items(): - attr = eos.gamedata.DynamicItemAttribute() - attr.typeID = mutaID - attr.attributeID = attrID - attr.min = attrData['min'] - attr.max = attrData['max'] - eos.db.gamedata_session.add(attr) + eveTypesData = processEveTypes() + eveGroupsData = processEveGroups() + processEveCategories() + processDogmaAttributes() + dogmaTypeAttributesData = processDogmaTypeAttributes(eveTypesData) + processDynamicItemAttributes() + processDogmaEffects() + dogmaTypeEffectsData = processDogmaTypeEffects(eveTypesData) + processDogmaUnits() + processMarketGroups() + processMetaGroups() + processCloneGrades() + processTraits() + processMetadata() + processReplacements(eveTypesData, eveGroupsData, dogmaTypeAttributesData, dogmaTypeEffectsData) # Add schema version to prevent further updates metadata_schema_version = eos.gamedata.MetaData()