Merge branch 'singularity'

db_update.py

@@ -22,6 +22,7 @@
 import functools
 import itertools
 import json
+import math
 import os
 import sqlite3
 import sys
@@ -32,7 +33,7 @@ DB_PATH = os.path.join(ROOT_DIR, 'eve.db')
 JSON_DIR = os.path.join(ROOT_DIR, 'staticdata')
 if ROOT_DIR not in sys.path:
     sys.path.insert(0, ROOT_DIR)
-GAMEDATA_SCHEMA_VERSION = 1
+GAMEDATA_SCHEMA_VERSION = 2
 
 
 def db_needs_update():
@@ -88,52 +89,173 @@ def update_db():
     # Create the database tables
     eos.db.gamedata_meta.create_all()
 
-    # Config dict
+    def _readData(minerName, jsonName, keyIdName=None):
-    tables = {
+        with open(os.path.join(JSON_DIR, minerName, '{}.json'.format(jsonName)), encoding='utf-8') as f:
-        'clonegrades': ('fsd_lite', eos.gamedata.AlphaCloneSkill),
+            rawData = json.load(f)
-        'dogmaattributes': ('bulkdata', eos.gamedata.AttributeInfo),
+        if not keyIdName:
-        'dogmaeffects': ('bulkdata', eos.gamedata.Effect),
+            return rawData
-        'dogmatypeattributes': ('bulkdata', eos.gamedata.Attribute),
+        # IDs in keys, rows in values
-        'dogmatypeeffects': ('bulkdata', eos.gamedata.ItemEffect),
+        data = []
-        'dogmaunits': ('bulkdata', eos.gamedata.Unit),
+        for k, v in rawData.items():
-        'evecategories': ('fsd_lite', eos.gamedata.Category),
+            row = {}
-        'evegroups': ('fsd_lite', eos.gamedata.Group),
+            row.update(v)
-        'metagroups': ('fsd_binary', eos.gamedata.MetaGroup),
+            if keyIdName not in row:
-        'evetypes': ('fsd_lite', eos.gamedata.Item),
+                row[keyIdName] = int(k)
-        'traits': ('phobos', eos.gamedata.Traits),
+            data.append(row)
-        'metadata': ('phobos', eos.gamedata.MetaData),
+        return data
-        'marketgroups': ('fsd_binary', eos.gamedata.MarketGroup)}
 
-    fieldMapping = {
+    def _addRows(data, cls, fieldMap=None):
-        'marketgroups': {
+        if fieldMap is None:
-            'id': 'marketGroupID',
+            fieldMap = {}
-            'name': 'marketGroupName'},
+        for row in data:
-        'metagroups': {
+            instance = cls()
-            'id': 'metaGroupID'}}
+            for k, v in row.items():
+                if isinstance(v, str):
+                    v = v.strip()
+                setattr(instance, fieldMap.get(k, k), v)
+            eos.db.gamedata_session.add(instance)
 
-    rowsInValues = (
+    def _roundToPrec(val, prec):
-        'evetypes',
+        if int(val) == val:
-        'evegroups',
+            return val
-        'evecategories',
+        roundFactor = int(prec - math.floor(math.log10(abs(val))) - 1)
-        'marketgroups',
+        return round(val, roundFactor)
-        'metagroups')
 
-    def convertIcons(data):
+    def processEveTypes():
-        new = []
+        print('processing evetypes')
-        for k, v in list(data.items()):
+        data = _readData('fsd_lite', 'evetypes', keyIdName='typeID')
-            v['iconID'] = k
+        for row in data:
-            new.append(v)
+            if (
-        return new
+                # Apparently people really want Civilian modules available
+                (row['typeName'].startswith('Civilian') and "Shuttle" not in row['typeName']) or
+                row['typeName'] in ('Capsule', 'Dark Blood Tracking Disruptor')
+            ):
+                row['published'] = True
 
-    def convertClones(data):
         newData = []
+        for row in data:
+            if (
+                row['published'] or
+                # group Ship Modifiers, for items like tactical t3 ship modes
+                row['groupID'] == 1306 or
+                # Micro Bombs (Fighters)
+                row['typeID'] in (41549, 41548, 41551, 41550) or
+                # Abyssal weather (environment)
+                row['groupID'] in (
+                1882,
+                1975,
+                1971,
+                # the "container" for the abyssal environments
+                1983)
+            ):
+                newData.append(row)
 
+        _addRows(newData, eos.gamedata.Item)
+        return newData
+
+    def processEveGroups():
+        print('processing evegroups')
+        data = _readData('fsd_lite', 'evegroups', keyIdName='groupID')
+        _addRows(data, eos.gamedata.Group)
+        return data
+
+    def processEveCategories():
+        print('processing evecategories')
+        data = _readData('fsd_lite', 'evecategories', keyIdName='categoryID')
+        _addRows(data, eos.gamedata.Category)
+
+    def processDogmaAttributes():
+        print('processing dogmaattributes')
+        data = _readData('fsd_binary', 'dogmaattributes', keyIdName='attributeID')
+        _addRows(data, eos.gamedata.AttributeInfo)
+
+    def processDogmaTypeAttributes(eveTypesData):
+        print('processing dogmatypeattributes')
+        data = _readData('fsd_binary', 'typedogma', keyIdName='typeID')
+        eveTypeIds = set(r['typeID'] for r in eveTypesData)
+        newData = []
+        for row in eveTypesData:
+            for attrId, attrName in {4: 'mass', 38: 'capacity', 161: 'volume', 162: 'radius'}.items():
+                if attrName in row:
+                    newData.append({'typeID': row['typeID'], 'attributeID': attrId, 'value': row[attrName]})
+        for typeData in data:
+            if typeData['typeID'] not in eveTypeIds:
+                continue
+            for row in typeData.get('dogmaAttributes', ()):
+                row['typeID'] = typeData['typeID']
+                # As of dec 2019, CCP uses single-precision floats in their loader which results
+                # in 'ugly' numbers. Temporarily work around it by rounding here
+                row['value'] = _roundToPrec(row['value'], 7)
+                newData.append(row)
+        _addRows(newData, eos.gamedata.Attribute)
+        return newData
+
+    def processDynamicItemAttributes():
+        print('processing dynamicitemattributes')
+        data = _readData('fsd_binary', 'dynamicitemattributes')
+        for mutaID, mutaData in data.items():
+            muta = eos.gamedata.DynamicItem()
+            muta.typeID = mutaID
+            muta.resultingTypeID = mutaData['inputOutputMapping'][0]['resultingType']
+            eos.db.gamedata_session.add(muta)
+
+            for x in mutaData['inputOutputMapping'][0]['applicableTypes']:
+                item = eos.gamedata.DynamicItemItem()
+                item.typeID = mutaID
+                item.applicableTypeID = x
+                eos.db.gamedata_session.add(item)
+
+            for attrID, attrData in mutaData['attributeIDs'].items():
+                attr = eos.gamedata.DynamicItemAttribute()
+                attr.typeID = mutaID
+                attr.attributeID = attrID
+                attr.min = attrData['min']
+                attr.max = attrData['max']
+                eos.db.gamedata_session.add(attr)
+
+    def processDogmaEffects():
+        print('processing dogmaeffects')
+        data = _readData('fsd_binary', 'dogmaeffects', keyIdName='effectID')
+        _addRows(data, eos.gamedata.Effect)
+
+    def processDogmaTypeEffects(eveTypesData):
+        print('processing dogmatypeeffects')
+        data = _readData('fsd_binary', 'typedogma', keyIdName='typeID')
+        eveTypeIds = set(r['typeID'] for r in eveTypesData)
+        newData = []
+        for typeData in data:
+            if typeData['typeID'] not in eveTypeIds:
+                continue
+            for row in typeData.get('dogmaEffects', ()):
+                row['typeID'] = typeData['typeID']
+                newData.append(row)
+        _addRows(newData, eos.gamedata.ItemEffect)
+        return newData
+
+    def processDogmaUnits():
+        print('processing dogmaunits')
+        data = _readData('fsd_binary', 'dogmaunits', keyIdName='unitID')
+        _addRows(data, eos.gamedata.Unit, fieldMap={'name': 'unitName'})
+
+    def processMarketGroups():
+        print('processing marketgroups')
+        data = _readData('fsd_binary', 'marketgroups', keyIdName='marketGroupID')
+        _addRows(data, eos.gamedata.MarketGroup, fieldMap={'name': 'marketGroupName'})
+
+    def processMetaGroups():
+        print('processing metagroups')
+        data = _readData('fsd_binary', 'metagroups', keyIdName='metaGroupID')
+        _addRows(data, eos.gamedata.MetaGroup)
+
+    def processCloneGrades():
+        print('processing clonegrades')
+        data = _readData('fsd_lite', 'clonegrades')
+
+        newData = []
         # December, 2017 - CCP decided to use only one set of skill levels for alpha clones. However, this is still
         # represented in the data as a skillset per race. To ensure that all skills are the same, we store them in a way
         # that we can check to make sure all races have the same skills, as well as skill levels
-
         check = {}
-
         for ID in data:
             for skill in data[ID]['skills']:
                 newData.append({
@@ -144,18 +266,25 @@ def update_db():
                 if ID not in check:
                     check[ID] = {}
                 check[ID][int(skill['typeID'])] = int(skill['level'])
-
         if not functools.reduce(lambda a, b: a if a == b else False, [v for _, v in check.items()]):
             raise Exception('Alpha Clones not all equal')
-
         newData = [x for x in newData if x['alphaCloneID'] == 1]
-
         if len(newData) == 0:
             raise Exception('Alpha Clone processing failed')
 
-        return newData
+        tmp = []
+        for row in newData:
+            if row['alphaCloneID'] not in tmp:
+                cloneParent = eos.gamedata.AlphaClone()
+                setattr(cloneParent, 'alphaCloneID', row['alphaCloneID'])
+                setattr(cloneParent, 'alphaCloneName', row['alphaCloneName'])
+                eos.db.gamedata_session.add(cloneParent)
+                tmp.append(row['alphaCloneID'])
+        _addRows(newData, eos.gamedata.AlphaCloneSkill)
 
-    def convertTraits(data):
+    def processTraits():
+        print('processing traits')
+        data = _readData('phobos', 'traits')
 
         def convertSection(sectionData):
             sectionLines = []
@@ -182,9 +311,42 @@ def update_db():
             traitLine = '<br />\n<br />\n'.join(typeLines)
             newRow = {'typeID': typeId, 'traitText': traitLine}
             newData.append(newRow)
-        return newData
 
-    def fillReplacements(tables):
+        _addRows(newData, eos.gamedata.Traits)
+
+    def processMetadata():
+        print('processing metadata')
+        data = _readData('phobos', 'metadata')
+        _addRows(data, eos.gamedata.MetaData)
+
+    def processReqSkills(eveTypesData):
+        print('processing requiredskillsfortypes')
+
+        def composeReqSkills(raw):
+            reqSkills = {}
+            for skillTypeID, skillLevels in raw.items():
+                reqSkills[int(skillTypeID)] = skillLevels[0]
+            return reqSkills
+
+        eveTypeIds = set(r['typeID'] for r in eveTypesData)
+        data = _readData('fsd_binary', 'requiredskillsfortypes')
+        reqsByItem = {}
+        itemsByReq = {}
+        for typeID, skillreqData in data.items():
+            typeID = int(typeID)
+            if typeID not in eveTypeIds:
+                continue
+            for skillTypeID, skillLevel in composeReqSkills(skillreqData).items():
+                reqsByItem.setdefault(typeID, {})[skillTypeID] = skillLevel
+                itemsByReq.setdefault(skillTypeID, {})[typeID] = skillLevel
+        for item in eos.db.gamedata_session.query(eos.gamedata.Item).all():
+            if item.typeID in reqsByItem:
+                item.reqskills = json.dumps(reqsByItem[item.typeID])
+            if item.typeID in itemsByReq:
+                item.requiredfor = json.dumps(itemsByReq[item.typeID])
+
+    def processReplacements(eveTypesData, eveGroupsData, dogmaTypeAttributesData, dogmaTypeEffectsData):
+        print('finding item replacements')
 
         def compareAttrs(attrs1, attrs2):
             # Consider items as different if they have no attrs
@@ -196,7 +358,6 @@ def update_db():
                 return True
             return False
 
-        print('finding replacements')
         skillReqAttribs = {
             182: 277,
             183: 278,
@@ -208,18 +369,18 @@ def update_db():
         # Get data on type groups
         # Format: {type ID: group ID}
         typesGroups = {}
-        for row in tables['evetypes']:
+        for row in eveTypesData:
             typesGroups[row['typeID']] = row['groupID']
         # Get data on item effects
         # Format: {type ID: set(effect, IDs)}
         typesEffects = {}
-        for row in tables['dogmatypeeffects']:
+        for row in dogmaTypeEffectsData:
             typesEffects.setdefault(row['typeID'], set()).add(row['effectID'])
         # Get data on type attributes
         # Format: {type ID: {attribute ID: attribute value}}
         typesNormalAttribs = {}
         typesSkillAttribs = {}
-        for row in tables['dogmatypeattributes']:
+        for row in dogmaTypeAttributesData:
             attributeID = row['attributeID']
             if attributeID in skillReqAttribsFlat:
                 typeSkillAttribs = typesSkillAttribs.setdefault(row['typeID'], {})
@@ -258,13 +419,13 @@ def update_db():
                 typeSkillReqs[skillType] = skillLevel
         # Format: {group ID: category ID}
         groupCategories = {}
-        for row in tables['evegroups']:
+        for row in eveGroupsData:
             groupCategories[row['groupID']] = row['categoryID']
         # As EVE affects various types mostly depending on their group or skill requirements,
         # we're going to group various types up this way
         # Format: {(group ID, frozenset(skillreq, type, IDs), frozenset(type, effect, IDs): [type ID, {attribute ID: attribute value}]}
         groupedData = {}
-        for row in tables['evetypes']:
+        for row in eveTypesData:
             typeID = row['typeID']
             # Ignore items outside of categories we need
             if groupCategories[typesGroups[typeID]] not in (
@@ -301,134 +462,30 @@ def update_db():
                 if compareAttrs(type1[1], type2[1]):
                     replacements.setdefault(type1[0], set()).add(type2[0])
                     replacements.setdefault(type2[0], set()).add(type1[0])
-        # Put this data into types table so that normal process hooks it up
+        # Update DB session with data we generated
-        for row in tables['evetypes']:
+        for item in eos.db.gamedata_session.query(eos.gamedata.Item).all():
-            row['replacements'] = ','.join('{}'.format(tid) for tid in sorted(replacements.get(row['typeID'], ())))
+            itemReplacements = replacements.get(item.typeID)
+            if itemReplacements is not None:
+                item.replacements = ','.join('{}'.format(tid) for tid in sorted(itemReplacements))
 
-    data = {}
+    eveTypesData = processEveTypes()
+    eveGroupsData = processEveGroups()
+    processEveCategories()
+    processDogmaAttributes()
+    dogmaTypeAttributesData = processDogmaTypeAttributes(eveTypesData)
+    processDynamicItemAttributes()
+    processDogmaEffects()
+    dogmaTypeEffectsData = processDogmaTypeEffects(eveTypesData)
+    processDogmaUnits()
+    processMarketGroups()
+    processMetaGroups()
+    processCloneGrades()
+    processTraits()
+    processMetadata()
 
-    # Dump all data to memory so we can easely cross check ignored rows
+    eos.db.gamedata_session.flush()
-    for jsonName, (minerName, cls) in tables.items():
+    processReqSkills(eveTypesData)
-        with open(os.path.join(JSON_DIR, minerName, '{}.json'.format(jsonName)), encoding='utf-8') as f:
+    processReplacements(eveTypesData, eveGroupsData, dogmaTypeAttributesData, dogmaTypeEffectsData)
-            tableData = json.load(f)
-        if jsonName in rowsInValues:
-            newTableData = []
-            for k, v in tableData.items():
-                row = {}
-                row.update(v)
-                if 'id' not in row:
-                    row['id'] = int(k)
-                newTableData.append(row)
-            tableData = newTableData
-        if jsonName == 'icons':
-            tableData = convertIcons(tableData)
-        if jsonName == 'traits':
-            tableData = convertTraits(tableData)
-        if jsonName == 'clonegrades':
-            tableData = convertClones(tableData)
-        data[jsonName] = tableData
-
-    fillReplacements(data)
-
-    # Set with typeIDs which we will have in our database
-    # Sometimes CCP unpublishes some items we want to have published, we
-    # can do it here - just add them to initial set
-    eveTypes = set()
-    for row in data['evetypes']:
-        if (
-            row['published'] or
-            row['typeName'] == 'Capsule' or
-            # group Ship Modifiers, for items like tactical t3 ship modes
-            row['groupID'] == 1306 or
-            # Civilian weapons
-            (row['typeName'].startswith('Civilian') and "Shuttle" not in row['typeName']) or
-            # Micro Bombs (Fighters)
-            row['typeID'] in (41549, 41548, 41551, 41550) or
-            # Abyssal weather (environment)
-            row['groupID'] in (
-                1882,
-                1975,
-                1971,
-                # the "container" for the abyssal environments
-                1983) or
-            # Dark Blood Tracking Disruptor (drops, but rarely)
-            row['typeID'] == 32416
-        ):
-            eveTypes.add(row['typeID'])
-
-    # ignore checker
-    def isIgnored(file, row):
-        if file in ('evetypes', 'dogmatypeeffects', 'dogmatypeattributes') and row['typeID'] not in eveTypes:
-            return True
-        return False
-
-    # Loop through each json file and write it away, checking ignored rows
-    for jsonName, table in data.items():
-        fieldMap = fieldMapping.get(jsonName, {})
-        tmp = []
-
-        print('processing {}'.format(jsonName))
-
-        for row in table:
-            # We don't care about some kind of rows, filter it out if so
-            if not isIgnored(jsonName, row):
-                if (
-                    jsonName == 'evetypes' and (
-                        # Apparently people really want Civilian modules available
-                        (row['typeName'].startswith('Civilian') and "Shuttle" not in row['typeName']) or
-                        row['typeName'] in ('Capsule', 'Dark Blood Tracking Disruptor'))
-                ):
-                    row['published'] = True
-
-                instance = tables[jsonName][1]()
-                # fix for issue 80
-                if jsonName is 'icons' and 'res:/ui/texture/icons/' in str(row['iconFile']).lower():
-                    row['iconFile'] = row['iconFile'].lower().replace('res:/ui/texture/icons/', '').replace('.png', '')
-                    # with res:/ui... references, it points to the actual icon file (including it's size variation of #_size_#)
-                    # strip this info out and get the identifying info
-                    split = row['iconFile'].split('_')
-                    if len(split) == 3:
-                        row['iconFile'] = '{}_{}'.format(split[0], split[2])
-                if jsonName is 'icons' and 'modules/' in str(row['iconFile']).lower():
-                    row['iconFile'] = row['iconFile'].lower().replace('modules/', '').replace('.png', '')
-
-                if jsonName is 'clonegrades':
-                    if row['alphaCloneID'] not in tmp:
-                        cloneParent = eos.gamedata.AlphaClone()
-                        setattr(cloneParent, 'alphaCloneID', row['alphaCloneID'])
-                        setattr(cloneParent, 'alphaCloneName', row['alphaCloneName'])
-                        eos.db.gamedata_session.add(cloneParent)
-                        tmp.append(row['alphaCloneID'])
-
-                for k, v in row.items():
-                    if isinstance(v, str):
-                        v = v.strip()
-                    setattr(instance, fieldMap.get(k, k), v)
-
-                eos.db.gamedata_session.add(instance)
-
-    # quick and dirty hack to get this data in
-    with open(os.path.join(JSON_DIR, 'fsd_binary', 'dynamicitemattributes.json'), encoding='utf-8') as f:
-        bulkdata = json.load(f)
-        for mutaID, data in bulkdata.items():
-            muta = eos.gamedata.DynamicItem()
-            muta.typeID = mutaID
-            muta.resultingTypeID = data['inputOutputMapping'][0]['resultingType']
-            eos.db.gamedata_session.add(muta)
-
-            for x in data['inputOutputMapping'][0]['applicableTypes']:
-                item = eos.gamedata.DynamicItemItem()
-                item.typeID = mutaID
-                item.applicableTypeID = x
-                eos.db.gamedata_session.add(item)
-
-            for attrID, attrData in data['attributeIDs'].items():
-                attr = eos.gamedata.DynamicItemAttribute()
-                attr.typeID = mutaID
-                attr.attributeID = attrID
-                attr.min = attrData['min']
-                attr.max = attrData['max']
-                eos.db.gamedata_session.add(attr)
 
     # Add schema version to prevent further updates
     metadata_schema_version = eos.gamedata.MetaData()
@@ -436,7 +493,7 @@ def update_db():
     metadata_schema_version.field_value = GAMEDATA_SCHEMA_VERSION
     eos.db.gamedata_session.add(metadata_schema_version)
 
-    eos.db.gamedata_session.commit()
+    eos.db.gamedata_session.flush()
 
     # CCP still has 5 subsystems assigned to T3Cs, even though only 4 are available / usable. They probably have some
     # old legacy requirement or assumption that makes it difficult for them to change this value in the data. But for

eos/db/gamedata/item.py

@@ -44,7 +44,9 @@ items_table = Table("invtypes", gamedata_meta,
                     Column("metaLevel", Integer),
                     Column("metaGroupID", Integer, ForeignKey("invmetagroups.metaGroupID"), index=True),
                     Column("variationParentTypeID", Integer, ForeignKey("invtypes.typeID"), index=True),
-                    Column("replacements", String))
+                    Column("replacements", String),
+                    Column("reqskills", String),
+                    Column("requiredfor", String))
 
 from .traits import traits_table  # noqa
 

eos/db/gamedata/queries.py

@@ -424,25 +424,3 @@ def getDynamicItem(itemID, eager=None):
     except exc.NoResultFound:
         result = None
     return result
-
-
-def getRequiredFor(itemID, attrMapping):
-    Attribute1 = aliased(Attribute)
-    Attribute2 = aliased(Attribute)
-
-    skillToLevelClauses = []
-
-    for attrSkill, attrLevel in attrMapping.items():
-        skillToLevelClauses.append(and_(Attribute1.attributeID == attrSkill, Attribute2.attributeID == attrLevel))
-
-    queryOr = or_(*skillToLevelClauses)
-
-    q = select((Attribute2.typeID, Attribute2.value),
-               and_(Attribute1.value == itemID, queryOr),
-               from_obj=[
-                   join(Attribute1, Attribute2, Attribute1.typeID == Attribute2.typeID)
-               ])
-
-    result = gamedata_session.execute(q).fetchall()
-
-    return result

eos/gamedata.py

@@ -18,6 +18,7 @@
 # ===============================================================================
 
 
+import json
 from collections import OrderedDict
 
 from logbook import Logger
@@ -314,50 +315,26 @@ class Item(EqBase):
         eos.db.saveddata_session.delete(override)
         eos.db.commit()
 
-    srqIDMap = {182: 277, 183: 278, 184: 279, 1285: 1286, 1289: 1287, 1290: 1288}
-
     @property
     def requiredSkills(self):
         if self.__requiredSkills is None:
-            requiredSkills = OrderedDict()
+            self.__requiredSkills = {}
-            self.__requiredSkills = requiredSkills
+            if self.reqskills:
-            # Map containing attribute IDs we may need for required skills
+                for skillTypeID, skillLevel in json.loads(self.reqskills).items():
-            # { requiredSkillX : requiredSkillXLevel }
+                    skillItem = eos.db.getItem(int(skillTypeID))
-            combinedAttrIDs = set(self.srqIDMap.keys()).union(set(self.srqIDMap.values()))
+                    if skillItem:
-            # Map containing result of the request
+                        self.__requiredSkills[skillItem] = skillLevel
-            # { attributeID : attributeValue }
-            skillAttrs = {}
-            # Get relevant attribute values from db (required skill IDs and levels) for our item
-            for attrInfo in eos.db.directAttributeRequest((self.ID,), tuple(combinedAttrIDs)):
-                attrID = attrInfo[1]
-                attrVal = attrInfo[2]
-                skillAttrs[attrID] = attrVal
-            # Go through all attributeID pairs
-            for srqIDAtrr, srqLvlAttr in self.srqIDMap.items():
-                # Check if we have both in returned result
-                if srqIDAtrr in skillAttrs and srqLvlAttr in skillAttrs:
-                    skillID = int(skillAttrs[srqIDAtrr])
-                    skillLvl = skillAttrs[srqLvlAttr]
-                    # Fetch item from database and fill map
-                    item = eos.db.getItem(skillID)
-                    requiredSkills[item] = skillLvl
         return self.__requiredSkills
 
     @property
     def requiredFor(self):
         if self.__requiredFor is None:
-            self.__requiredFor = dict()
+            self.__requiredFor = {}
-
+            if self.requiredfor:
-            # Map containing attribute IDs we may need for required skills
+                for typeID, skillLevel in json.loads(self.requiredfor).items():
-
+                    requiredForItem = eos.db.getItem(int(typeID))
-            # Get relevant attribute values from db (required skill IDs and levels) for our item
+                    if requiredForItem:
-            q = eos.db.getRequiredFor(self.ID, self.srqIDMap)
+                        self.__requiredFor[requiredForItem] = skillLevel
-
-            for itemID, lvl in q:
-                # Fetch item from database and fill map
-                item = eos.db.getItem(itemID)
-                self.__requiredFor[item] = lvl
-
         return self.__requiredFor
 
     factionMap = {

eos/saveddata/fit.py

@@ -165,6 +165,9 @@ class Fit:
         self.__capUsed = None
         self.__capRecharge = None
         self.__savedCapSimData.clear()
+        # Ancillary tank modules affect this
+        self.__sustainableTank = None
+        self.__effectiveSustainableTank = None
 
     @property
     def targetProfile(self):

gui/builtinViewColumns/misc.py

@@ -590,7 +590,7 @@ class Miscellanea(ViewColumn):
         ):
             if "Armor" in itemGroup or "Shield" in itemGroup:
                 boosted_attribute = "HP"
-                reload_time = item.getAttribute("reloadTime", 0) / 1000
+                reload_time = stuff.getModifiedItemAttr("reloadTime", 0) / 1000
             elif "Capacitor" in itemGroup:
                 boosted_attribute = "Cap"
                 reload_time = 10

service/market.py

@@ -804,7 +804,7 @@ class Market:
     def getReplacements(self, identity):
         item = self.getItem(identity)
         # We already store needed type IDs in database
-        replTypeIDs = {int(i) for i in item.replacements.split(",") if i}
+        replTypeIDs = {int(i) for i in item.replacements.split(",") if i} if item.replacements is not None else {}
         if not replTypeIDs:
             return ()
         # As replacements were generated without keeping track which items were published,

staticdata/bulkdata/dogmaunits.json

@@ -1,524 +0,0 @@
-[
-  {
-    "dataID": 16545434, 
-    "description": "Meter", 
-    "descriptionID": 77965, 
-    "displayName": "m", 
-    "displayNameID": 78005, 
-    "unitID": 1, 
-    "unitName": "Length"
-  }, 
-  {
-    "dataID": 16545435, 
-    "description": "Kilogram", 
-    "descriptionID": 77966, 
-    "displayName": "kg", 
-    "displayNameID": 78006, 
-    "unitID": 2, 
-    "unitName": "Mass"
-  }, 
-  {
-    "dataID": 16545436, 
-    "description": "Second", 
-    "descriptionID": 77967, 
-    "displayName": "sec", 
-    "displayNameID": 78007, 
-    "unitID": 3, 
-    "unitName": "Time"
-  }, 
-  {
-    "dataID": 16545437, 
-    "description": "Ampere", 
-    "descriptionID": 77968, 
-    "displayName": "A", 
-    "displayNameID": 78008, 
-    "unitID": 4, 
-    "unitName": "Electric Current"
-  }, 
-  {
-    "dataID": 16545438, 
-    "description": "Kelvin", 
-    "descriptionID": 77969, 
-    "displayName": "K", 
-    "displayNameID": 78009, 
-    "unitID": 5, 
-    "unitName": "Temperature"
-  }, 
-  {
-    "dataID": 16545439, 
-    "description": "Mole", 
-    "descriptionID": 77970, 
-    "displayName": "mol", 
-    "displayNameID": 78010, 
-    "unitID": 6, 
-    "unitName": "Amount Of Substance"
-  }, 
-  {
-    "dataID": 16545440, 
-    "description": "Candela", 
-    "descriptionID": 77971, 
-    "displayName": "cd", 
-    "displayNameID": 78011, 
-    "unitID": 7, 
-    "unitName": "Luminous Intensity"
-  }, 
-  {
-    "dataID": 16545441, 
-    "description": "Square meter", 
-    "descriptionID": 77972, 
-    "displayName": "m2", 
-    "displayNameID": 78012, 
-    "unitID": 8, 
-    "unitName": "Area"
-  }, 
-  {
-    "dataID": 16545442, 
-    "description": "Cubic meter", 
-    "descriptionID": 77973, 
-    "displayName": "m3", 
-    "displayNameID": 78013, 
-    "unitID": 9, 
-    "unitName": "Volume"
-  }, 
-  {
-    "dataID": 16545443, 
-    "description": "Meter per second", 
-    "descriptionID": 77974, 
-    "displayName": "m/sec", 
-    "displayNameID": 78014, 
-    "unitID": 10, 
-    "unitName": "Speed"
-  }, 
-  {
-    "dataID": 16545444, 
-    "description": "Meter per second squared", 
-    "descriptionID": 77975, 
-    "displayName": "m/sec", 
-    "displayNameID": 78015, 
-    "unitID": 11, 
-    "unitName": "Acceleration"
-  }, 
-  {
-    "dataID": 16545445, 
-    "description": "Reciprocal meter", 
-    "descriptionID": 77976, 
-    "displayName": "m-1", 
-    "displayNameID": 78016, 
-    "unitID": 12, 
-    "unitName": "Wave Number"
-  }, 
-  {
-    "dataID": 16545446, 
-    "description": "Kilogram per cubic meter", 
-    "descriptionID": 77977, 
-    "displayName": "kg/m3", 
-    "displayNameID": 78017, 
-    "unitID": 13, 
-    "unitName": "Mass Density"
-  }, 
-  {
-    "dataID": 16545447, 
-    "description": "Cubic meter per kilogram", 
-    "descriptionID": 77978, 
-    "displayName": "m3/kg", 
-    "displayNameID": 78018, 
-    "unitID": 14, 
-    "unitName": "Specific Volume"
-  }, 
-  {
-    "dataID": 16545448, 
-    "description": "Ampere per square meter", 
-    "descriptionID": 77979, 
-    "displayName": "A/m2", 
-    "displayNameID": 78019, 
-    "unitID": 15, 
-    "unitName": "Current Density"
-  }, 
-  {
-    "dataID": 16545449, 
-    "description": "Ampere per meter", 
-    "descriptionID": 77980, 
-    "displayName": "A/m", 
-    "displayNameID": 78020, 
-    "unitID": 16, 
-    "unitName": "Magnetic Field Strength"
-  }, 
-  {
-    "dataID": 16545450, 
-    "description": "Mole per cubic meter", 
-    "descriptionID": 77981, 
-    "displayName": "mol/m3", 
-    "displayNameID": 78021, 
-    "unitID": 17, 
-    "unitName": "Amount-Of-Substance Concentration"
-  }, 
-  {
-    "dataID": 16545451, 
-    "description": "Candela per square meter", 
-    "descriptionID": 77982, 
-    "displayName": "cd/m2", 
-    "displayNameID": 78022, 
-    "unitID": 18, 
-    "unitName": "Luminance"
-  }, 
-  {
-    "dataID": 16545452, 
-    "description": "Kilogram per kilogram, which may be represented by the number 1", 
-    "descriptionID": 77983, 
-    "displayName": "kg/kg = 1", 
-    "displayNameID": 78023, 
-    "unitID": 19, 
-    "unitName": "Mass Fraction"
-  }, 
-  {
-    "dataID": 16545453, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "s", 
-    "displayNameID": 78024, 
-    "unitID": 101, 
-    "unitName": "Milliseconds"
-  }, 
-  {
-    "dataID": 16545454, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "mm", 
-    "displayNameID": 78025, 
-    "unitID": 102, 
-    "unitName": "Millimeters"
-  }, 
-  {
-    "dataID": 13353825, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "", 
-    "displayNameID": null, 
-    "unitID": 103, 
-    "unitName": "MegaPascals"
-  }, 
-  {
-    "dataID": 16545455, 
-    "description": "Indicates that the unit is a multiplier.", 
-    "descriptionID": 77984, 
-    "displayName": "x", 
-    "displayNameID": 78026, 
-    "unitID": 104, 
-    "unitName": "Multiplier"
-  }, 
-  {
-    "dataID": 16545456, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "%", 
-    "displayNameID": 78027, 
-    "unitID": 105, 
-    "unitName": "Percentage"
-  }, 
-  {
-    "dataID": 16545457, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "tf", 
-    "displayNameID": 78028, 
-    "unitID": 106, 
-    "unitName": "Teraflops"
-  }, 
-  {
-    "dataID": 16545458, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "MW", 
-    "displayNameID": 78029, 
-    "unitID": 107, 
-    "unitName": "MegaWatts"
-  }, 
-  {
-    "dataID": 16545459, 
-    "description": "Used for resistance.\r\n0.0 = 100% 1.0 = 0%\r\n", 
-    "descriptionID": 77985, 
-    "displayName": "%", 
-    "displayNameID": 78030, 
-    "unitID": 108, 
-    "unitName": "Inverse Absolute Percent"
-  }, 
-  {
-    "dataID": 16545460, 
-    "description": "Used for multipliers displayed as %\r\n1.1 = +10%\r\n0.9 = -10%", 
-    "descriptionID": 77986, 
-    "displayName": "%", 
-    "displayNameID": 78031, 
-    "unitID": 109, 
-    "unitName": "Modifier Percent"
-  }, 
-  {
-    "dataID": 16545461, 
-    "description": "Used to modify damage resistance. Damage resistance bonus.\r\n0.1 = 90%\r\n0.9 = 10%", 
-    "descriptionID": 77987, 
-    "displayName": "%", 
-    "displayNameID": 78032, 
-    "unitID": 111, 
-    "unitName": "Inversed Modifier Percent"
-  }, 
-  {
-    "dataID": 16545462, 
-    "description": "Rotation speed.", 
-    "descriptionID": 77988, 
-    "displayName": "rad/sec", 
-    "displayNameID": 78033, 
-    "unitID": 112, 
-    "unitName": "Radians/Second"
-  }, 
-  {
-    "dataID": 16545463, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "HP", 
-    "displayNameID": 78034, 
-    "unitID": 113, 
-    "unitName": "Hitpoints"
-  }, 
-  {
-    "dataID": 16545464, 
-    "description": "Giga Joule", 
-    "descriptionID": 77989, 
-    "displayName": "GJ", 
-    "displayNameID": 78035, 
-    "unitID": 114, 
-    "unitName": "capacitor units"
-  }, 
-  {
-    "dataID": 16545465, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "groupID", 
-    "displayNameID": 78036, 
-    "unitID": 115, 
-    "unitName": "groupID"
-  }, 
-  {
-    "dataID": 16545466, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "typeID", 
-    "displayNameID": 78037, 
-    "unitID": 116, 
-    "unitName": "typeID"
-  }, 
-  {
-    "dataID": 100671817, 
-    "description": "1=small 2=medium 3=large 4=x-large", 
-    "descriptionID": 318074, 
-    "displayName": "1=small 2=medium 3=l", 
-    "displayNameID": 78038, 
-    "unitID": 117, 
-    "unitName": "Sizeclass"
-  }, 
-  {
-    "dataID": 16545468, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "Ore units", 
-    "displayNameID": 78039, 
-    "unitID": 118, 
-    "unitName": "Ore units"
-  }, 
-  {
-    "dataID": 16545469, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "attributeID", 
-    "displayNameID": 78040, 
-    "unitID": 119, 
-    "unitName": "attributeID"
-  }, 
-  {
-    "dataID": 16545470, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "points", 
-    "displayNameID": 78041, 
-    "unitID": 120, 
-    "unitName": "attributePoints"
-  }, 
-  {
-    "dataID": 16545471, 
-    "description": "Used for real percentages, i.e. the number 5 is 5%", 
-    "descriptionID": 77990, 
-    "displayName": "%", 
-    "displayNameID": 78042, 
-    "unitID": 121, 
-    "unitName": "realPercent"
-  }, 
-  {
-    "dataID": 13353933, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "", 
-    "displayNameID": null, 
-    "unitID": 122, 
-    "unitName": "Fitting slots"
-  }, 
-  {
-    "dataID": 16545472, 
-    "description": "Shows seconds directly", 
-    "descriptionID": 77991, 
-    "displayName": "sec", 
-    "displayNameID": 78043, 
-    "unitID": 123, 
-    "unitName": "trueTime"
-  }, 
-  {
-    "dataID": 16545473, 
-    "description": "Used for relative percentages displayed as %", 
-    "descriptionID": 77992, 
-    "displayName": "%", 
-    "displayNameID": 78044, 
-    "unitID": 124, 
-    "unitName": "Modifier Relative Percent"
-  }, 
-  {
-    "dataID": 16545474, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "N", 
-    "displayNameID": 78045, 
-    "unitID": 125, 
-    "unitName": "Newton"
-  }, 
-  {
-    "dataID": 16545475, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "ly", 
-    "displayNameID": 78046, 
-    "unitID": 126, 
-    "unitName": "Light Year"
-  }, 
-  {
-    "dataID": 16545476, 
-    "description": "0.0 = 0% 1.0 = 100%", 
-    "descriptionID": 77993, 
-    "displayName": "%", 
-    "displayNameID": 78047, 
-    "unitID": 127, 
-    "unitName": "Absolute Percent"
-  }, 
-  {
-    "dataID": 16545477, 
-    "description": "Mega bits per second", 
-    "descriptionID": 77994, 
-    "displayName": "Mbit/sec", 
-    "displayNameID": 78048, 
-    "unitID": 128, 
-    "unitName": "Drone bandwidth"
-  }, 
-  {
-    "dataID": 16545488, 
-    "description": "Hours", 
-    "descriptionID": 77995, 
-    "displayName": "", 
-    "displayNameID": null, 
-    "unitID": 129, 
-    "unitName": "Hours"
-  }, 
-  {
-    "dataID": 16545478, 
-    "description": "ISK", 
-    "descriptionID": 77996, 
-    "displayName": "ISK", 
-    "displayNameID": 78049, 
-    "unitID": 133, 
-    "unitName": "Money"
-  }, 
-  {
-    "dataID": 16545479, 
-    "description": "Bandwidth for PI", 
-    "descriptionID": 77997, 
-    "displayName": "m3/hour", 
-    "displayNameID": 78050, 
-    "unitID": 134, 
-    "unitName": "Logistical Capacity"
-  }, 
-  {
-    "dataID": 16545480, 
-    "description": "Used to denote distance, 1AU = The distance from the Earth to the Sun.", 
-    "descriptionID": 77998, 
-    "displayName": "AU", 
-    "displayNameID": 78051, 
-    "unitID": 135, 
-    "unitName": "Astronomical Unit"
-  }, 
-  {
-    "dataID": 16545481, 
-    "description": "Slot number prefix for various purposes", 
-    "descriptionID": 77999, 
-    "displayName": "Slot", 
-    "displayNameID": 78052, 
-    "unitID": 136, 
-    "unitName": "Slot"
-  }, 
-  {
-    "dataID": 16545482, 
-    "description": "For displaying boolean flags 1=True 0=False", 
-    "descriptionID": 78000, 
-    "displayName": "1=True 0=False", 
-    "displayNameID": 78053, 
-    "unitID": 137, 
-    "unitName": "Boolean"
-  }, 
-  {
-    "dataID": 16545483, 
-    "description": "Units of something, for example fuel", 
-    "descriptionID": 78001, 
-    "displayName": "units", 
-    "displayNameID": 78054, 
-    "unitID": 138, 
-    "unitName": "Units"
-  }, 
-  {
-    "dataID": 16545484, 
-    "description": "Forces a plus sign for positive values", 
-    "descriptionID": 78002, 
-    "displayName": "+", 
-    "displayNameID": 78055, 
-    "unitID": 139, 
-    "unitName": "Bonus"
-  }, 
-  {
-    "dataID": 16545485, 
-    "description": "For anything which is divided by levels", 
-    "descriptionID": 78003, 
-    "displayName": "Level", 
-    "displayNameID": 78056, 
-    "unitID": 140, 
-    "unitName": "Level"
-  }, 
-  {
-    "dataID": 16545486, 
-    "description": "For various counts to do with turret, launcher and rig hardpoints", 
-    "descriptionID": 78004, 
-    "displayName": "hardpoints", 
-    "displayNameID": 78057, 
-    "unitID": 141, 
-    "unitName": "Hardpoints"
-  }, 
-  {
-    "dataID": 16545487, 
-    "description": "", 
-    "descriptionID": null, 
-    "displayName": "1=Male 2=Unisex 3=Female", 
-    "displayNameID": 78058, 
-    "unitID": 142, 
-    "unitName": "Sex"
-  }, 
-  {
-    "dataID": 97574714, 
-    "description": "Date and time", 
-    "descriptionID": 312106, 
-    "displayName": "", 
-    "displayNameID": null, 
-    "unitID": 143, 
-    "unitName": "Datetime"
-  }
-]

staticdata/fsd_binary/marketgroups.json

@@ -64,7 +64,7 @@
     "descriptionID": 64453, 
     "hasTypes": 0, 
     "iconID": 365, 
-    "name": "Turrets & Bays", 
+    "name": "Turrets & Launchers", 
     "nameID": 65527, 
     "parentGroupID": 9
   }, 
@@ -605,7 +605,7 @@
     "nameID": 65587
   }, 
   "157": {
-    "description": "Drones are semi-autonomous robotic devices used for military and industrial purposes throughout space.", 
+    "description": "Drones are semi-autonomous robotic devices used for military and industrial purposes throughout space", 
     "descriptionID": 64514, 
     "hasTypes": 0, 
     "iconID": 1084, 
@@ -16002,7 +16002,7 @@
     "description": "Capital ship entropic disintegrators, for use on dreadnoughts and titans.", 
     "descriptionID": 553053, 
     "hasTypes": 1, 
-    "iconID": 21921, 
+    "iconID": 24237, 
     "name": "Extra Large", 
     "nameID": 553052, 
     "parentGroupID": 2432
@@ -16022,5 +16022,23 @@
     "name": "Triglavian", 
     "nameID": 553059, 
     "parentGroupID": 1883
+  }, 
+  "2701": {
+    "description": "Items used in the HyperNet Relay", 
+    "descriptionID": 554068, 
+    "hasTypes": 1, 
+    "iconID": 24205, 
+    "name": "HyperNet Relay", 
+    "nameID": 554066, 
+    "parentGroupID": 1922
+  }, 
+  "2702": {
+    "description": "Corvettes that have been offered to capsuleers on occasion for limited periods.", 
+    "descriptionID": 554136, 
+    "hasTypes": 1, 
+    "iconID": 1443, 
+    "name": "Special Edition Corvettes", 
+    "nameID": 554135, 
+    "parentGroupID": 1612
   }
 }

staticdata/fsd_lite/evegroups.json

@@ -9544,6 +9544,17 @@
     "published": true, 
     "useBasePrice": false
   }, 
+  "4041": {
+    "anchorable": false, 
+    "anchored": false, 
+    "categoryID": 17, 
+    "fittableNonSingleton": false, 
+    "groupID": 4041, 
+    "groupName": "Jump Filaments", 
+    "groupNameID": 553695, 
+    "published": true, 
+    "useBasePrice": false
+  }, 
   "405": {
     "anchorable": false, 
     "anchored": false, 

staticdata/phobos/metadata.json

@@ -1,10 +1,10 @@
 [
   {
     "field_name": "client_build", 
-    "field_value": 1618828
+    "field_value": 1635499
   }, 
   {
     "field_name": "dump_time", 
-    "field_value": 1574856401
+    "field_value": 1576677195
   }
 ]

staticdata/phobos/traits.json

@@ -3945,11 +3945,11 @@
           "bonuses": [
             {
               "number": "5%", 
-              "text": "bonus to Light Missile and Rocket Launcher explosion velocity"
+              "text": "bonus to Light Missile and Rocket explosion velocity"
             }, 
             {
               "number": "5%", 
-              "text": "bonus to Light Missile and Rocket Launcher damage"
+              "text": "bonus to Light Missile and Rocket damage"
             }
           ], 
           "header": "Assault Frigates bonuses (per skill level):"
@@ -17728,7 +17728,7 @@
               "text": "bonus to Ultratidal Entropic Disintegrator rate of fire"
             }
           ], 
-          "header": "Zirnitra bonuses (per skill level):"
+          "header": "Precursor Dreadnought bonuses (per skill level):"
         }
       ]
     },