Performance improvement when collecting counts for categories. Instead of making n calls for each category to sql, we get a grouped count by shipID, then process this in python, which is much faster.

eos/config.py

@@ -19,7 +19,7 @@ if istravis is True or hasattr(sys, '_called_from_test'):
     # Running in Travis. Run saveddata database in memory.
     saveddata_connectionstring = 'sqlite:///:memory:'
 else:
-    saveddata_connectionstring = 'sqlite:///' + realpath(join(dirname(abspath(__file__)), "..", "saveddata", "saveddata-py3-db.db"))
+    saveddata_connectionstring = 'sqlite:///' + realpath(join(dirname(abspath(__file__)), "..", "saveddata", "saveddata.db"))
 
 pyfalog.debug("Saveddata connection string: {0}", saveddata_connectionstring)
 

eos/db/saveddata/queries.py

@@ -21,6 +21,7 @@ import sys
 
 from sqlalchemy.sql import and_
 from sqlalchemy import desc, select
+from sqlalchemy import func
 
 from eos.db import saveddata_session, sd_lock
 from eos.db.saveddata.fit import projectedFits_table
@@ -283,6 +284,12 @@ def countAllFits():
     return count
 
 
+def countFitGroupedByShip():
+    with sd_lock:
+        count = eos.db.saveddata_session.query(Fit.shipID, func.count(Fit.shipID)).group_by(Fit.shipID).all()
+    return count
+
+
 def countFitsWithShip(lookfor, ownerID=None, where=None, eager=None):
     """
     Get all the fits using a certain ship.

gui/shipBrowser.py

@@ -164,11 +164,12 @@ class ShipBrowser(wx.Panel):
             self.categoryList = list(sMkt.getShipRoot())
             self.categoryList.sort(key=lambda _ship: _ship.name)
 
+            counts = sFit.countAllFitsGroupedByShip()
+
             # set map & cache of fittings per category
             for cat in self.categoryList:
                 itemIDs = [x.ID for x in cat.items]
-                num = sFit.countFitsWithShip(itemIDs)
-                self.categoryFitCache[cat.ID] = num > 0
+                self.categoryFitCache[cat.ID] = sum([count for shipID, count in counts if shipID in itemIDs]) > 0
 
         for ship in self.categoryList:
             if self.filterShipsWithNoFits and not self.categoryFitCache[ship.ID]:

service/fit.py

@@ -146,6 +146,11 @@ class Fit(FitDeprecated):
         pyfalog.debug("Getting count of all fits.")
         return eos.db.countAllFits()
 
+    @staticmethod
+    def countAllFitsGroupedByShip():
+        count = eos.db.countFitGroupedByShip()
+        return count
+
     @staticmethod
     def countFitsWithShip(stuff):
         pyfalog.debug("Getting count of all fits for: {0}", stuff)