From e6b4ecdaf701fb85920399b644eb88292057c08c Mon Sep 17 00:00:00 2001 From: Ebag333 Date: Fri, 27 Jan 2017 13:37:51 -0800 Subject: [PATCH 1/3] Additional cleanup for orphaned records --- eos/db/saveddata/databaseRepair.py | 39 ++++++++++++++++++++++++++++++ service/prefetch.py | 2 ++ 2 files changed, 41 insertions(+) diff --git a/eos/db/saveddata/databaseRepair.py b/eos/db/saveddata/databaseRepair.py index e9685381a..53f6ae29f 100644 --- a/eos/db/saveddata/databaseRepair.py +++ b/eos/db/saveddata/databaseRepair.py @@ -160,3 +160,42 @@ class DatabaseCleanup: query = "UPDATE 'targetResists' SET 'name' = 'Unknown' WHERE name IS NULL OR name = ''" delete = DatabaseCleanup.ExecuteSQLQuery(saveddata_engine, query) logger.error("Database corruption found. Cleaning up %d records.", delete.rowcount) + + @staticmethod + def OrphanedFitIDItemID(saveddata_engine): + # Orphaned items that are missing the fit ID or item ID. + # See issue #954 + for table in ['drones', 'cargo', 'fighters', 'modules']: + logger.debug("Running database cleanup for orphaned %s items.", table) + query = "SELECT COUNT(*) AS num FROM " + table + " WHERE itemID IS NULL OR itemID = '' or itemID = '0' or fitID IS NULL OR fitID = '' or fitID = '0'" + results = DatabaseCleanup.ExecuteSQLQuery(saveddata_engine, query) + + if results is None: + return + + row = results.first() + + if row and row['num']: + query = "DELETE FROM " + table + " WHERE itemID IS NULL OR itemID = '' or itemID = '0' or fitID IS NULL OR fitID = '' or fitID = '0'" + delete = DatabaseCleanup.ExecuteSQLQuery(saveddata_engine, query) + logger.error("Database corruption found. Cleaning up %d records.", delete.rowcount) + + @staticmethod + def NullDamageTargetPatternValues(saveddata_engine): + # Find patterns that have null values + # See issue #954 + for profileType in ['damagePatterns', 'targetResists']: + for damageType in ['em', 'thermal', 'kinetic', 'explosive']: + logger.debug("Running database cleanup for null %s values. (%s)", profileType, damageType) + query = "SELECT COUNT(*) AS num FROM " + profileType + " WHERE " + damageType + "Amount IS NULL OR " + damageType + "Amount = ''" + results = DatabaseCleanup.ExecuteSQLQuery(saveddata_engine, query) + + if results is None: + return + + row = results.first() + + if row and row['num']: + query = "UPDATE '" + profileType + "' SET '" + damageType + "Amount' = '0' WHERE " + damageType + "Amount IS NULL OR emAmount = ''" + delete = DatabaseCleanup.ExecuteSQLQuery(saveddata_engine, query) + logger.error("Database corruption found. Cleaning up %d records.", delete.rowcount) diff --git a/service/prefetch.py b/service/prefetch.py index 46f9cf5f5..9990fea10 100644 --- a/service/prefetch.py +++ b/service/prefetch.py @@ -67,6 +67,8 @@ if os.path.isfile(config.saveDB): database_cleanup_instance.OrphanedCharacterSkills(eos.db.saveddata_engine) database_cleanup_instance.OrphanedFitCharacterIDs(eos.db.saveddata_engine) database_cleanup_instance.OrphanedFitDamagePatterns(eos.db.saveddata_engine) + database_cleanup_instance.OrphanedFitIDItemID(eos.db.saveddata_engine) + database_cleanup_instance.NullDamageTargetPatternValues(eos.db.saveddata_engine) logging.debug("Completed database validation.") else: From fc99d6f0a6be36de72dbb0743655cf23a3bd2057 Mon Sep 17 00:00:00 2001 From: Ebag333 Date: Fri, 27 Jan 2017 13:46:20 -0800 Subject: [PATCH 2/3] catch duplicate selected ammo --- eos/db/saveddata/databaseRepair.py | 18 ++++++++++++++++++ service/prefetch.py | 1 + 2 files changed, 19 insertions(+) diff --git a/eos/db/saveddata/databaseRepair.py b/eos/db/saveddata/databaseRepair.py index 53f6ae29f..074730fe0 100644 --- a/eos/db/saveddata/databaseRepair.py +++ b/eos/db/saveddata/databaseRepair.py @@ -199,3 +199,21 @@ class DatabaseCleanup: query = "UPDATE '" + profileType + "' SET '" + damageType + "Amount' = '0' WHERE " + damageType + "Amount IS NULL OR emAmount = ''" delete = DatabaseCleanup.ExecuteSQLQuery(saveddata_engine, query) logger.error("Database corruption found. Cleaning up %d records.", delete.rowcount) + + @staticmethod + def DuplicateSelectedAmmoName(saveddata_engine): + # Orphaned items that are missing the fit ID or item ID. + # See issue #954 + logger.debug("Running database cleanup for duplicated selected ammo profiles.") + query = "SELECT COUNT(*) AS num FROM damagePatterns WHERE name = 'Selected Ammo'" + results = DatabaseCleanup.ExecuteSQLQuery(saveddata_engine, query) + + if results is None: + return + + row = results.first() + + if row and row['num'] > 1: + query = "DELETE FROM damagePatterns WHERE name = 'Selected Ammo'" + delete = DatabaseCleanup.ExecuteSQLQuery(saveddata_engine, query) + logger.error("Database corruption found. Cleaning up %d records.", delete.rowcount) diff --git a/service/prefetch.py b/service/prefetch.py index 9990fea10..f04220b3c 100644 --- a/service/prefetch.py +++ b/service/prefetch.py @@ -69,6 +69,7 @@ if os.path.isfile(config.saveDB): database_cleanup_instance.OrphanedFitDamagePatterns(eos.db.saveddata_engine) database_cleanup_instance.OrphanedFitIDItemID(eos.db.saveddata_engine) database_cleanup_instance.NullDamageTargetPatternValues(eos.db.saveddata_engine) + database_cleanup_instance.DuplicateSelectedAmmoName(eos.db.saveddata_engine) logging.debug("Completed database validation.") else: From 200023234cf75081d3f12859963867d15ea49848 Mon Sep 17 00:00:00 2001 From: Ebag333 Date: Fri, 27 Jan 2017 14:49:30 -0800 Subject: [PATCH 3/3] Give modules the special snowflake experience Modules can legitimately have a null item value. So don't catch null or '', only 0's (which would be an invalid item ID). --- eos/db/saveddata/databaseRepair.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/eos/db/saveddata/databaseRepair.py b/eos/db/saveddata/databaseRepair.py index 074730fe0..3a48cc40a 100644 --- a/eos/db/saveddata/databaseRepair.py +++ b/eos/db/saveddata/databaseRepair.py @@ -165,7 +165,7 @@ class DatabaseCleanup: def OrphanedFitIDItemID(saveddata_engine): # Orphaned items that are missing the fit ID or item ID. # See issue #954 - for table in ['drones', 'cargo', 'fighters', 'modules']: + for table in ['drones', 'cargo', 'fighters']: logger.debug("Running database cleanup for orphaned %s items.", table) query = "SELECT COUNT(*) AS num FROM " + table + " WHERE itemID IS NULL OR itemID = '' or itemID = '0' or fitID IS NULL OR fitID = '' or fitID = '0'" results = DatabaseCleanup.ExecuteSQLQuery(saveddata_engine, query) @@ -180,6 +180,21 @@ class DatabaseCleanup: delete = DatabaseCleanup.ExecuteSQLQuery(saveddata_engine, query) logger.error("Database corruption found. Cleaning up %d records.", delete.rowcount) + for table in ['modules']: + logger.debug("Running database cleanup for orphaned %s items.", table) + query = "SELECT COUNT(*) AS num FROM " + table + " WHERE itemID = '0' or fitID IS NULL OR fitID = '' or fitID = '0'" + results = DatabaseCleanup.ExecuteSQLQuery(saveddata_engine, query) + + if results is None: + return + + row = results.first() + + if row and row['num']: + query = "DELETE FROM " + table + " WHERE itemID = '0' or fitID IS NULL OR fitID = '' or fitID = '0'" + delete = DatabaseCleanup.ExecuteSQLQuery(saveddata_engine, query) + logger.error("Database corruption found. Cleaning up %d records.", delete.rowcount) + @staticmethod def NullDamageTargetPatternValues(saveddata_engine): # Find patterns that have null values