From cef58423441ca45ae2f2209d1fbe68b65a5c7b23 Mon Sep 17 00:00:00 2001 From: Ebag333 Date: Mon, 26 Dec 2016 21:54:52 -0800 Subject: [PATCH] Database validation and cleanup --- eos/db/saveddata/databaseRepair.py | 122 +++++++++++++++++++++++++++++ service/prefetch.py | 14 +++- 2 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 eos/db/saveddata/databaseRepair.py diff --git a/eos/db/saveddata/databaseRepair.py b/eos/db/saveddata/databaseRepair.py new file mode 100644 index 000000000..be26c8231 --- /dev/null +++ b/eos/db/saveddata/databaseRepair.py @@ -0,0 +1,122 @@ +# =============================================================================== +# Copyright (C) 2010 Diego Duclos +# +# This file is part of pyfa. +# +# pyfa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyfa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyfa. If not, see . +# =============================================================================== + +import sqlalchemy +import logging + +logger = logging.getLogger(__name__) + + +class DatabaseCleanup: + def __init__(self): + pass + + @staticmethod + def OrphanedCharacterSkills(saveddata_engine): + # Finds and fixes database corruption issues. + logger.debug("Start databsae validation and cleanup.") + + # Find orphaned character skills. + # This solves an issue where the character doesn't exist, but skills for that character do. + # See issue #917 + try: + logger.debug("Running database cleanup for character skills.") + results = saveddata_engine.execute("SELECT * FROM characterSkills " + "WHERE characterID NOT IN (SELECT ID from characters)") + + # Count how many records exist. This is ugly, but SQLAlchemy doesn't return a count from a select query. + result_count = 0 + for _ in results: + result_count += 1 + + if result_count > 0: + logger.error("Database corruption found. Cleaning up %d records.", result_count) + saveddata_engine.execute("DELETE FROM characterSkills WHERE characterID NOT IN (SELECT ID from characters)") + except sqlalchemy.exc.DatabaseError: + logger.error("Failed to connect to database.") + + @staticmethod + def OrphanedFitDamagePatterns(saveddata_engine): + # Find orphaned damage patterns. + # This solves an issue where the damage pattern doesn't exist, but fits reference the pattern. + # See issue #777 + try: + logger.debug("Running database cleanup for orphaned damage patterns attached to fits.") + results = saveddata_engine.execute("SELECT * FROM fits WHERE damagePatternID not in (select ID from damagePatterns)") + + # Count how many records exist. This is ugly, but SQLAlchemy doesn't return a count from a select query. + result_count = 0 + for _ in results: + result_count += 1 + + if result_count > 0: + # Get Uniform damage pattern ID + uniform_results = saveddata_engine.execute("select ID from damagePatterns WHERE name = 'Uniform'") + + uniform_result_count = 0 + uniform_damage_pattern_id = 0 + for uniform_result in uniform_results: + uniform_damage_pattern_id = uniform_result[0] + uniform_result_count += 1 + + if uniform_result_count == 0: + logger.error("Missing uniform damage pattern.") + elif uniform_result_count > 1: + logger.error("More than one uniform damage pattern found.") + else: + logger.error("Database corruption found. Cleaning up %d records.", result_count) + saveddata_engine.execute("UPDATE 'fits' SET 'damagePatternID' = ? " + "WHERE damagePatternID NOT IN (SELECT ID FROM damagePatterns)", + uniform_damage_pattern_id) + except sqlalchemy.exc.DatabaseError: + logger.error("Failed to connect to database.") + + @staticmethod + def OrphanedFitCharacterIDs(saveddata_engine): + # Find orphaned character IDs. This solves an issue where the chaaracter doesn't exist, but fits reference the pattern. + try: + logger.debug("Running database cleanup for orphaned characters attached to fits.") + results = saveddata_engine.execute("SELECT * FROM fits WHERE characterID NOT IN (SELECT ID FROM characters)") + + # Count how many records exist. This is ugly, but SQLAlchemy doesn't return a count from a select query. + result_count = 0 + for _ in results: + result_count += 1 + + if result_count > 0: + # Get All 5 character ID + all5_results = saveddata_engine.execute("SELECT ID FROM characters WHERE name = 'All 5'") + + all5_result_count = 0 + all5_id = 0 + for all5_result in all5_results: + all5_id = all5_result[0] + all5_result_count += 1 + + if all5_result_count == 0: + logger.error("Missing 'All 5' character.") + elif all5_result_count > 1: + logger.error("More than one 'All 5' character found.") + else: + logger.error("Database corruption found. Cleaning up %d records.", result_count) + saveddata_engine.execute("UPDATE 'fits' SET 'damagePatternID' = ? " + "WHERE damagePatternID not in (select ID from damagePatterns)", + all5_id) + except sqlalchemy.exc.DatabaseError: + logger.error("Failed to connect to database.") diff --git a/service/prefetch.py b/service/prefetch.py index b16f52d40..6c45ffc63 100644 --- a/service/prefetch.py +++ b/service/prefetch.py @@ -23,6 +23,11 @@ import os import eos.types import eos.db.migration as migration from eos.db.saveddata.loadDefaultDatabaseValues import DefaultDatabaseValues +from eos.db.saveddata.databaseRepair import DatabaseCleanup +import logging + +logger = logging.getLogger(__name__) + class PrefetchThread(threading.Thread): def run(self): @@ -55,6 +60,14 @@ if os.path.isfile(config.saveDB): # Import default database values # Import values that must exist otherwise Pyfa breaks DefaultDatabaseValues.importRequiredDefaults() + + logging.debug("Starting database validation.") + database_cleanup_instance = DatabaseCleanup() + database_cleanup_instance.OrphanedCharacterSkills(eos.db.saveddata_engine) + database_cleanup_instance.OrphanedFitCharacterIDs(eos.db.saveddata_engine) + database_cleanup_instance.OrphanedFitDamagePatterns(eos.db.saveddata_engine) + logging.debug("Completed database validation.") + else: # If database does not exist, do not worry about migration. Simply # create and set version @@ -67,4 +80,3 @@ else: DefaultDatabaseValues.importDamageProfileDefaults() # Import default values for target resist profiles DefaultDatabaseValues.importResistProfileDefaults() -