From cef58423441ca45ae2f2209d1fbe68b65a5c7b23 Mon Sep 17 00:00:00 2001 From: Ebag333 Date: Mon, 26 Dec 2016 21:54:52 -0800 Subject: [PATCH 1/6] Database validation and cleanup --- eos/db/saveddata/databaseRepair.py | 122 +++++++++++++++++++++++++++++ service/prefetch.py | 14 +++- 2 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 eos/db/saveddata/databaseRepair.py diff --git a/eos/db/saveddata/databaseRepair.py b/eos/db/saveddata/databaseRepair.py new file mode 100644 index 000000000..be26c8231 --- /dev/null +++ b/eos/db/saveddata/databaseRepair.py @@ -0,0 +1,122 @@ +# =============================================================================== +# Copyright (C) 2010 Diego Duclos +# +# This file is part of pyfa. +# +# pyfa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyfa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyfa. If not, see . +# =============================================================================== + +import sqlalchemy +import logging + +logger = logging.getLogger(__name__) + + +class DatabaseCleanup: + def __init__(self): + pass + + @staticmethod + def OrphanedCharacterSkills(saveddata_engine): + # Finds and fixes database corruption issues. + logger.debug("Start databsae validation and cleanup.") + + # Find orphaned character skills. + # This solves an issue where the character doesn't exist, but skills for that character do. + # See issue #917 + try: + logger.debug("Running database cleanup for character skills.") + results = saveddata_engine.execute("SELECT * FROM characterSkills " + "WHERE characterID NOT IN (SELECT ID from characters)") + + # Count how many records exist. This is ugly, but SQLAlchemy doesn't return a count from a select query. + result_count = 0 + for _ in results: + result_count += 1 + + if result_count > 0: + logger.error("Database corruption found. Cleaning up %d records.", result_count) + saveddata_engine.execute("DELETE FROM characterSkills WHERE characterID NOT IN (SELECT ID from characters)") + except sqlalchemy.exc.DatabaseError: + logger.error("Failed to connect to database.") + + @staticmethod + def OrphanedFitDamagePatterns(saveddata_engine): + # Find orphaned damage patterns. + # This solves an issue where the damage pattern doesn't exist, but fits reference the pattern. + # See issue #777 + try: + logger.debug("Running database cleanup for orphaned damage patterns attached to fits.") + results = saveddata_engine.execute("SELECT * FROM fits WHERE damagePatternID not in (select ID from damagePatterns)") + + # Count how many records exist. This is ugly, but SQLAlchemy doesn't return a count from a select query. + result_count = 0 + for _ in results: + result_count += 1 + + if result_count > 0: + # Get Uniform damage pattern ID + uniform_results = saveddata_engine.execute("select ID from damagePatterns WHERE name = 'Uniform'") + + uniform_result_count = 0 + uniform_damage_pattern_id = 0 + for uniform_result in uniform_results: + uniform_damage_pattern_id = uniform_result[0] + uniform_result_count += 1 + + if uniform_result_count == 0: + logger.error("Missing uniform damage pattern.") + elif uniform_result_count > 1: + logger.error("More than one uniform damage pattern found.") + else: + logger.error("Database corruption found. Cleaning up %d records.", result_count) + saveddata_engine.execute("UPDATE 'fits' SET 'damagePatternID' = ? " + "WHERE damagePatternID NOT IN (SELECT ID FROM damagePatterns)", + uniform_damage_pattern_id) + except sqlalchemy.exc.DatabaseError: + logger.error("Failed to connect to database.") + + @staticmethod + def OrphanedFitCharacterIDs(saveddata_engine): + # Find orphaned character IDs. This solves an issue where the chaaracter doesn't exist, but fits reference the pattern. + try: + logger.debug("Running database cleanup for orphaned characters attached to fits.") + results = saveddata_engine.execute("SELECT * FROM fits WHERE characterID NOT IN (SELECT ID FROM characters)") + + # Count how many records exist. This is ugly, but SQLAlchemy doesn't return a count from a select query. + result_count = 0 + for _ in results: + result_count += 1 + + if result_count > 0: + # Get All 5 character ID + all5_results = saveddata_engine.execute("SELECT ID FROM characters WHERE name = 'All 5'") + + all5_result_count = 0 + all5_id = 0 + for all5_result in all5_results: + all5_id = all5_result[0] + all5_result_count += 1 + + if all5_result_count == 0: + logger.error("Missing 'All 5' character.") + elif all5_result_count > 1: + logger.error("More than one 'All 5' character found.") + else: + logger.error("Database corruption found. Cleaning up %d records.", result_count) + saveddata_engine.execute("UPDATE 'fits' SET 'damagePatternID' = ? " + "WHERE damagePatternID not in (select ID from damagePatterns)", + all5_id) + except sqlalchemy.exc.DatabaseError: + logger.error("Failed to connect to database.") diff --git a/service/prefetch.py b/service/prefetch.py index b16f52d40..6c45ffc63 100644 --- a/service/prefetch.py +++ b/service/prefetch.py @@ -23,6 +23,11 @@ import os import eos.types import eos.db.migration as migration from eos.db.saveddata.loadDefaultDatabaseValues import DefaultDatabaseValues +from eos.db.saveddata.databaseRepair import DatabaseCleanup +import logging + +logger = logging.getLogger(__name__) + class PrefetchThread(threading.Thread): def run(self): @@ -55,6 +60,14 @@ if os.path.isfile(config.saveDB): # Import default database values # Import values that must exist otherwise Pyfa breaks DefaultDatabaseValues.importRequiredDefaults() + + logging.debug("Starting database validation.") + database_cleanup_instance = DatabaseCleanup() + database_cleanup_instance.OrphanedCharacterSkills(eos.db.saveddata_engine) + database_cleanup_instance.OrphanedFitCharacterIDs(eos.db.saveddata_engine) + database_cleanup_instance.OrphanedFitDamagePatterns(eos.db.saveddata_engine) + logging.debug("Completed database validation.") + else: # If database does not exist, do not worry about migration. Simply # create and set version @@ -67,4 +80,3 @@ else: DefaultDatabaseValues.importDamageProfileDefaults() # Import default values for target resist profiles DefaultDatabaseValues.importResistProfileDefaults() - From 85a89339a73ee9a040e98011b27db2c87835bdd6 Mon Sep 17 00:00:00 2001 From: blitzman Date: Mon, 2 Jan 2017 00:58:54 -0500 Subject: [PATCH 2/6] Cleaned up database corruption stuff --- eos/db/saveddata/databaseRepair.py | 37 ++++++++++-------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/eos/db/saveddata/databaseRepair.py b/eos/db/saveddata/databaseRepair.py index be26c8231..5561b41a1 100644 --- a/eos/db/saveddata/databaseRepair.py +++ b/eos/db/saveddata/databaseRepair.py @@ -37,17 +37,13 @@ class DatabaseCleanup: # See issue #917 try: logger.debug("Running database cleanup for character skills.") - results = saveddata_engine.execute("SELECT * FROM characterSkills " + results = saveddata_engine.execute("SELECT COUNT(*) AS num FROM characterSkills " "WHERE characterID NOT IN (SELECT ID from characters)") - # Count how many records exist. This is ugly, but SQLAlchemy doesn't return a count from a select query. - result_count = 0 - for _ in results: - result_count += 1 + if results.fetchone()['num'] > 0: + delete = saveddata_engine.execute("DELETE FROM characterSkills WHERE characterID NOT IN (SELECT ID from characters)") + logger.error("Database corruption found. Cleaning up %d records.", delete.rowcount) - if result_count > 0: - logger.error("Database corruption found. Cleaning up %d records.", result_count) - saveddata_engine.execute("DELETE FROM characterSkills WHERE characterID NOT IN (SELECT ID from characters)") except sqlalchemy.exc.DatabaseError: logger.error("Failed to connect to database.") @@ -58,14 +54,9 @@ class DatabaseCleanup: # See issue #777 try: logger.debug("Running database cleanup for orphaned damage patterns attached to fits.") - results = saveddata_engine.execute("SELECT * FROM fits WHERE damagePatternID not in (select ID from damagePatterns)") + results = saveddata_engine.execute("SELECT COUNT(*) AS num FROM fits WHERE damagePatternID not in (select ID from damagePatterns)") - # Count how many records exist. This is ugly, but SQLAlchemy doesn't return a count from a select query. - result_count = 0 - for _ in results: - result_count += 1 - - if result_count > 0: + if results.fetchone()['num'] > 0: # Get Uniform damage pattern ID uniform_results = saveddata_engine.execute("select ID from damagePatterns WHERE name = 'Uniform'") @@ -80,10 +71,10 @@ class DatabaseCleanup: elif uniform_result_count > 1: logger.error("More than one uniform damage pattern found.") else: - logger.error("Database corruption found. Cleaning up %d records.", result_count) - saveddata_engine.execute("UPDATE 'fits' SET 'damagePatternID' = ? " + update = saveddata_engine.execute("UPDATE 'fits' SET 'damagePatternID' = ? " "WHERE damagePatternID NOT IN (SELECT ID FROM damagePatterns)", uniform_damage_pattern_id) + logger.error("Database corruption found. Cleaning up %d records.", update.rowcount) except sqlalchemy.exc.DatabaseError: logger.error("Failed to connect to database.") @@ -92,14 +83,10 @@ class DatabaseCleanup: # Find orphaned character IDs. This solves an issue where the chaaracter doesn't exist, but fits reference the pattern. try: logger.debug("Running database cleanup for orphaned characters attached to fits.") - results = saveddata_engine.execute("SELECT * FROM fits WHERE characterID NOT IN (SELECT ID FROM characters)") + results = saveddata_engine.execute("SELECT COUNT(*) AS num FROM fits WHERE characterID NOT IN (SELECT ID FROM characters)") # Count how many records exist. This is ugly, but SQLAlchemy doesn't return a count from a select query. - result_count = 0 - for _ in results: - result_count += 1 - - if result_count > 0: + if results.fetchone()['num'] > 0: # Get All 5 character ID all5_results = saveddata_engine.execute("SELECT ID FROM characters WHERE name = 'All 5'") @@ -114,9 +101,9 @@ class DatabaseCleanup: elif all5_result_count > 1: logger.error("More than one 'All 5' character found.") else: - logger.error("Database corruption found. Cleaning up %d records.", result_count) - saveddata_engine.execute("UPDATE 'fits' SET 'damagePatternID' = ? " + update = saveddata_engine.execute("UPDATE 'fits' SET 'damagePatternID' = ? " "WHERE damagePatternID not in (select ID from damagePatterns)", all5_id) + logger.error("Database corruption found. Cleaning up %d records.", update.rowcount) except sqlalchemy.exc.DatabaseError: logger.error("Failed to connect to database.") From c1ead5fe27eda8be9f15c4c9f3133ff9ec5e37e4 Mon Sep 17 00:00:00 2001 From: Ebag333 Date: Mon, 2 Jan 2017 00:29:44 -0800 Subject: [PATCH 3/6] change method for retreiving count --- eos/db/saveddata/databaseRepair.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/eos/db/saveddata/databaseRepair.py b/eos/db/saveddata/databaseRepair.py index 5561b41a1..bfc6e8569 100644 --- a/eos/db/saveddata/databaseRepair.py +++ b/eos/db/saveddata/databaseRepair.py @@ -40,7 +40,13 @@ class DatabaseCleanup: results = saveddata_engine.execute("SELECT COUNT(*) AS num FROM characterSkills " "WHERE characterID NOT IN (SELECT ID from characters)") - if results.fetchone()['num'] > 0: + count_results = 0 + for result in results: + count_results = result[0] + break + + # Count how many records exist. This is ugly, but SQLAlchemy doesn't return a count from a select query. + if count_results > 0: delete = saveddata_engine.execute("DELETE FROM characterSkills WHERE characterID NOT IN (SELECT ID from characters)") logger.error("Database corruption found. Cleaning up %d records.", delete.rowcount) @@ -56,7 +62,13 @@ class DatabaseCleanup: logger.debug("Running database cleanup for orphaned damage patterns attached to fits.") results = saveddata_engine.execute("SELECT COUNT(*) AS num FROM fits WHERE damagePatternID not in (select ID from damagePatterns)") - if results.fetchone()['num'] > 0: + count_results = 0 + for result in results: + count_results = result[0] + break + + # Count how many records exist. This is ugly, but SQLAlchemy doesn't return a count from a select query. + if count_results > 0: # Get Uniform damage pattern ID uniform_results = saveddata_engine.execute("select ID from damagePatterns WHERE name = 'Uniform'") @@ -80,13 +92,18 @@ class DatabaseCleanup: @staticmethod def OrphanedFitCharacterIDs(saveddata_engine): - # Find orphaned character IDs. This solves an issue where the chaaracter doesn't exist, but fits reference the pattern. + # Find orphaned character IDs. This solves an issue where the character doesn't exist, but fits reference the pattern. try: logger.debug("Running database cleanup for orphaned characters attached to fits.") results = saveddata_engine.execute("SELECT COUNT(*) AS num FROM fits WHERE characterID NOT IN (SELECT ID FROM characters)") + count_results = 0 + for result in results: + count_results = result[0] + break + # Count how many records exist. This is ugly, but SQLAlchemy doesn't return a count from a select query. - if results.fetchone()['num'] > 0: + if count_results > 0: # Get All 5 character ID all5_results = saveddata_engine.execute("SELECT ID FROM characters WHERE name = 'All 5'") From a76ee6b2fc42e69add53ce9ba7d1db1bc87315fb Mon Sep 17 00:00:00 2001 From: blitzman Date: Mon, 2 Jan 2017 12:07:15 -0500 Subject: [PATCH 4/6] more tweaks --- eos/db/saveddata/databaseRepair.py | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/eos/db/saveddata/databaseRepair.py b/eos/db/saveddata/databaseRepair.py index bfc6e8569..d11f97aed 100644 --- a/eos/db/saveddata/databaseRepair.py +++ b/eos/db/saveddata/databaseRepair.py @@ -40,13 +40,9 @@ class DatabaseCleanup: results = saveddata_engine.execute("SELECT COUNT(*) AS num FROM characterSkills " "WHERE characterID NOT IN (SELECT ID from characters)") - count_results = 0 - for result in results: - count_results = result[0] - break + row = results.first() - # Count how many records exist. This is ugly, but SQLAlchemy doesn't return a count from a select query. - if count_results > 0: + if row and row['num'] > 0: delete = saveddata_engine.execute("DELETE FROM characterSkills WHERE characterID NOT IN (SELECT ID from characters)") logger.error("Database corruption found. Cleaning up %d records.", delete.rowcount) @@ -62,13 +58,9 @@ class DatabaseCleanup: logger.debug("Running database cleanup for orphaned damage patterns attached to fits.") results = saveddata_engine.execute("SELECT COUNT(*) AS num FROM fits WHERE damagePatternID not in (select ID from damagePatterns)") - count_results = 0 - for result in results: - count_results = result[0] - break + row = results.first() - # Count how many records exist. This is ugly, but SQLAlchemy doesn't return a count from a select query. - if count_results > 0: + if row and row['num'] > 0: # Get Uniform damage pattern ID uniform_results = saveddata_engine.execute("select ID from damagePatterns WHERE name = 'Uniform'") @@ -97,13 +89,9 @@ class DatabaseCleanup: logger.debug("Running database cleanup for orphaned characters attached to fits.") results = saveddata_engine.execute("SELECT COUNT(*) AS num FROM fits WHERE characterID NOT IN (SELECT ID FROM characters)") - count_results = 0 - for result in results: - count_results = result[0] - break + row = results.first() - # Count how many records exist. This is ugly, but SQLAlchemy doesn't return a count from a select query. - if count_results > 0: + if row and row['num'] > 0: # Get All 5 character ID all5_results = saveddata_engine.execute("SELECT ID FROM characters WHERE name = 'All 5'") From 3313b7421f9b343b8adc500bed246a61e0328c3c Mon Sep 17 00:00:00 2001 From: Ebag333 Date: Mon, 2 Jan 2017 13:41:39 -0800 Subject: [PATCH 5/6] Move query logic into own function in queries. Tighten up flow in DB repair. Catch null values. --- eos/db/saveddata/databaseRepair.py | 45 +++++++++++------------------- eos/db/saveddata/queries.py | 16 +++++++++++ 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/eos/db/saveddata/databaseRepair.py b/eos/db/saveddata/databaseRepair.py index d11f97aed..c0e1b427e 100644 --- a/eos/db/saveddata/databaseRepair.py +++ b/eos/db/saveddata/databaseRepair.py @@ -19,6 +19,7 @@ import sqlalchemy import logging +from eos.db.saveddata.queries import executeDatabaseQuery logger = logging.getLogger(__name__) @@ -39,10 +40,9 @@ class DatabaseCleanup: logger.debug("Running database cleanup for character skills.") results = saveddata_engine.execute("SELECT COUNT(*) AS num FROM characterSkills " "WHERE characterID NOT IN (SELECT ID from characters)") - row = results.first() - if row and row['num'] > 0: + if row['num']: delete = saveddata_engine.execute("DELETE FROM characterSkills WHERE characterID NOT IN (SELECT ID from characters)") logger.error("Database corruption found. Cleaning up %d records.", delete.rowcount) @@ -56,27 +56,22 @@ class DatabaseCleanup: # See issue #777 try: logger.debug("Running database cleanup for orphaned damage patterns attached to fits.") - results = saveddata_engine.execute("SELECT COUNT(*) AS num FROM fits WHERE damagePatternID not in (select ID from damagePatterns)") + results = saveddata_engine.execute("SELECT COUNT(*) AS num FROM fits WHERE damagePatternID NOT IN (SELECT ID FROM damagePatterns) OR damagePatternID IS NULL") row = results.first() - if row and row['num'] > 0: + if row['num']: # Get Uniform damage pattern ID - uniform_results = saveddata_engine.execute("select ID from damagePatterns WHERE name = 'Uniform'") + query_return = executeDatabaseQuery(saveddata_engine, "SELECT ID FROM damagePatterns WHERE name = 'Uniform'") - uniform_result_count = 0 - uniform_damage_pattern_id = 0 - for uniform_result in uniform_results: - uniform_damage_pattern_id = uniform_result[0] - uniform_result_count += 1 - - if uniform_result_count == 0: + if query_return.__len__() <= 0: logger.error("Missing uniform damage pattern.") - elif uniform_result_count > 1: + elif query_return.__len__() > 1: logger.error("More than one uniform damage pattern found.") else: + uniform_damage_pattern_id = query_return[0]['ID'] update = saveddata_engine.execute("UPDATE 'fits' SET 'damagePatternID' = ? " - "WHERE damagePatternID NOT IN (SELECT ID FROM damagePatterns)", + "WHERE damagePatternID NOT IN (SELECT ID FROM damagePatterns) OR damagePatternID IS NULL", uniform_damage_pattern_id) logger.error("Database corruption found. Cleaning up %d records.", update.rowcount) except sqlalchemy.exc.DatabaseError: @@ -87,27 +82,21 @@ class DatabaseCleanup: # Find orphaned character IDs. This solves an issue where the character doesn't exist, but fits reference the pattern. try: logger.debug("Running database cleanup for orphaned characters attached to fits.") - results = saveddata_engine.execute("SELECT COUNT(*) AS num FROM fits WHERE characterID NOT IN (SELECT ID FROM characters)") - + results = saveddata_engine.execute("SELECT COUNT(*) AS num FROM fits WHERE characterID NOT IN (SELECT ID FROM characters) OR characterID IS NULL") row = results.first() - if row and row['num'] > 0: + if row['num']: # Get All 5 character ID - all5_results = saveddata_engine.execute("SELECT ID FROM characters WHERE name = 'All 5'") + query_return = executeDatabaseQuery(saveddata_engine, "SELECT ID FROM characters WHERE name = 'All 5'") - all5_result_count = 0 - all5_id = 0 - for all5_result in all5_results: - all5_id = all5_result[0] - all5_result_count += 1 - - if all5_result_count == 0: + if query_return.__len__() <= 0: logger.error("Missing 'All 5' character.") - elif all5_result_count > 1: + elif query_return.__len__() > 1: logger.error("More than one 'All 5' character found.") else: - update = saveddata_engine.execute("UPDATE 'fits' SET 'damagePatternID' = ? " - "WHERE damagePatternID not in (select ID from damagePatterns)", + all5_id = query_return[0]['ID'] + update = saveddata_engine.execute("UPDATE 'fits' SET 'characterID' = ? " + "WHERE characterID not in (select ID from characters) OR characterID IS NULL", all5_id) logger.error("Database corruption found. Cleaning up %d records.", update.rowcount) except sqlalchemy.exc.DatabaseError: diff --git a/eos/db/saveddata/queries.py b/eos/db/saveddata/queries.py index cb3033cc1..c18dff912 100644 --- a/eos/db/saveddata/queries.py +++ b/eos/db/saveddata/queries.py @@ -407,6 +407,22 @@ def getCrestCharacter(lookfor, eager=None): raise TypeError("Need integer or string as argument") return character +def executeDatabaseQuery(saveddata_engine, query): + # Executes a query against the database, and returns a dict instead of a resultsproxy + results = saveddata_engine.execute(query) + + return_list = [] + + for row in results: + internal_row = {} + for key in row._keymap: + idx = row._keymap[key][2] + internal_row.update({key: row._row[idx]}) + + return_list.append(internal_row) + + return return_list + def getOverrides(itemID, eager=None): if isinstance(itemID, int): return saveddata_session.query(Override).filter(Override.itemID == itemID).all() From a0ec13d2ed3242acbf9260e8353bda1cfe64dc29 Mon Sep 17 00:00:00 2001 From: blitzman Date: Mon, 2 Jan 2017 18:45:08 -0500 Subject: [PATCH 6/6] More tweaks --- eos/db/saveddata/databaseRepair.py | 31 +++++++++++++++--------------- eos/db/saveddata/queries.py | 16 --------------- 2 files changed, 16 insertions(+), 31 deletions(-) diff --git a/eos/db/saveddata/databaseRepair.py b/eos/db/saveddata/databaseRepair.py index c0e1b427e..ede0f2fde 100644 --- a/eos/db/saveddata/databaseRepair.py +++ b/eos/db/saveddata/databaseRepair.py @@ -19,7 +19,6 @@ import sqlalchemy import logging -from eos.db.saveddata.queries import executeDatabaseQuery logger = logging.getLogger(__name__) @@ -42,7 +41,7 @@ class DatabaseCleanup: "WHERE characterID NOT IN (SELECT ID from characters)") row = results.first() - if row['num']: + if row and row['num']: delete = saveddata_engine.execute("DELETE FROM characterSkills WHERE characterID NOT IN (SELECT ID from characters)") logger.error("Database corruption found. Cleaning up %d records.", delete.rowcount) @@ -60,19 +59,20 @@ class DatabaseCleanup: results = saveddata_engine.execute("SELECT COUNT(*) AS num FROM fits WHERE damagePatternID NOT IN (SELECT ID FROM damagePatterns) OR damagePatternID IS NULL") row = results.first() - if row['num']: + if row and row['num']: # Get Uniform damage pattern ID - query_return = executeDatabaseQuery(saveddata_engine, "SELECT ID FROM damagePatterns WHERE name = 'Uniform'") + query = saveddata_engine.execute("SELECT ID FROM damagePatterns WHERE name = 'Uniform'") + rows = query.fetchall() - if query_return.__len__() <= 0: + if len(rows) == 0: logger.error("Missing uniform damage pattern.") - elif query_return.__len__() > 1: + elif len(rows) > 1: logger.error("More than one uniform damage pattern found.") else: - uniform_damage_pattern_id = query_return[0]['ID'] + uniform_damage_pattern_id = rows[0]['ID'] update = saveddata_engine.execute("UPDATE 'fits' SET 'damagePatternID' = ? " - "WHERE damagePatternID NOT IN (SELECT ID FROM damagePatterns) OR damagePatternID IS NULL", - uniform_damage_pattern_id) + "WHERE damagePatternID NOT IN (SELECT ID FROM damagePatterns) OR damagePatternID IS NULL", + uniform_damage_pattern_id) logger.error("Database corruption found. Cleaning up %d records.", update.rowcount) except sqlalchemy.exc.DatabaseError: logger.error("Failed to connect to database.") @@ -87,17 +87,18 @@ class DatabaseCleanup: if row['num']: # Get All 5 character ID - query_return = executeDatabaseQuery(saveddata_engine, "SELECT ID FROM characters WHERE name = 'All 5'") + query = saveddata_engine.execute("SELECT ID FROM characters WHERE name = 'All 5'") + rows = query.fetchall() - if query_return.__len__() <= 0: + if len(rows) == 0: logger.error("Missing 'All 5' character.") - elif query_return.__len__() > 1: + elif len(rows) > 1: logger.error("More than one 'All 5' character found.") else: - all5_id = query_return[0]['ID'] + all5_id = rows[0]['ID'] update = saveddata_engine.execute("UPDATE 'fits' SET 'characterID' = ? " - "WHERE characterID not in (select ID from characters) OR characterID IS NULL", - all5_id) + "WHERE characterID not in (select ID from characters) OR characterID IS NULL", + all5_id) logger.error("Database corruption found. Cleaning up %d records.", update.rowcount) except sqlalchemy.exc.DatabaseError: logger.error("Failed to connect to database.") diff --git a/eos/db/saveddata/queries.py b/eos/db/saveddata/queries.py index c18dff912..cb3033cc1 100644 --- a/eos/db/saveddata/queries.py +++ b/eos/db/saveddata/queries.py @@ -407,22 +407,6 @@ def getCrestCharacter(lookfor, eager=None): raise TypeError("Need integer or string as argument") return character -def executeDatabaseQuery(saveddata_engine, query): - # Executes a query against the database, and returns a dict instead of a resultsproxy - results = saveddata_engine.execute(query) - - return_list = [] - - for row in results: - internal_row = {} - for key in row._keymap: - idx = row._keymap[key][2] - internal_row.update({key: row._row[idx]}) - - return_list.append(internal_row) - - return return_list - def getOverrides(itemID, eager=None): if isinstance(itemID, int): return saveddata_session.query(Override).filter(Override.itemID == itemID).all()