From fc7ca56f8bd19cbc16b1d3ece92ed9df59d0ddbf Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Mon, 10 Apr 2017 09:33:54 +0900 Subject: [PATCH] searching for a way to open db with minimal changes required --- _development/__init__.py | 0 _development/helpers.py | 143 +++ _development/helpers_fits.py | 66 ++ service/settings.py | 11 +- tests/jeffy_ja-en[99].xml | 2116 ++++++++++++++++++++++++++++++++++ tests/test_fitcache.py | 232 ++++ tests/test_unread_desc.py | 69 ++ 7 files changed, 2633 insertions(+), 4 deletions(-) create mode 100644 _development/__init__.py create mode 100644 _development/helpers.py create mode 100644 _development/helpers_fits.py create mode 100644 tests/jeffy_ja-en[99].xml create mode 100644 tests/test_fitcache.py create mode 100644 tests/test_unread_desc.py diff --git a/_development/__init__.py b/_development/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/_development/helpers.py b/_development/helpers.py new file mode 100644 index 000000000..b35ac835f --- /dev/null +++ b/_development/helpers.py @@ -0,0 +1,143 @@ +# noinspection PyPackageRequirements +import pytest + +import os +import sys +import threading + +from sqlalchemy import MetaData, create_engine +from sqlalchemy.orm import sessionmaker + +script_dir = os.path.dirname(os.path.abspath(__file__)) +# Add root folder to python paths +sys.path.append(os.path.realpath(os.path.join(script_dir, '..', '..'))) +sys._called_from_test = True + +# noinspection PyUnresolvedReferences,PyUnusedLocal +@pytest.fixture +def DBInMemory_test(): + def rollback(): + with sd_lock: + saveddata_session.rollback() + + + print("Creating database in memory") + from os.path import realpath, join, dirname, abspath + + debug = False + gamedataCache = True + saveddataCache = True + gamedata_version = "" + gamedata_connectionstring = 'sqlite:///' + realpath(join(dirname(abspath(unicode(__file__))), "..", "eve.db")) + saveddata_connectionstring = 'sqlite:///:memory:' + + class ReadOnlyException(Exception): + pass + + if callable(gamedata_connectionstring): + gamedata_engine = create_engine("sqlite://", creator=gamedata_connectionstring, echo=debug) + else: + gamedata_engine = create_engine(gamedata_connectionstring, echo=debug) + + gamedata_meta = MetaData() + gamedata_meta.bind = gamedata_engine + gamedata_session = sessionmaker(bind=gamedata_engine, autoflush=False, expire_on_commit=False)() + + # This should be moved elsewhere, maybe as an actual query. Current, without try-except, it breaks when making a new + # game db because we haven't reached gamedata_meta.create_all() + try: + gamedata_version = gamedata_session.execute( + "SELECT `field_value` FROM `metadata` WHERE `field_name` LIKE 'client_build'" + ).fetchone()[0] + except Exception as e: + print("Missing gamedata version.") + gamedata_version = None + + if saveddata_connectionstring is not None: + if callable(saveddata_connectionstring): + saveddata_engine = create_engine(creator=saveddata_connectionstring, echo=debug) + else: + saveddata_engine = create_engine(saveddata_connectionstring, echo=debug) + + saveddata_meta = MetaData() + saveddata_meta.bind = saveddata_engine + saveddata_session = sessionmaker(bind=saveddata_engine, autoflush=False, expire_on_commit=False)() + else: + saveddata_meta = None + + # Lock controlling any changes introduced to session + sd_lock = threading.Lock() + + # Import all the definitions for all our database stuff + # noinspection PyPep8 + #from eos.db.gamedata import alphaClones, attribute, category, effect, group, icon, item, marketGroup, metaData, metaGroup, queries, traits, unit + # noinspection PyPep8 + #from eos.db.saveddata import booster, cargo, character, crest, damagePattern, databaseRepair, drone, fighter, fit, implant, implantSet, loadDefaultDatabaseValues, miscData, module, override, price, queries, skill, targetResists, user + + # If using in memory saveddata, you'll want to reflect it so the data structure is good. + if saveddata_connectionstring == "sqlite:///:memory:": + saveddata_meta.create_all() + + # Output debug info to help us troubleshoot Travis + print(saveddata_engine) + print(gamedata_engine) + + helper = { + #'config': eos.config, + 'gamedata_session' : gamedata_session, + 'saveddata_session' : saveddata_session, + } + return helper + +# noinspection PyUnresolvedReferences,PyUnusedLocal +@pytest.fixture +def DBInMemory(): + print("Creating database in memory") + + import eos.config + + import eos + import eos.db + + # Output debug info to help us troubleshoot Travis + print(eos.db.saveddata_engine) + print(eos.db.gamedata_engine) + + helper = { + 'config': eos.config, + 'db' : eos.db, + 'gamedata_session' : eos.db.gamedata_session, + 'saveddata_session' : eos.db.saveddata_session, + } + return helper + + +@pytest.fixture +def Gamedata(): + print("Building Gamedata") + from eos.gamedata import Item + + helper = { + 'Item': Item, + } + return helper + + +@pytest.fixture +def Saveddata(): + print("Building Saveddata") + from eos.saveddata.ship import Ship + from eos.saveddata.fit import Fit + from eos.saveddata.character import Character + from eos.saveddata.module import Module, State + from eos.saveddata.citadel import Citadel + + helper = { + 'Structure': Citadel, + 'Ship' : Ship, + 'Fit' : Fit, + 'Character': Character, + 'Module' : Module, + 'State' : State, + } + return helper diff --git a/_development/helpers_fits.py b/_development/helpers_fits.py new file mode 100644 index 000000000..f164a6bb3 --- /dev/null +++ b/_development/helpers_fits.py @@ -0,0 +1,66 @@ +import pytest + +# noinspection PyPackageRequirements +from _development.helpers import DBInMemory as DB, Gamedata, Saveddata + + +# noinspection PyShadowingNames +@pytest.fixture +def RifterFit(DB, Gamedata, Saveddata): + print("Creating Rifter") + item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Rifter").first() + ship = Saveddata['Ship'](item) + # setup fit + fit = Saveddata['Fit'](ship, "My Rifter Fit") + + return fit + + +# noinspection PyShadowingNames +@pytest.fixture +def KeepstarFit(DB, Gamedata, Saveddata): + print("Creating Keepstar") + item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Keepstar").first() + ship = Saveddata['Structure'](item) + # setup fit + fit = Saveddata['Fit'](ship, "Keepstar Fit") + + return fit + + +# noinspection PyShadowingNames +@pytest.fixture +def CurseFit(DB, Gamedata, Saveddata): + print("Creating Curse - With Neuts") + item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Curse").first() + ship = Saveddata['Ship'](item) + # setup fit + fit = Saveddata['Fit'](ship, "Curse - With Neuts") + + mod = Saveddata['Module'](DB['db'].getItem("Medium Energy Neutralizer II")) + mod.state = Saveddata['State'].ONLINE + + # Add 5 neuts + for _ in xrange(5): + fit.modules.append(mod) + + return fit + + +# noinspection PyShadowingNames +@pytest.fixture +def HeronFit(DB, Gamedata, Saveddata): + print("Creating Heron - RemoteSebo") + item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Heron").first() + ship = Saveddata['Ship'](item) + # setup fit + fit = Saveddata['Fit'](ship, "Heron - RemoteSebo") + + mod = Saveddata['Module'](DB['db'].getItem("Remote Sensor Booster II")) + mod.state = Saveddata['State'].ONLINE + + # Add 5 neuts + for _ in xrange(4): + fit.modules.append(mod) + + return fit diff --git a/service/settings.py b/service/settings.py index cfc48fc4a..3da94b084 100644 --- a/service/settings.py +++ b/service/settings.py @@ -29,7 +29,8 @@ pyfalog = Logger(__name__) class SettingsProvider(object): - BASE_PATH = os.path.join(config.savePath, 'settings') + if config.savePath: + BASE_PATH = os.path.join(config.savePath, 'settings') settings = {} _instance = None @@ -41,13 +42,15 @@ class SettingsProvider(object): return cls._instance def __init__(self): - if not os.path.exists(self.BASE_PATH): - os.mkdir(self.BASE_PATH) + if hasattr(self, 'BASE_PATH'): + if not os.path.exists(self.BASE_PATH): + os.mkdir(self.BASE_PATH) def getSettings(self, area, defaults=None): s = self.settings.get(area) - if s is None: + + if s is None and hasattr(self, 'BASE_PATH'): p = os.path.join(self.BASE_PATH, area) if not os.path.exists(p): diff --git a/tests/jeffy_ja-en[99].xml b/tests/jeffy_ja-en[99].xml new file mode 100644 index 000000000..8ec178b16 --- /dev/null +++ b/tests/jeffy_ja-en[99].xml @@ -0,0 +1,2116 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test_fitcache.py b/tests/test_fitcache.py new file mode 100644 index 000000000..9de98d097 --- /dev/null +++ b/tests/test_fitcache.py @@ -0,0 +1,232 @@ +# Add root folder to python paths +# This must be done on every test in order to pass in Travis +import os +import sys +from time import time +script_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.realpath(os.path.join(script_dir, '..'))) + +# +# noinspection PyPackageRequirements +from _development.helpers import DBInMemory as DB, Gamedata, Saveddata +# noinspection PyPackageRequirements +from _development.helpers_fits import RifterFit, KeepstarFit, HeronFit, CurseFit +from service.fit import Fit +# +# # Fake import wx +# # todo: fix this +# # from types import ModuleType +# # wx = ModuleType("fake_module") +# # sys.modules[wx.__name__] = wx +# +# def test_getAllFits(DB, RifterFit, KeepstarFit): +# assert len(Fit.getAllFits()) == 0 +# DB['db'].save(RifterFit) +# assert len(Fit.getAllFits()) == 1 +# DB['db'].save(KeepstarFit) +# assert len(Fit.getAllFits()) == 2 +# +# # Cleanup after ourselves +# DB['db'].remove(RifterFit) +# DB['db'].remove(KeepstarFit) +# +# +# def test_getFitsWithShip_RifterFit(DB, RifterFit): +# DB['db'].save(RifterFit) +# +# assert Fit.getFitsWithShip(587)[0][1] == 'My Rifter Fit' +# +# DB['db'].remove(RifterFit) + + +def test_RifterSingleNew(DB, RifterFit, KeepstarFit, HeronFit, CurseFit): + DB['db'].save(RifterFit) + DB['db'].save(KeepstarFit) + DB['db'].save(HeronFit) + DB['db'].save(CurseFit) + sFit = Fit.getInstance() + sFit.serviceFittingOptions = { + "useGlobalCharacter" : False, + "useGlobalDamagePattern": False, + "useGlobalForceReload" : False, + "colorFitBySlot" : False, + "rackSlots" : True, + "rackLabels" : True, + "compactSkills" : True, + "showTooltip" : True, + "showMarketShortcuts" : False, + "enableGaugeAnimation" : True, + "exportCharges" : True, + "openFitInNew" : False, + "priceSystem" : "Jita", + "showShipBrowserTooltip": True, + } + + cached_fits = [] + fit = DB["db"].getFit(1) + cached_fits.append(fit) + fit = None + + time_start = time() + + for _ in xrange(1000000): + + fit = next((x for x in cached_fits if x.ID == 1), None) + + fit = None + + print("1000000 of the Rifter fit (new): " + str(time()-time_start)) + + # fit = DB["db"].getFit(1) + + # Cleanup after ourselves + DB['db'].remove(RifterFit) + DB['db'].remove(KeepstarFit) + DB['db'].remove(HeronFit) + DB['db'].remove(CurseFit) + +def test_RifterSingleOld(DB, RifterFit, KeepstarFit, HeronFit, CurseFit): + DB['db'].save(RifterFit) + DB['db'].save(KeepstarFit) + DB['db'].save(HeronFit) + DB['db'].save(CurseFit) + sFit = Fit.getInstance() + sFit.serviceFittingOptions = { + "useGlobalCharacter" : False, + "useGlobalDamagePattern": False, + "useGlobalForceReload" : False, + "colorFitBySlot" : False, + "rackSlots" : True, + "rackLabels" : True, + "compactSkills" : True, + "showTooltip" : True, + "showMarketShortcuts" : False, + "enableGaugeAnimation" : True, + "exportCharges" : True, + "openFitInNew" : False, + "priceSystem" : "Jita", + "showShipBrowserTooltip": True, + } + + cached_fits = [] + fit = DB["db"].getFit(1) + cached_fits.append(fit) + fit = None + + time_start = time() + + for _ in xrange(1000000): + + fit = DB["db"].getFit(1) + + fit = None + + print("1000000 of the Rifter fit (old): " + str(time()-time_start)) + + # Cleanup after ourselves + DB['db'].remove(RifterFit) + DB['db'].remove(KeepstarFit) + DB['db'].remove(HeronFit) + DB['db'].remove(CurseFit) + +def test_FourNew(DB, RifterFit, KeepstarFit, HeronFit, CurseFit): + DB['db'].save(RifterFit) + DB['db'].save(KeepstarFit) + DB['db'].save(HeronFit) + DB['db'].save(CurseFit) + sFit = Fit.getInstance() + sFit.serviceFittingOptions = { + "useGlobalCharacter" : False, + "useGlobalDamagePattern": False, + "useGlobalForceReload" : False, + "colorFitBySlot" : False, + "rackSlots" : True, + "rackLabels" : True, + "compactSkills" : True, + "showTooltip" : True, + "showMarketShortcuts" : False, + "enableGaugeAnimation" : True, + "exportCharges" : True, + "openFitInNew" : False, + "priceSystem" : "Jita", + "showShipBrowserTooltip": True, + } + + cached_fits = [] + fit = DB["db"].getFit(1) + cached_fits.append(fit) + fit = None + + time_start = time() + + for _ in xrange(250000): + + fit = next((x for x in cached_fits if x.ID == 1), None) + fit = None + fit = next((x for x in cached_fits if x.ID == 2), None) + fit = None + fit = next((x for x in cached_fits if x.ID == 3), None) + fit = None + fit = next((x for x in cached_fits if x.ID == 4), None) + fit = None + + + print("1000000 of the four fits (new): " + str(time()-time_start)) + + # fit = DB["db"].getFit(1) + + # Cleanup after ourselves + DB['db'].remove(RifterFit) + DB['db'].remove(KeepstarFit) + DB['db'].remove(HeronFit) + DB['db'].remove(CurseFit) + +def test_FourOld(DB, RifterFit, KeepstarFit, HeronFit, CurseFit): + DB['db'].save(RifterFit) + DB['db'].save(KeepstarFit) + DB['db'].save(HeronFit) + DB['db'].save(CurseFit) + sFit = Fit.getInstance() + sFit.serviceFittingOptions = { + "useGlobalCharacter" : False, + "useGlobalDamagePattern": False, + "useGlobalForceReload" : False, + "colorFitBySlot" : False, + "rackSlots" : True, + "rackLabels" : True, + "compactSkills" : True, + "showTooltip" : True, + "showMarketShortcuts" : False, + "enableGaugeAnimation" : True, + "exportCharges" : True, + "openFitInNew" : False, + "priceSystem" : "Jita", + "showShipBrowserTooltip": True, + } + + cached_fits = [] + fit = DB["db"].getFit(1) + cached_fits.append(fit) + fit = None + + time_start = time() + + for _ in xrange(250000): + + fit = DB["db"].getFit(1) + fit = None + fit = DB["db"].getFit(2) + fit = None + fit = DB["db"].getFit(3) + fit = None + fit = DB["db"].getFit(4) + fit = None + + print("1000000 of the four fits (old): " + str(time()-time_start)) + + # Cleanup after ourselves + DB['db'].remove(RifterFit) + DB['db'].remove(KeepstarFit) + DB['db'].remove(HeronFit) + DB['db'].remove(CurseFit) + diff --git a/tests/test_unread_desc.py b/tests/test_unread_desc.py new file mode 100644 index 000000000..8cd2d40a6 --- /dev/null +++ b/tests/test_unread_desc.py @@ -0,0 +1,69 @@ +""" + 2017/04/05: unread description tests module. +""" +# Add root folder to python paths +# This must be done on every test in order to pass in Travis +import os +import sys +script_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.realpath(os.path.join(script_dir, '..'))) + +import re +# +# noinspection PyPackageRequirements +from _development.helpers import DBInMemory as DB, Gamedata, Saveddata +# noinspection PyPep8 +from service.port import Port, IPortUser + +# from utils.strfunctions import sequential_rep, replace_ltgt +from utils.stopwatch import Stopwatch + +""" +NOTE: + description character length is restricted 4hundred by EVE client. + these things apply to multi byte environment too. + + + o read xml fit data (and encode to utf-8 if need. + + o construct xml dom object, and extract "fitting" elements. + + o apply _resolve_ship method to each "fitting" elements. (time measurement + + o extract "hardware" elements from "fitting" element. + + o apply _resolve_module method to each "hardware" elements. (time measurement + +xml files: + "jeffy_ja-en[99].xml" + +NOTE of @decorator: + o Function to receive arguments of function to be decorated + o A function that accepts the decorate target function itself as an argument + o A function that accepts arguments of the decorator itself +""" + +class PortUser(IPortUser): + + def on_port_processing(self, action, data=None): + print(data) + return True + + +stpw = Stopwatch('test measurementer') + +def test_import_xml(): + usr = PortUser() +# for path in XML_FILES: + xml_file = "jeffy_ja-en[99].xml" + fit_count = int(re.search(r"\[(\d+)\]", xml_file).group(1)) + with open(os.path.join(script_dir, xml_file), "r") as file_: + srcString = file_.read() + srcString = unicode(srcString, "utf-8") + # (basestring, IPortUser, basestring) -> list[eos.saveddata.fit.Fit] + usr.on_port_process_start() + stpw.reset() + with stpw: + fits = Port.importXml(srcString, usr) + + assert fits is not None and len(fits) is fit_count