From c560d17bdd22e0be277d428193da0dd9bd161dc3 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Mon, 3 Apr 2017 01:34:07 +0900 Subject: [PATCH 01/11] Improved processing status notification for import and export --- eos/saveddata/ship.py | 5 + gui/mainFrame.py | 83 ++++-- gui/notesView.py | 8 +- service/port.py | 616 ++++++++++++++++++++++++++---------------- utils/strfunctions.py | 34 +++ 5 files changed, 490 insertions(+), 256 deletions(-) create mode 100644 utils/strfunctions.py diff --git a/eos/saveddata/ship.py b/eos/saveddata/ship.py index 1e3fd41f0..10b047c71 100644 --- a/eos/saveddata/ship.py +++ b/eos/saveddata/ship.py @@ -71,6 +71,11 @@ class Ship(ItemAttrShortcut, HandledItem): def item(self): return self.__item + @property + def name(self): + # NOTE: add name property + return self.__item.name + @property def itemModifiedAttributes(self): return self.__itemModifiedAttributes diff --git a/gui/mainFrame.py b/gui/mainFrame.py index 7af37d875..60e2b7f26 100644 --- a/gui/mainFrame.py +++ b/gui/mainFrame.py @@ -72,7 +72,7 @@ from service.update import Update from eos.modifiedAttributeDict import ModifiedAttributeDict from eos.db.saveddata.loadDefaultDatabaseValues import DefaultDatabaseValues from eos.db.saveddata.queries import getFit as db_getFit -from service.port import Port +from service.port import Port, IPortUser from service.settings import HTMLExportSettings from time import gmtime, strftime @@ -137,7 +137,7 @@ class OpenFitsThread(threading.Thread): wx.CallAfter(self.callback) -class MainFrame(wx.Frame): +class MainFrame(wx.Frame, IPortUser): __instance = None @classmethod @@ -419,8 +419,7 @@ class MainFrame(wx.Frame): format_ = dlg.GetFilterIndex() path = dlg.GetPath() if format_ == 0: - sPort = Port.getInstance() - output = sPort.exportXml(None, fit) + output = Port.exportXml(None, fit) if '.' not in os.path.basename(path): path += ".xml" else: @@ -790,7 +789,6 @@ class MainFrame(wx.Frame): def fileImportDialog(self, event): """Handles importing single/multiple EVE XML / EFT cfg fit files""" - sPort = Port.getInstance() dlg = wx.FileDialog( self, "Open One Or More Fitting Files", @@ -804,10 +802,13 @@ class MainFrame(wx.Frame): "Importing fits", " " * 100, # set some arbitrary spacing to create width in window parent=self, - style=wx.PD_APP_MODAL | wx.PD_ELAPSED_TIME + style=wx.PD_CAN_ABORT + | wx.PD_SMOOTH + | wx.PD_ELAPSED_TIME + | wx.PD_APP_MODAL ) - self.progressDialog.message = None - sPort.importFitsThreaded(dlg.GetPaths(), self.fileImportCallback) + # self.progressDialog.message = None + Port.importFitsThreaded(dlg.GetPaths(), self) self.progressDialog.ShowModal() try: dlg.Destroy() @@ -839,9 +840,12 @@ class MainFrame(wx.Frame): "Backing up %d fits to: %s" % (max_, filePath), maximum=max_, parent=self, - style=wx.PD_APP_MODAL | wx.PD_ELAPSED_TIME, + style=wx.PD_CAN_ABORT + | wx.PD_SMOOTH + | wx.PD_ELAPSED_TIME + | wx.PD_APP_MODAL ) - Port().backupFits(filePath, self.backupCallback) + Port.backupFits(filePath, self) self.progressDialog.ShowModal() def exportHtml(self, event): @@ -879,7 +883,17 @@ class MainFrame(wx.Frame): else: self.progressDialog.Update(info) - def fileImportCallback(self, action, data=None): + def onPortProcessStart(self): + # flag for progress dialog. + self.__progress_flag = True + # 2017/03/29 NOTE: implementation like interface + def onPortProcessing(self, action, data=None): + wx.CallAfter( + self._onPortProcessing, action, data + ) + return self.__progress_flag + + def _onPortProcessing(self, action, data): """ While importing fits from file, the logic calls back to this function to update progress bar to show activity. XML files can contain multiple @@ -893,22 +907,39 @@ class MainFrame(wx.Frame): 1: Replace message with data other: Close dialog and handle based on :action (-1 open fits, -2 display error) """ - - if action is None: - self.progressDialog.Pulse() - elif action == 1 and data != self.progressDialog.message: - self.progressDialog.message = data - self.progressDialog.Pulse(data) - else: + _message = None + if action & IPortUser.ID_ERROR: self.closeProgressDialog() - if action == -1: - self._openAfterImport(data) - elif action == -2: - dlg = wx.MessageDialog(self, - "The following error was generated\n\n%s\n\nBe aware that already processed fits were not saved" % data, - "Import Error", wx.OK | wx.ICON_ERROR) - if dlg.ShowModal() == wx.ID_OK: - return + _message = "Import Error" if action & IPortUser.PROCESS_IMPORT else "Export Error" + dlg = wx.MessageDialog(self, + "The following error was generated\n\n%s\n\nBe aware that already processed fits were not saved" % data, + _message, wx.OK | wx.ICON_ERROR) +# if dlg.ShowModal() == wx.ID_OK: +# return + dlg.ShowModal() + return + + # data is str + if action & IPortUser.PROCESS_IMPORT: + if action & IPortUser.ID_PULSE: + _message = () + # update message + elif action & IPortUser.ID_UPDATE: # and data != self.progressDialog.message: + _message = data + + if _message is not None: + self.__progress_flag, _unuse = self.progressDialog.Pulse(_message) + else: + self.closeProgressDialog() + if action & IPortUser.ID_DONE: + self._openAfterImport(data) + # data is tuple(int, str) + elif action & IPortUser.PROCESS_EXPORT: + if action & IPortUser.ID_DONE: + self.closeProgressDialog() + else: + self.__progress_flag, _unuse = self.progressDialog.Update(data[0], data[1]) + def _openAfterImport(self, fits): if len(fits) > 0: diff --git a/gui/notesView.py b/gui/notesView.py index d4f8703cc..031e64941 100644 --- a/gui/notesView.py +++ b/gui/notesView.py @@ -40,8 +40,8 @@ class NotesView(wx.Panel): self.saveTimer.Start(1000, True) def delayedSave(self, event): - sFit = Fit.getInstance() - fit = sFit.getFit(self.lastFitId) - newNotes = self.editNotes.GetValue() - fit.notes = newNotes + fit = Fit.getInstance().getFit(self.lastFitId) + # NOTE: encounter a situation where fit is None + if fit is not None: + fit.notes = self.editNotes.GetValue() wx.PostEvent(self.mainFrame, GE.FitChanged(fitID=fit.ID)) diff --git a/service/port.py b/service/port.py index 3e923ed0f..a63dafd7e 100644 --- a/service/port.py +++ b/service/port.py @@ -1,3 +1,4 @@ +# coding: utf-8 # ============================================================================= # Copyright (C) 2014 Ryan Holmes # @@ -71,7 +72,123 @@ INV_FLAG_DRONEBAY = 87 INV_FLAG_FIGHTER = 158 +from utils.strfunctions import sequential_rep, replaceLTGT +# -- 170327 Ignored description -- +localized_pattern = re.compile(r'([^\*]+)\*') + + +def _resolveShip(fitting, sMkt): + fitobj = Fit() + # 2017/03/29 NOTE: + # if fit name contained "<" or ">" then reprace to named html entity by EVE client + fitobj.name = replaceLTGT(fitting.getAttribute("name")) + # Maelstrom + shipType = fitting.getElementsByTagName("shipType").item(0).getAttribute("value") + emergency = None + matches = localized_pattern.match(shipType) + if matches: + # emergency cache + emergency = matches.group(2) + # expect an official name + shipType = matches.group(1) + limit = 2 + while True: + must_retry = False + try: + try: + fitobj.ship = Ship(sMkt.getItem(shipType)) + except ValueError: + fitobj.ship = Citadel(sMkt.getItem(shipType)) + except Exception as e: + pyfalog.warning("Caught exception on _resolveShip") + pyfalog.error(e) + shipType = emergency + must_retry = True + limit -= 1 + if not must_retry or limit is 0: + break + # True means localized + return matches is not None, fitobj + +def _resolveModule(hardware, sMkt, b_localized): + moduleName = hardware.getAttribute("type") + emergency = None + if b_localized: + emergency = localized_pattern.sub("\g<2>", moduleName) + # expect an official name + moduleName = localized_pattern.sub("\g<1>", moduleName) + item = None + limit = 2 + while True: + must_retry = False + try: + item = sMkt.getItem(moduleName, eager="group.category") + except Exception as e: + pyfalog.warning("Caught exception on _resolveModule") + pyfalog.error(e) + moduleName = emergency + must_retry = True + limit -= 1 + if not must_retry or limit is 0: + break + return item + + +class UserCancelException(Exception): + """when user cancel on port processing.""" + pass + + +from abc import ABCMeta, abstractmethod + + +class IPortUser: + + __metaclass__ = ABCMeta + + ID_PULSE = 1 + # Pulse the progress bar + ID_UPDATE = ID_PULSE << 1 + # Replace message with data: update messate + ID_DONE = ID_PULSE << 2 + # open fits: import process done + ID_ERROR = ID_PULSE << 3 + # display error: raise some error + + PROCESS_IMPORT = ID_PULSE << 4 + # means import process. + PROCESS_EXPORT = ID_PULSE << 5 + # means import process. + + @abstractmethod + def onPortProcessing(self, action, data=None): + """ + While importing fits from file, the logic calls back to this function to + update progress bar to show activity. XML files can contain multiple + ships with multiple fits, whereas EFT cfg files contain many fits of + a single ship. When iterating through the files, we update the message + when we start a new file, and then Pulse the progress bar with every fit + that is processed. + + action : a flag that lets us know how to deal with :data + None: Pulse the progress bar + 1: Replace message with data + other: Close dialog and handle based on :action (-1 open fits, -2 display error) + """ + + """return: True is continue process, False is cancel.""" + pass + + def onPortProcessStart(self): + pass + + class Port(object): + """ + 2017/03/31 NOTE: About change + 1. want to keep the description recorded in fit + 2. i think should not write wx.CallAfter in here + """ instance = None @classmethod @@ -82,19 +199,28 @@ class Port(object): return cls.instance @staticmethod - def backupFits(path, callback): + def backupFits(path, iportuser): pyfalog.debug("Starting backup fits thread.") - thread = FitBackupThread(path, callback) - thread.start() +# thread = FitBackupThread(path, callback) +# thread.start() + threading.Thread( + target=PortProcessing.backupFits, + args=(path, iportuser) + ).start() @staticmethod - def importFitsThreaded(paths, callback): + def importFitsThreaded(paths, iportuser): + """param iportuser: IPortUser implemented class""" pyfalog.debug("Starting import fits thread.") - thread = FitImportThread(paths, callback) - thread.start() +# thread = FitImportThread(paths, iportuser) +# thread.start() + threading.Thread( + target=PortProcessing.importFitsFromFile, + args=(paths, iportuser) + ).start() @staticmethod - def importFitFromFiles(paths, callback=None): + def importFitFromFiles(paths, iportuser=None): """ Imports fits from file(s). First processes all provided paths and stores assembled fits into a list. This allows us to call back to the GUI as @@ -104,96 +230,104 @@ class Port(object): defcodepage = locale.getpreferredencoding() sFit = svcFit.getInstance() - fits = [] - for path in paths: - if callback: # Pulse - pyfalog.debug("Processing file:\n{0}", path) - wx.CallAfter(callback, 1, "Processing file:\n%s" % path) + fit_list = [] + try: + for path in paths: + if iportuser: # Pulse + msg = "Processing file:\n%s" % path + pyfalog.debug(msg) + PortProcessing.notify(iportuser, IPortUser.PROCESS_IMPORT | IPortUser.ID_UPDATE, msg) + #wx.CallAfter(callback, 1, msg) - file_ = open(path, "r") - srcString = file_.read() + file_ = open(path, "r") + srcString = file_.read() - if len(srcString) == 0: # ignore blank files - pyfalog.debug("File is blank.") - continue + if len(srcString) == 0: # ignore blank files + pyfalog.debug("File is blank.") + continue - codec_found = None - # If file had ANSI encoding, decode it to unicode using detection - # of BOM header or if there is no header try default - # codepage then fallback to utf-16, cp1252 + codec_found = None + # If file had ANSI encoding, decode it to unicode using detection + # of BOM header or if there is no header try default + # codepage then fallback to utf-16, cp1252 - if isinstance(srcString, str): - savebom = None + if isinstance(srcString, str): + savebom = None - encoding_map = ( - ('\xef\xbb\xbf', 'utf-8'), - ('\xff\xfe\0\0', 'utf-32'), - ('\0\0\xfe\xff', 'UTF-32BE'), - ('\xff\xfe', 'utf-16'), - ('\xfe\xff', 'UTF-16BE')) + encoding_map = ( + ('\xef\xbb\xbf', 'utf-8'), + ('\xff\xfe\0\0', 'utf-32'), + ('\0\0\xfe\xff', 'UTF-32BE'), + ('\xff\xfe', 'utf-16'), + ('\xfe\xff', 'UTF-16BE')) - for bom, encoding in encoding_map: - if srcString.startswith(bom): - codec_found = encoding - savebom = bom + for bom, encoding in encoding_map: + if srcString.startswith(bom): + codec_found = encoding + savebom = bom + + if codec_found is None: + pyfalog.info("Unicode BOM not found in file {0}.", path) + attempt_codecs = (defcodepage, "utf-8", "utf-16", "cp1252") + + for page in attempt_codecs: + try: + pyfalog.info("Attempting to decode file {0} using {1} page.", path, page) + srcString = unicode(srcString, page) + codec_found = page + pyfalog.info("File {0} decoded using {1} page.", path, page) + except UnicodeDecodeError: + pyfalog.info("Error unicode decoding {0} from page {1}, trying next codec", path, page) + else: + break + else: + pyfalog.info("Unicode BOM detected in {0}, using {1} page.", path, codec_found) + srcString = unicode(srcString[len(savebom):], codec_found) + + else: + # nasty hack to detect other transparent utf-16 loading + if srcString[0] == '<' and 'utf-16' in srcString[:128].lower(): + codec_found = "utf-16" + else: + codec_found = "utf-8" if codec_found is None: - pyfalog.info("Unicode BOM not found in file {0}.", path) - attempt_codecs = (defcodepage, "utf-8", "utf-16", "cp1252") + return False, "Proper codec could not be established for %s" % path - for page in attempt_codecs: - try: - pyfalog.info("Attempting to decode file {0} using {1} page.", path, page) - srcString = unicode(srcString, page) - codec_found = page - pyfalog.info("File {0} decoded using {1} page.", path, page) - except UnicodeDecodeError: - pyfalog.info("Error unicode decoding {0} from page {1}, trying next codec", path, page) - else: - break - else: - pyfalog.info("Unicode BOM detected in {0}, using {1} page.", path, codec_found) - srcString = unicode(srcString[len(savebom):], codec_found) + try: + _, fitsImport = Port.importAuto(srcString, path, iportuser=iportuser, encoding=codec_found) + fit_list += fitsImport + except xml.parsers.expat.ExpatError: + pyfalog.warning("Malformed XML in:\n{0}", path) + return False, "Malformed XML in %s" % path - else: - # nasty hack to detect other transparent utf-16 loading - if srcString[0] == '<' and 'utf-16' in srcString[:128].lower(): - codec_found = "utf-16" - else: - codec_found = "utf-8" + # IDs = [] # NOTE: what use for IDs? + numFits = len(fit_list) + for idx, fit in enumerate(fit_list): + # Set some more fit attributes and save + fit.character = sFit.character + fit.damagePattern = sFit.pattern + fit.targetResists = sFit.targetResists + db.save(fit) + # IDs.append(fit.ID) + if iportuser: # Pulse + pyfalog.debug("Processing complete, saving fits to database: {0}/{1}", idx + 1, numFits) + PortProcessing.notify( + iportuser, IPortUser.PROCESS_IMPORT | IPortUser.ID_UPDATE, + "Processing complete, saving fits to database\n(%d/%d) %s" % (idx + 1, numFits, fit.ship.name) + ) - if codec_found is None: - return False, "Proper codec could not be established for %s" % path + except UserCancelException: + return False, "Processing has been canceled.\n" + except Exception as e: + pyfalog.critical("Unknown exception processing: {0}", path) + pyfalog.critical(e) + # TypeError: not all arguments converted during string formatting +# return False, "Unknown Error while processing {0}" % path + return False, "Unknown Error while processing %s\n\n message: %s" % (path, e.message) - try: - _, fitsImport = Port.importAuto(srcString, path, callback=callback, encoding=codec_found) - fits += fitsImport - except xml.parsers.expat.ExpatError: - pyfalog.warning("Malformed XML in:\n{0}", path) - return False, "Malformed XML in %s" % path - except Exception as e: - pyfalog.critical("Unknown exception processing: {0}", path) - pyfalog.critical(e) - return False, "Unknown Error while processing {0}" % path + return True, fit_list - IDs = [] - numFits = len(fits) - for i, fit in enumerate(fits): - # Set some more fit attributes and save - fit.character = sFit.character - fit.damagePattern = sFit.pattern - fit.targetResists = sFit.targetResists - db.save(fit) - IDs.append(fit.ID) - if callback: # Pulse - pyfalog.debug("Processing complete, saving fits to database: {0}/{1}", i + 1, numFits) - wx.CallAfter( - callback, 1, - "Processing complete, saving fits to database\n(%d/%d)" % - (i + 1, numFits) - ) - - return True, fits @staticmethod def importFitFromBuffer(bufferStr, activeFit=None): @@ -228,7 +362,9 @@ class Port(object): fit['ship']['id'] = ofit.ship.item.ID fit['ship']['name'] = '' - fit['description'] = "" % ofit.ID + # 2017/03/29 NOTE: "<" or "<" is Ignored + # fit['description'] = "" % ofit.ID + fit['description'] = ofit.notes if ofit.notes is not None else "" fit['items'] = [] slotNum = {} @@ -302,7 +438,7 @@ class Port(object): return json.dumps(fit) @classmethod - def importAuto(cls, string, path=None, activeFit=None, callback=None, encoding=None): + def importAuto(cls, string, path=None, activeFit=None, iportuser=None, encoding=None): # Get first line and strip space symbols of it to avoid possible detection errors firstLine = re.split("[\n\r]+", string.strip(), maxsplit=1)[0] firstLine = firstLine.strip() @@ -310,9 +446,9 @@ class Port(object): # If XML-style start of tag encountered, detect as XML if re.match("<", firstLine): if encoding: - return "XML", cls.importXml(string, callback, encoding) + return "XML", cls.importXml(string, iportuser, encoding) else: - return "XML", cls.importXml(string, callback) + return "XML", cls.importXml(string, iportuser) # If JSON-style start, parse as CREST/JSON if firstLine[0] == '{': @@ -323,7 +459,7 @@ class Port(object): if re.match("\[.*\]", firstLine) and path is not None: filename = os.path.split(path)[1] shipName = filename.rsplit('.')[0] - return "EFT Config", cls.importEftCfg(shipName, string, callback) + return "EFT Config", cls.importEftCfg(shipName, string, iportuser) # If no file is specified and there's comma between brackets, # consider that we have [ship, setup name] and detect like eft export format @@ -335,22 +471,26 @@ class Port(object): @staticmethod def importCrest(str_): - fit = json.loads(str_) - sMkt = Market.getInstance() - f = Fit() - f.name = fit['name'] + sMkt = Market.getInstance() + fitobj = Fit() + refobj = json.loads(str_) + items = refobj['items'] + fitobj.name = refobj['name'] + # 2017/03/29: read description + # "<" and ">" is replace to "<", ">" by EVE client + fitobj.notes = refobj['description'] try: + refobj = refobj['ship']['id'] try: - f.ship = Ship(sMkt.getItem(fit['ship']['id'])) + fitobj.ship = Ship(sMkt.getItem(refobj)) except ValueError: - f.ship = Citadel(sMkt.getItem(fit['ship']['id'])) + fitobj.ship = Citadel(sMkt.getItem(refobj)) except: pyfalog.warning("Caught exception in importCrest") return None - items = fit['items'] items.sort(key=lambda k: k['flag']) moduleList = [] @@ -360,14 +500,14 @@ class Port(object): if module['flag'] == INV_FLAG_DRONEBAY: d = Drone(item) d.amount = module['quantity'] - f.drones.append(d) + fitobj.drones.append(d) elif module['flag'] == INV_FLAG_CARGOBAY: c = Cargo(item) c.amount = module['quantity'] - f.cargo.append(c) + fitobj.cargo.append(c) elif module['flag'] == INV_FLAG_FIGHTER: fighter = Fighter(item) - f.fighters.append(fighter) + fitobj.fighters.append(fighter) else: try: m = Module(item) @@ -377,8 +517,8 @@ class Port(object): continue # Add subsystems before modules to make sure T3 cruisers have subsystems installed if item.category.name == "Subsystem": - if m.fits(f): - f.modules.append(m) + if m.fits(fitobj): + fitobj.modules.append(m) else: if m.isValidState(State.ACTIVE): m.state = State.ACTIVE @@ -390,13 +530,13 @@ class Port(object): continue # Recalc to get slot numbers correct for T3 cruisers - svcFit.getInstance().recalc(f) + svcFit.getInstance().recalc(fitobj) for module in moduleList: - if module.fits(f): - f.modules.append(module) + if module.fits(fitobj): + fitobj.modules.append(module) - return f + return fitobj @staticmethod def importDna(string): @@ -635,7 +775,7 @@ class Port(object): return fit @staticmethod - def importEftCfg(shipname, contents, callback=None): + def importEftCfg(shipname, contents, iportuser=None): """Handle import from EFT config store file""" # Check if we have such ship in database, bail if we don't @@ -648,7 +788,7 @@ class Port(object): # If client didn't take care of encoding file contents into Unicode, # do it using fallback encoding ourselves if isinstance(contents, str): - contents = unicode(contents, "cp1252") + contents = unicode(contents, locale.getpreferredencoding()) fits = [] # List for fits fitIndices = [] # List for starting line numbers for each fit @@ -671,14 +811,14 @@ class Port(object): try: # Create fit object - f = Fit() + fitobj = Fit() # Strip square brackets and pull out a fit name - f.name = fitLines[0][1:-1] + fitobj.name = fitLines[0][1:-1] # Assign ship to fitting try: - f.ship = Ship(sMkt.getItem(shipname)) + fitobj.ship = Ship(sMkt.getItem(shipname)) except ValueError: - f.ship = Citadel(sMkt.getItem(shipname)) + fitobj.ship = Citadel(sMkt.getItem(shipname)) moduleList = [] for x in range(1, len(fitLines)): @@ -689,6 +829,8 @@ class Port(object): # Parse line into some data we will need misc = re.match("(Drones|Implant|Booster)_(Active|Inactive)=(.+)", line) cargo = re.match("Cargohold=(.+)", line) + # 2017/03/27 NOTE: store description from EFT + description = re.match("Description=(.+)", line) if misc: entityType = misc.group(1) @@ -713,11 +855,11 @@ class Port(object): d.amountActive = droneAmount elif entityState == "Inactive": d.amountActive = 0 - f.drones.append(d) + fitobj.drones.append(d) elif droneItem.category.name == "Fighter": # EFT saves fighter as drones ft = Fighter(droneItem) ft.amount = int(droneAmount) if ft.amount <= ft.fighterSquadronMaxSize else ft.fighterSquadronMaxSize - f.fighters.append(ft) + fitobj.fighters.append(ft) else: continue elif entityType == "Implant": @@ -735,7 +877,7 @@ class Port(object): imp.active = True elif entityState == "Inactive": imp.active = False - f.implants.append(imp) + fitobj.implants.append(imp) elif entityType == "Booster": # Bail if we can't get item or it's not from implant category try: @@ -752,7 +894,7 @@ class Port(object): b.active = True elif entityState == "Inactive": b.active = False - f.boosters.append(b) + fitobj.boosters.append(b) # If we don't have any prefixes, then it's a module elif cargo: cargoData = re.match("(.+),([0-9]+)", cargo.group(1)) @@ -767,7 +909,10 @@ class Port(object): # Add Cargo to the fitting c = Cargo(item) c.amount = cargoAmount - f.cargo.append(c) + fitobj.cargo.append(c) + # 2017/03/27 NOTE: store description from EFT + elif description: + fitobj.notes = description.group(1).replace("|", "\n") else: withCharge = re.match("(.+),(.+)", line) modName = withCharge.group(1) if withCharge else line @@ -784,10 +929,10 @@ class Port(object): # Add subsystems before modules to make sure T3 cruisers have subsystems installed if modItem.category.name == "Subsystem": - if m.fits(f): - f.modules.append(m) + if m.fits(fitobj): + fitobj.modules.append(m) else: - m.owner = f + m.owner = fitobj # Activate mod if it is activable if m.isValidState(State.ACTIVE): m.state = State.ACTIVE @@ -804,17 +949,21 @@ class Port(object): moduleList.append(m) # Recalc to get slot numbers correct for T3 cruisers - svcFit.getInstance().recalc(f) + svcFit.getInstance().recalc(fitobj) for module in moduleList: - if module.fits(f): - f.modules.append(module) + if module.fits(fitobj): + fitobj.modules.append(module) # Append fit to list of fits - fits.append(f) + fits.append(fitobj) + + if iportuser: # NOTE: Send current processing status + PortProcessing.notify( + iportuser, IPortUser.PROCESS_IMPORT | IPortUser.ID_UPDATE, + "%s:\n%s" % (fitobj.ship.name, fitobj.name) + ) - if callback: - wx.CallAfter(callback, None) # Skip fit silently if we get an exception except Exception as e: pyfalog.error("Caught exception on fit.") @@ -824,93 +973,95 @@ class Port(object): return fits @staticmethod - def importXml(text, callback=None, encoding="utf-8"): - sMkt = Market.getInstance() + def importXml(text, iportuser=None, encoding="utf-8"): + sMkt = Market.getInstance() doc = xml.dom.minidom.parseString(text.encode(encoding)) fittings = doc.getElementsByTagName("fittings").item(0) fittings = fittings.getElementsByTagName("fitting") - fits = [] + fit_list = [] + + for fitting in (fittings): + b_localized, fitobj = _resolveShip(fitting, sMkt) + # -- 170327 Ignored description -- + # read description from exported xml. (EVE client, EFT) + description = fitting.getElementsByTagName("description").item(0).getAttribute("value") + if description is None: + description = "" + elif len(description): + # convert
to "\n" and remove html tags. + description = sequential_rep(description, r"<(br|BR)>", "\n",r"<[^<>]+>", "") +# description = re.sub(r"<(br|BR)>", "\n", description) +# description = re.sub(r"<[^<>]+>", "", description) + fitobj.notes = description - for i, fitting in enumerate(fittings): - f = Fit() - f.name = fitting.getAttribute("name") - # Maelstrom - shipType = fitting.getElementsByTagName("shipType").item(0).getAttribute("value") - try: - try: - f.ship = Ship(sMkt.getItem(shipType)) - except ValueError: - f.ship = Citadel(sMkt.getItem(shipType)) - except Exception as e: - pyfalog.warning("Caught exception on importXml") - pyfalog.error(e) - continue hardwares = fitting.getElementsByTagName("hardware") moduleList = [] for hardware in hardwares: try: - moduleName = hardware.getAttribute("type") - try: - item = sMkt.getItem(moduleName, eager="group.category") - except Exception as e: - pyfalog.warning("Caught exception on importXml") - pyfalog.error(e) + item = _resolveModule(hardware, sMkt, b_localized) + if not item: continue - if item: - if item.category.name == "Drone": - d = Drone(item) - d.amount = int(hardware.getAttribute("qty")) - f.drones.append(d) - elif item.category.name == "Fighter": - ft = Fighter(item) - ft.amount = int(hardware.getAttribute("qty")) if ft.amount <= ft.fighterSquadronMaxSize else ft.fighterSquadronMaxSize - f.fighters.append(ft) - elif hardware.getAttribute("slot").lower() == "cargo": - # although the eve client only support charges in cargo, third-party programs - # may support items or "refits" in cargo. Support these by blindly adding all - # cargo, not just charges - c = Cargo(item) - c.amount = int(hardware.getAttribute("qty")) - f.cargo.append(c) - else: - try: - m = Module(item) - # When item can't be added to any slot (unknown item or just charge), ignore it - except ValueError: - pyfalog.warning("item can't be added to any slot (unknown item or just charge), ignore it") - continue - # Add subsystems before modules to make sure T3 cruisers have subsystems installed - if item.category.name == "Subsystem": - if m.fits(f): - m.owner = f - f.modules.append(m) - else: - if m.isValidState(State.ACTIVE): - m.state = State.ACTIVE - moduleList.append(m) + if item.category.name == "Drone": + d = Drone(item) + d.amount = int(hardware.getAttribute("qty")) + fitobj.drones.append(d) + elif item.category.name == "Fighter": + ft = Fighter(item) + ft.amount = int(hardware.getAttribute("qty")) if ft.amount <= ft.fighterSquadronMaxSize else ft.fighterSquadronMaxSize + fitobj.fighters.append(ft) + elif hardware.getAttribute("slot").lower() == "cargo": + # although the eve client only support charges in cargo, third-party programs + # may support items or "refits" in cargo. Support these by blindly adding all + # cargo, not just charges + c = Cargo(item) + c.amount = int(hardware.getAttribute("qty")) + fitobj.cargo.append(c) + else: + try: + m = Module(item) + # When item can't be added to any slot (unknown item or just charge), ignore it + except ValueError: + pyfalog.warning("item can't be added to any slot (unknown item or just charge), ignore it") + continue + # Add subsystems before modules to make sure T3 cruisers have subsystems installed + if item.category.name == "Subsystem": + if m.fits(fitobj): + m.owner = fitobj + fitobj.modules.append(m) + else: + if m.isValidState(State.ACTIVE): + m.state = State.ACTIVE + + moduleList.append(m) except KeyboardInterrupt: pyfalog.warning("Keyboard Interrupt") continue # Recalc to get slot numbers correct for T3 cruisers - svcFit.getInstance().recalc(f) + svcFit.getInstance().recalc(fitobj) for module in moduleList: - if module.fits(f): - module.owner = f - f.modules.append(module) + if module.fits(fitobj): + module.owner = fitobj + fitobj.modules.append(module) - fits.append(f) - if callback: - wx.CallAfter(callback, None) + fit_list.append(fitobj) + if iportuser: # NOTE: Send current processing status + PortProcessing.notify( + iportuser, IPortUser.PROCESS_IMPORT | IPortUser.ID_UPDATE, + "analysis :%s\n%s" % (fitobj.ship.name, fitobj.name) + ) - return fits + return fit_list @staticmethod def _exportEftBase(fit): + """Basically EFT format does not require blank lines + also, it's OK to arrange modules randomly? + """ offineSuffix = " /OFFLINE" export = "[%s, %s]\n" % (fit.ship.item.name, fit.name) stuff = {} @@ -1038,10 +1189,13 @@ class Port(object): return dna + "::" - @classmethod - def exportXml(cls, callback=None, *fits): + @staticmethod + def exportXml(iportuser=None, *fits): doc = xml.dom.minidom.Document() fittings = doc.createElement("fittings") + # fit count + fit_count = len(fits) + fittings.setAttribute("count", "%s" % fit_count) doc.appendChild(fittings) sFit = svcFit.getInstance() @@ -1051,10 +1205,18 @@ class Port(object): fitting.setAttribute("name", fit.name) fittings.appendChild(fitting) description = doc.createElement("description") - description.setAttribute("value", "") + # -- 170327 Ignored description -- + try: + notes = fit.notes # unicode + description.setAttribute( + "value", re.sub("[\r\n]+", "
", notes) if notes is not None else "" + ) + except Exception as e: + pyfalog.warning("read description is failed, msg=%s\n" % e.args) + fitting.appendChild(description) shipType = doc.createElement("shipType") - shipType.setAttribute("value", fit.ship.item.name) + shipType.setAttribute("value", fit.ship.name) fitting.appendChild(shipType) charges = {} @@ -1113,12 +1275,17 @@ class Port(object): hardware.setAttribute("slot", "cargo") hardware.setAttribute("type", name) fitting.appendChild(hardware) - except: - print("Failed on fitID: %d" % fit.ID) + except Exception as e: +# print("Failed on fitID: %d" % fit.ID) + pyfalog.error("Failed on fitID: %d, message: %s" % e.message) continue finally: - if callback: - wx.CallAfter(callback, i) + if iportuser: + PortProcessing.notify( + iportuser, IPortUser.PROCESS_EXPORT | IPortUser.ID_UPDATE, + (i, "convert to xml (%s/%s) %s" % (i + 1, fit_count, fit.ship.name)) + ) +# wx.CallAfter(callback, i, "(%s/%s) %s" % (i, fit_count, fit.ship.name)) return doc.toprettyxml() @@ -1169,37 +1336,34 @@ class Port(object): return export -class FitBackupThread(threading.Thread): - def __init__(self, path, callback): - threading.Thread.__init__(self) - self.path = path - self.callback = callback - - def run(self): - path = self.path - sFit = svcFit.getInstance() - sPort = Port.getInstance() - backedUpFits = sPort.exportXml(self.callback, *sFit.getAllFits()) - backupFile = open(path, "w", encoding="utf-8") - backupFile.write(backedUpFits) - backupFile.close() - +class PortProcessing(object): + """Port Processing class """ + @staticmethod + def backupFits(path, iportuser): + success = True + try: + iportuser.onPortProcessStart() + backedUpFits = Port.exportXml(iportuser, *svcFit.getInstance().getAllFits()) + backupFile = open(path, "w", encoding="utf-8") + backupFile.write(backedUpFits) + backupFile.close() + except UserCancelException: + success = False # Send done signal to GUI - wx.CallAfter(self.callback, -1) +# wx.CallAfter(callback, -1, "Done.") + flag = IPortUser.ID_ERROR if not success else IPortUser.ID_DONE + iportuser.onPortProcessing(IPortUser.PROCESS_EXPORT | flag, + "User canceled or some error occurrence." if not success else "Done.") + @staticmethod + def importFitsFromFile(paths, iportuser): + iportuser.onPortProcessStart() + success, result = Port.importFitFromFiles(paths, iportuser) + flag = IPortUser.ID_ERROR if not success else IPortUser.ID_DONE + iportuser.onPortProcessing(IPortUser.PROCESS_IMPORT | flag, result) -class FitImportThread(threading.Thread): - def __init__(self, paths, callback): - threading.Thread.__init__(self) - self.paths = paths - self.callback = callback + @staticmethod + def notify(iportuser, flag, data): + if not iportuser.onPortProcessing(flag, data): + raise UserCancelException - def run(self): - sPort = Port.getInstance() - success, result = sPort.importFitFromFiles(self.paths, self.callback) - - if not success: # there was an error during processing - pyfalog.error("Error while processing file import: {0}", result) - wx.CallAfter(self.callback, -2, result) - else: # Send done signal to GUI - wx.CallAfter(self.callback, -1, result) diff --git a/utils/strfunctions.py b/utils/strfunctions.py new file mode 100644 index 000000000..a1decbd1c --- /dev/null +++ b/utils/strfunctions.py @@ -0,0 +1,34 @@ +''' + string manipulation module +''' +import re + + +def sequential_rep(text_, *args): + """ + params + text_: string content + args : , , , , ... + + return + empty string when text_ length was zero or invalid. + """ + + if not text_ or not len(text_): + return "" + + arg_len = len(args) + i = 0 + while i < arg_len: + text_ = re.sub(args[i], args[i + 1], text_) + i += 2 + + return text_ + +def replaceLTGT(text_): + """if fit name contained "<" or ">" then reprace to named html entity by EVE client. + + for fit name. + """ + return text_.replace("<", "<").replace(">", ">") if isinstance(text_, unicode) else text_ + From 4d8c3eadbc1d6e6758c9aae9a3ede37c328e1e9d Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Mon, 3 Apr 2017 01:50:26 +0900 Subject: [PATCH 02/11] pep8 --- eos/saveddata/ship.py | 2 +- gui/mainFrame.py | 14 ++++---------- service/port.py | 7 +++---- utils/strfunctions.py | 2 +- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/eos/saveddata/ship.py b/eos/saveddata/ship.py index 10b047c71..d2b7b7119 100644 --- a/eos/saveddata/ship.py +++ b/eos/saveddata/ship.py @@ -73,7 +73,7 @@ class Ship(ItemAttrShortcut, HandledItem): @property def name(self): - # NOTE: add name property + # NOTE: add name property return self.__item.name @property diff --git a/gui/mainFrame.py b/gui/mainFrame.py index 60e2b7f26..901c6fecb 100644 --- a/gui/mainFrame.py +++ b/gui/mainFrame.py @@ -802,10 +802,7 @@ class MainFrame(wx.Frame, IPortUser): "Importing fits", " " * 100, # set some arbitrary spacing to create width in window parent=self, - style=wx.PD_CAN_ABORT - | wx.PD_SMOOTH - | wx.PD_ELAPSED_TIME - | wx.PD_APP_MODAL + style=wx.PD_CAN_ABORT | wx.PD_SMOOTH | wx.PD_ELAPSED_TIME | wx.PD_APP_MODAL ) # self.progressDialog.message = None Port.importFitsThreaded(dlg.GetPaths(), self) @@ -840,10 +837,7 @@ class MainFrame(wx.Frame, IPortUser): "Backing up %d fits to: %s" % (max_, filePath), maximum=max_, parent=self, - style=wx.PD_CAN_ABORT - | wx.PD_SMOOTH - | wx.PD_ELAPSED_TIME - | wx.PD_APP_MODAL + style=wx.PD_CAN_ABORT | wx.PD_SMOOTH | wx.PD_ELAPSED_TIME | wx.PD_APP_MODAL ) Port.backupFits(filePath, self) self.progressDialog.ShowModal() @@ -891,6 +885,7 @@ class MainFrame(wx.Frame, IPortUser): wx.CallAfter( self._onPortProcessing, action, data ) + return self.__progress_flag def _onPortProcessing(self, action, data): @@ -924,7 +919,7 @@ class MainFrame(wx.Frame, IPortUser): if action & IPortUser.ID_PULSE: _message = () # update message - elif action & IPortUser.ID_UPDATE: # and data != self.progressDialog.message: + elif action & IPortUser.ID_UPDATE: # and data != self.progressDialog.message: _message = data if _message is not None: @@ -940,7 +935,6 @@ class MainFrame(wx.Frame, IPortUser): else: self.__progress_flag, _unuse = self.progressDialog.Update(data[0], data[1]) - def _openAfterImport(self, fits): if len(fits) > 0: if len(fits) == 1: diff --git a/service/port.py b/service/port.py index a63dafd7e..aa3776ff3 100644 --- a/service/port.py +++ b/service/port.py @@ -47,6 +47,8 @@ from eos.saveddata.ship import Ship from eos.saveddata.citadel import Citadel from eos.saveddata.fit import Fit from service.market import Market +from utils.strfunctions import sequential_rep, replaceLTGT +from abc import ABCMeta, abstractmethod if 'wxMac' not in wx.PlatformInfo or ('wxMac' in wx.PlatformInfo and wx.VERSION >= (3, 0)): from service.crest import Crest @@ -72,7 +74,6 @@ INV_FLAG_DRONEBAY = 87 INV_FLAG_FIGHTER = 158 -from utils.strfunctions import sequential_rep, replaceLTGT # -- 170327 Ignored description -- localized_pattern = re.compile(r'([^\*]+)\*') @@ -110,6 +111,7 @@ def _resolveShip(fitting, sMkt): # True means localized return matches is not None, fitobj + def _resolveModule(hardware, sMkt, b_localized): moduleName = hardware.getAttribute("type") emergency = None @@ -139,9 +141,6 @@ class UserCancelException(Exception): pass -from abc import ABCMeta, abstractmethod - - class IPortUser: __metaclass__ = ABCMeta diff --git a/utils/strfunctions.py b/utils/strfunctions.py index a1decbd1c..cd769d20b 100644 --- a/utils/strfunctions.py +++ b/utils/strfunctions.py @@ -25,10 +25,10 @@ def sequential_rep(text_, *args): return text_ + def replaceLTGT(text_): """if fit name contained "<" or ">" then reprace to named html entity by EVE client. for fit name. """ return text_.replace("<", "<").replace(">", ">") if isinstance(text_, unicode) else text_ - From 8fd424814a9adf9f7d098f9a9749e53674404907 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Mon, 3 Apr 2017 02:00:10 +0900 Subject: [PATCH 03/11] edit... --- gui/mainFrame.py | 7 ++++--- service/port.py | 18 ++++++++---------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/gui/mainFrame.py b/gui/mainFrame.py index 901c6fecb..fcc901b3b 100644 --- a/gui/mainFrame.py +++ b/gui/mainFrame.py @@ -880,8 +880,9 @@ class MainFrame(wx.Frame, IPortUser): def onPortProcessStart(self): # flag for progress dialog. self.__progress_flag = True - # 2017/03/29 NOTE: implementation like interface + def onPortProcessing(self, action, data=None): + # 2017/03/29 NOTE: implementation like interface wx.CallAfter( self._onPortProcessing, action, data ) @@ -909,8 +910,8 @@ class MainFrame(wx.Frame, IPortUser): dlg = wx.MessageDialog(self, "The following error was generated\n\n%s\n\nBe aware that already processed fits were not saved" % data, _message, wx.OK | wx.ICON_ERROR) -# if dlg.ShowModal() == wx.ID_OK: -# return + # if dlg.ShowModal() == wx.ID_OK: + # return dlg.ShowModal() return diff --git a/service/port.py b/service/port.py index aa3776ff3..b2b7163c1 100644 --- a/service/port.py +++ b/service/port.py @@ -145,13 +145,13 @@ class IPortUser: __metaclass__ = ABCMeta - ID_PULSE = 1 + ID_PULSE = 1 # Pulse the progress bar ID_UPDATE = ID_PULSE << 1 # Replace message with data: update messate - ID_DONE = ID_PULSE << 2 + ID_DONE = ID_PULSE << 2 # open fits: import process done - ID_ERROR = ID_PULSE << 3 + ID_ERROR = ID_PULSE << 3 # display error: raise some error PROCESS_IMPORT = ID_PULSE << 4 @@ -236,7 +236,7 @@ class Port(object): msg = "Processing file:\n%s" % path pyfalog.debug(msg) PortProcessing.notify(iportuser, IPortUser.PROCESS_IMPORT | IPortUser.ID_UPDATE, msg) - #wx.CallAfter(callback, 1, msg) + # wx.CallAfter(callback, 1, msg) file_ = open(path, "r") srcString = file_.read() @@ -327,7 +327,6 @@ class Port(object): return True, fit_list - @staticmethod def importFitFromBuffer(bufferStr, activeFit=None): sFit = svcFit.getInstance() @@ -471,10 +470,10 @@ class Port(object): @staticmethod def importCrest(str_): - sMkt = Market.getInstance() + sMkt = Market.getInstance() fitobj = Fit() refobj = json.loads(str_) - items = refobj['items'] + items = refobj['items'] fitobj.name = refobj['name'] # 2017/03/29: read description # "<" and ">" is replace to "<", ">" by EVE client @@ -989,7 +988,7 @@ class Port(object): description = "" elif len(description): # convert
to "\n" and remove html tags. - description = sequential_rep(description, r"<(br|BR)>", "\n",r"<[^<>]+>", "") + description = sequential_rep(description, r"<(br|BR)>", "\n", r"<[^<>]+>", "") # description = re.sub(r"<(br|BR)>", "\n", description) # description = re.sub(r"<[^<>]+>", "", description) fitobj.notes = description @@ -1275,7 +1274,7 @@ class Port(object): hardware.setAttribute("type", name) fitting.appendChild(hardware) except Exception as e: -# print("Failed on fitID: %d" % fit.ID) + # print("Failed on fitID: %d" % fit.ID) pyfalog.error("Failed on fitID: %d, message: %s" % e.message) continue finally: @@ -1365,4 +1364,3 @@ class PortProcessing(object): def notify(iportuser, flag, data): if not iportuser.onPortProcessing(flag, data): raise UserCancelException - From e9e8ef964cab55008453b7be5a0e291653fa2443 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Tue, 4 Apr 2017 21:23:28 +0900 Subject: [PATCH 04/11] refactoring some code and added docstring added stopwatch.py(modify from utils.timer.py) for test --- gui/mainFrame.py | 8 +-- service/port.py | 121 ++++++++++++++++++++++++++++-------------- utils/stopwatch.py | 107 +++++++++++++++++++++++++++++++++++++ utils/strfunctions.py | 32 +++++------ 4 files changed, 206 insertions(+), 62 deletions(-) create mode 100644 utils/stopwatch.py diff --git a/gui/mainFrame.py b/gui/mainFrame.py index fcc901b3b..777c608b5 100644 --- a/gui/mainFrame.py +++ b/gui/mainFrame.py @@ -877,19 +877,19 @@ class MainFrame(wx.Frame, IPortUser): else: self.progressDialog.Update(info) - def onPortProcessStart(self): + def on_port_process_start(self): # flag for progress dialog. self.__progress_flag = True - def onPortProcessing(self, action, data=None): + def on_port_processing(self, action, data=None): # 2017/03/29 NOTE: implementation like interface wx.CallAfter( - self._onPortProcessing, action, data + self._on_port_processing, action, data ) return self.__progress_flag - def _onPortProcessing(self, action, data): + def _on_port_processing(self, action, data): """ While importing fits from file, the logic calls back to this function to update progress bar to show activity. XML files can contain multiple diff --git a/service/port.py b/service/port.py index b2b7163c1..b7bdd95ac 100644 --- a/service/port.py +++ b/service/port.py @@ -47,7 +47,7 @@ from eos.saveddata.ship import Ship from eos.saveddata.citadel import Citadel from eos.saveddata.fit import Fit from service.market import Market -from utils.strfunctions import sequential_rep, replaceLTGT +from utils.strfunctions import sequential_rep, replace_ltgt from abc import ABCMeta, abstractmethod if 'wxMac' not in wx.PlatformInfo or ('wxMac' in wx.PlatformInfo and wx.VERSION >= (3, 0)): @@ -73,25 +73,39 @@ INV_FLAG_CARGOBAY = 5 INV_FLAG_DRONEBAY = 87 INV_FLAG_FIGHTER = 158 +# 2017/04/05 NOTE: simple validation, for xml file +RE_XML_START = r'<\?xml\s+version="1.0"\s+\?>' # -- 170327 Ignored description -- -localized_pattern = re.compile(r'([^\*]+)\*') +RE_LTGT = "&(lt|gt);" +L_MARK = "<localized hint="" +LOCALIZED_PATTERN = re.compile(r'([^\*]+)\*') -def _resolveShip(fitting, sMkt): +def _extract_matche(t): + m = LOCALIZED_PATTERN.match(t) + # hint attribute, text content + return m.group(1), m.group(2) + + +def _resolve_ship(fitting, sMkt, b_localized): + # type: (xml.dom.minidom.Element, Market, bool) -> eos.saveddata.fit.Fit + fitobj = Fit() + fitobj.name = fitting.getAttribute("name") # 2017/03/29 NOTE: # if fit name contained "<" or ">" then reprace to named html entity by EVE client - fitobj.name = replaceLTGT(fitting.getAttribute("name")) + # if re.search(RE_LTGT, fitobj.name): + if "<" in fitobj.name or ">" in fitobj.name: + fitobj.name = replace_ltgt(fitobj.name) + # Maelstrom shipType = fitting.getElementsByTagName("shipType").item(0).getAttribute("value") - emergency = None - matches = localized_pattern.match(shipType) - if matches: - # emergency cache - emergency = matches.group(2) - # expect an official name - shipType = matches.group(1) + # emergency = None + if b_localized: + # expect an official name, emergency cache + shipType, emergency = _extract_matche(shipType) + limit = 2 while True: must_retry = False @@ -101,24 +115,28 @@ def _resolveShip(fitting, sMkt): except ValueError: fitobj.ship = Citadel(sMkt.getItem(shipType)) except Exception as e: - pyfalog.warning("Caught exception on _resolveShip") + pyfalog.warning("Caught exception on _resolve_ship") pyfalog.error(e) + limit -= 1 + if limit is 0: + break shipType = emergency must_retry = True - limit -= 1 - if not must_retry or limit is 0: + if not must_retry: break - # True means localized - return matches is not None, fitobj + + return fitobj -def _resolveModule(hardware, sMkt, b_localized): +def _resolve_module(hardware, sMkt, b_localized): + # type: (xml.dom.minidom.Element, Market, bool) -> eos.saveddata.module.Module + moduleName = hardware.getAttribute("type") - emergency = None + # emergency = None if b_localized: - emergency = localized_pattern.sub("\g<2>", moduleName) - # expect an official name - moduleName = localized_pattern.sub("\g<1>", moduleName) + # expect an official name, emergency cache + moduleName, emergency = _extract_matche(moduleName) + item = None limit = 2 while True: @@ -126,12 +144,14 @@ def _resolveModule(hardware, sMkt, b_localized): try: item = sMkt.getItem(moduleName, eager="group.category") except Exception as e: - pyfalog.warning("Caught exception on _resolveModule") + pyfalog.warning("Caught exception on _resolve_module") pyfalog.error(e) + limit -= 1 + if limit is 0: + break moduleName = emergency must_retry = True - limit -= 1 - if not must_retry or limit is 0: + if not must_retry: break return item @@ -160,7 +180,7 @@ class IPortUser: # means import process. @abstractmethod - def onPortProcessing(self, action, data=None): + def on_port_processing(self, action, data=None): """ While importing fits from file, the logic calls back to this function to update progress bar to show activity. XML files can contain multiple @@ -178,7 +198,7 @@ class IPortUser: """return: True is continue process, False is cancel.""" pass - def onPortProcessStart(self): + def on_port_process_start(self): pass @@ -189,6 +209,7 @@ class Port(object): 2. i think should not write wx.CallAfter in here """ instance = None + __tag_replace_flag = True @classmethod def getInstance(cls): @@ -197,6 +218,16 @@ class Port(object): return cls.instance + @classmethod + def set_tag_replace(cls, b): + cls.__tag_replace_flag = b + + @classmethod + def is_tag_replace(cls): + # might there is a person who wants to hold tags. + # (item link in EVE client etc. When importing again to EVE) + return cls.__tag_replace_flag + @staticmethod def backupFits(path, iportuser): pyfalog.debug("Starting backup fits thread.") @@ -209,7 +240,12 @@ class Port(object): @staticmethod def importFitsThreaded(paths, iportuser): - """param iportuser: IPortUser implemented class""" + # type: (tuple, IPortUser) -> None + """ + :param paths: fits data file path list. + :param iportuser: IPortUser implemented class. + :rtype: None + """ pyfalog.debug("Starting import fits thread.") # thread = FitImportThread(paths, iportuser) # thread.start() @@ -238,8 +274,8 @@ class Port(object): PortProcessing.notify(iportuser, IPortUser.PROCESS_IMPORT | IPortUser.ID_UPDATE, msg) # wx.CallAfter(callback, 1, msg) - file_ = open(path, "r") - srcString = file_.read() + with open(path, "r") as file_: + srcString = file_.read() if len(srcString) == 0: # ignore blank files pyfalog.debug("File is blank.") @@ -442,7 +478,7 @@ class Port(object): firstLine = firstLine.strip() # If XML-style start of tag encountered, detect as XML - if re.match("<", firstLine): + if re.match(RE_XML_START, firstLine): if encoding: return "XML", cls.importXml(string, iportuser, encoding) else: @@ -975,12 +1011,16 @@ class Port(object): sMkt = Market.getInstance() doc = xml.dom.minidom.parseString(text.encode(encoding)) + # NOTE: + # When L_MARK is included at this point, + # Decided to be localized data + b_localized = L_MARK in text fittings = doc.getElementsByTagName("fittings").item(0) fittings = fittings.getElementsByTagName("fitting") fit_list = [] for fitting in (fittings): - b_localized, fitobj = _resolveShip(fitting, sMkt) + fitobj = _resolve_ship(fitting, sMkt, b_localized) # -- 170327 Ignored description -- # read description from exported xml. (EVE client, EFT) description = fitting.getElementsByTagName("description").item(0).getAttribute("value") @@ -988,16 +1028,17 @@ class Port(object): description = "" elif len(description): # convert
to "\n" and remove html tags. - description = sequential_rep(description, r"<(br|BR)>", "\n", r"<[^<>]+>", "") -# description = re.sub(r"<(br|BR)>", "\n", description) -# description = re.sub(r"<[^<>]+>", "", description) + if Port.is_tag_replace(): + description = replace_ltgt( + sequential_rep(description, r"<(br|BR)>", "\n", r"<[^<>]+>", "") + ) fitobj.notes = description hardwares = fitting.getElementsByTagName("hardware") moduleList = [] for hardware in hardwares: try: - item = _resolveModule(hardware, sMkt, b_localized) + item = _resolve_module(hardware, sMkt, b_localized) if not item: continue @@ -1340,7 +1381,7 @@ class PortProcessing(object): def backupFits(path, iportuser): success = True try: - iportuser.onPortProcessStart() + iportuser.on_port_process_start() backedUpFits = Port.exportXml(iportuser, *svcFit.getInstance().getAllFits()) backupFile = open(path, "w", encoding="utf-8") backupFile.write(backedUpFits) @@ -1350,17 +1391,17 @@ class PortProcessing(object): # Send done signal to GUI # wx.CallAfter(callback, -1, "Done.") flag = IPortUser.ID_ERROR if not success else IPortUser.ID_DONE - iportuser.onPortProcessing(IPortUser.PROCESS_EXPORT | flag, + iportuser.on_port_processing(IPortUser.PROCESS_EXPORT | flag, "User canceled or some error occurrence." if not success else "Done.") @staticmethod def importFitsFromFile(paths, iportuser): - iportuser.onPortProcessStart() + iportuser.on_port_process_start() success, result = Port.importFitFromFiles(paths, iportuser) flag = IPortUser.ID_ERROR if not success else IPortUser.ID_DONE - iportuser.onPortProcessing(IPortUser.PROCESS_IMPORT | flag, result) + iportuser.on_port_processing(IPortUser.PROCESS_IMPORT | flag, result) @staticmethod def notify(iportuser, flag, data): - if not iportuser.onPortProcessing(flag, data): + if not iportuser.on_port_processing(flag, data): raise UserCancelException diff --git a/utils/stopwatch.py b/utils/stopwatch.py new file mode 100644 index 000000000..c14339e90 --- /dev/null +++ b/utils/stopwatch.py @@ -0,0 +1,107 @@ +# coding: utf-8 + +import time +import os + +# 2017/04/05: +class Stopwatch(object): + """ + --- on python console --- +import re +from utils.stopwatch import Stopwatch +# measurement re.sub +def m_re_sub(t, set_count, executes, texts): + t.reset() + while set_count: + set_count -= 1 + with t: + while executes: + executes -= 1 + ret = re.sub("[a|s]+", "-", texts) + # stat string + return str(t) + +# measurementor +stpwth = Stopwatch("test") +# statistics loop: 1000(exec re.sub: 100000) +m_re_sub(stpwth, 1000, 100000, "asdfadsasdaasdfadsasda") + +----------- records ----------- + text: "asdfadsasda" + 'elapsed record(ms): min=0.000602411446948, max=220.85578571' + 'elapsed record(ms): min=0.000602411446948, max=217.331377504' + + text: "asdfadsasdaasdfadsasda" + 'elapsed record(ms): min=0.000602411446948, max=287.784902967' + 'elapsed record(ms): min=0.000602411432737, max=283.653264016' + + NOTE: about max + The value is large only at the first execution, + Will it be optimized, after that it will be significantly smaller + """ + + # time.clock() is μs? 1/1000ms + # https://docs.python.jp/2.7/library/time.html#time.clock + _tfunc = time.clock if os.name == "nt" else time.time + + def __init__(self, name='', logger=None): + self.name = name + self.start = Stopwatch._tfunc() + self.__last = self.start + # __last field is means last checkpoint system clock value? + self.logger = logger + self.reset() + + @property + def stat(self): + # :return: (float, float) + return (self.min, self.max) + + @property + def elapsed(self): + # :return: time as ms + return (Stopwatch._tfunc() - self.start) * 1000 + + @property + def last(self): + return self.__last * 1000 + + def __update_stat(self, v): + # :param v: unit of ms + if self.min == 0.0: self.min = v + if self.max < v: self.max = v + if self.min > v: self.min = v + + def checkpoint(self, name=''): + span = self.elapsed + self.__update_stat(span) + text = u'Stopwatch("{tname}") - {checkpoint} - {last:.6f}ms ({elapsed:.12f}ms elapsed)'.format( + tname=self.name, + checkpoint=unicode(name, "utf-8"), + last=self.last, + elapsed=span + ).strip() + self.__last = Stopwatch._tfunc() + if self.logger: + self.logger.debug(text) + else: + print(text) + + def reset(self): + self.min = 0.0 + self.max = 0.0 + + def __enter__(self): + self.start = Stopwatch._tfunc() + return self + + def __exit__(self, type, value, traceback): + # https://docs.python.org/2.7/reference/datamodel.html?highlight=__enter__#object.__exit__ + # If the context was exited without an exception, all three arguments will be None + self.checkpoint('finished') + # ex: "type=None, value=None, traceback=None" + # print "type=%s, value=%s, traceback=%s" % (type, value, traceback) + return True + + def __repr__(self): + return "elapsed record(ms): min=%s, max=%s" % self.stat diff --git a/utils/strfunctions.py b/utils/strfunctions.py index cd769d20b..ec6013284 100644 --- a/utils/strfunctions.py +++ b/utils/strfunctions.py @@ -5,30 +5,26 @@ import re def sequential_rep(text_, *args): + # type: (basestring, *list) -> basestring """ - params - text_: string content - args : , , , , ... - - return - empty string when text_ length was zero or invalid. + :param text_: string content + :param args: like , , , , ... + :return: if text_ length was zero or invalid parameters then no manipulation to text_ """ - - if not text_ or not len(text_): - return "" - arg_len = len(args) - i = 0 - while i < arg_len: - text_ = re.sub(args[i], args[i + 1], text_) - i += 2 + if arg_len % 2 == 0 and isinstance(text_, basestring) and len(text_) > 0: + i = 0 + while i < arg_len: + text_ = re.sub(args[i], args[i + 1], text_) + i += 2 return text_ -def replaceLTGT(text_): +def replace_ltgt(text_): + # type: (basestring) -> basestring """if fit name contained "<" or ">" then reprace to named html entity by EVE client. - - for fit name. + :param text_: string content of fit name from exported by EVE client. + :return: if text_ is not instance of basestring then no manipulation to text_. """ - return text_.replace("<", "<").replace(">", ">") if isinstance(text_, unicode) else text_ + return text_.replace("<", "<").replace(">", ">") if isinstance(text_, basestring) else text_ From 0ecaa4d36c31b50582a305ceb6535997c49167c4 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Wed, 5 Apr 2017 21:26:45 +0900 Subject: [PATCH 05/11] edit missied format... --- utils/stopwatch.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/utils/stopwatch.py b/utils/stopwatch.py index c14339e90..3885d97b1 100644 --- a/utils/stopwatch.py +++ b/utils/stopwatch.py @@ -68,9 +68,12 @@ m_re_sub(stpwth, 1000, 100000, "asdfadsasdaasdfadsasda") def __update_stat(self, v): # :param v: unit of ms - if self.min == 0.0: self.min = v - if self.max < v: self.max = v - if self.min > v: self.min = v + if self.min == 0.0: + self.min = v + if self.max < v: + self.max = v + if self.min > v: + self.min = v def checkpoint(self, name=''): span = self.elapsed From dd88d64d67fc35140c13807b42e4721c9a9e4928 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Wed, 5 Apr 2017 21:35:31 +0900 Subject: [PATCH 06/11] im trying PyCharm now! --- utils/stopwatch.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/utils/stopwatch.py b/utils/stopwatch.py index 3885d97b1..12e1de11b 100644 --- a/utils/stopwatch.py +++ b/utils/stopwatch.py @@ -3,7 +3,7 @@ import time import os -# 2017/04/05: + class Stopwatch(object): """ --- on python console --- @@ -50,12 +50,13 @@ m_re_sub(stpwth, 1000, 100000, "asdfadsasdaasdfadsasda") self.__last = self.start # __last field is means last checkpoint system clock value? self.logger = logger - self.reset() + self.min = 0.0 + self.max = 0.0 @property def stat(self): # :return: (float, float) - return (self.min, self.max) + return self.min, self.max @property def elapsed(self): @@ -98,7 +99,7 @@ m_re_sub(stpwth, 1000, 100000, "asdfadsasdaasdfadsasda") self.start = Stopwatch._tfunc() return self - def __exit__(self, type, value, traceback): + def __exit__(self, type_, value, traceback): # https://docs.python.org/2.7/reference/datamodel.html?highlight=__enter__#object.__exit__ # If the context was exited without an exception, all three arguments will be None self.checkpoint('finished') From d4d69522f52325d6b6900007f869a7975d4fe828 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Thu, 6 Apr 2017 03:37:40 +0900 Subject: [PATCH 07/11] edit source --- service/port.py | 2 +- utils/stopwatch.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/service/port.py b/service/port.py index b7bdd95ac..8fc7a70ab 100644 --- a/service/port.py +++ b/service/port.py @@ -478,7 +478,7 @@ class Port(object): firstLine = firstLine.strip() # If XML-style start of tag encountered, detect as XML - if re.match(RE_XML_START, firstLine): + if re.search(RE_XML_START, firstLine): if encoding: return "XML", cls.importXml(string, iportuser, encoding) else: diff --git a/utils/stopwatch.py b/utils/stopwatch.py index 12e1de11b..8b78f21af 100644 --- a/utils/stopwatch.py +++ b/utils/stopwatch.py @@ -68,7 +68,7 @@ m_re_sub(stpwth, 1000, 100000, "asdfadsasdaasdfadsasda") return self.__last * 1000 def __update_stat(self, v): - # :param v: unit of ms + # :param v: float unit of ms if self.min == 0.0: self.min = v if self.max < v: From 53957c24dfeb366106ac65e63f5e4685fc3e0e42 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Sat, 8 Apr 2017 22:31:25 +0900 Subject: [PATCH 08/11] Reflect changes when writing test code --- utils/stopwatch.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/utils/stopwatch.py b/utils/stopwatch.py index 8b78f21af..adf7e5832 100644 --- a/utils/stopwatch.py +++ b/utils/stopwatch.py @@ -9,6 +9,9 @@ class Stopwatch(object): --- on python console --- import re from utils.stopwatch import Stopwatch + +# measurementor +stpw = Stopwatch("test") # measurement re.sub def m_re_sub(t, set_count, executes, texts): t.reset() @@ -21,10 +24,8 @@ def m_re_sub(t, set_count, executes, texts): # stat string return str(t) -# measurementor -stpwth = Stopwatch("test") # statistics loop: 1000(exec re.sub: 100000) -m_re_sub(stpwth, 1000, 100000, "asdfadsasdaasdfadsasda") +m_re_sub(stpw, 1000, 100000, "asdfadsasdaasdfadsasda") ----------- records ----------- text: "asdfadsasda" @@ -52,6 +53,7 @@ m_re_sub(stpwth, 1000, 100000, "asdfadsasdaasdfadsasda") self.logger = logger self.min = 0.0 self.max = 0.0 + self.__first = True @property def stat(self): @@ -69,12 +71,13 @@ m_re_sub(stpwth, 1000, 100000, "asdfadsasdaasdfadsasda") def __update_stat(self, v): # :param v: float unit of ms - if self.min == 0.0: + if self.__first: + self.__first = False + return + if self.min == 0.0 or self.min > v: self.min = v if self.max < v: self.max = v - if self.min > v: - self.min = v def checkpoint(self, name=''): span = self.elapsed @@ -91,9 +94,17 @@ m_re_sub(stpwth, 1000, 100000, "asdfadsasdaasdfadsasda") else: print(text) + @staticmethod + def CpuClock(): + start = Stopwatch._tfunc() + time.sleep(1) + return Stopwatch._tfunc() - start + def reset(self): + # clear stat self.min = 0.0 self.max = 0.0 + self.__first = True def __enter__(self): self.start = Stopwatch._tfunc() From 4a964ab6bea13c32e018aa973c791c1802aa6ce4 Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Mon, 10 Apr 2017 03:05:44 +0900 Subject: [PATCH 09/11] apply recent improvements --- service/port.py | 64 +++++++++++++++++++++++++------------------ utils/strfunctions.py | 2 +- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/service/port.py b/service/port.py index 8fc7a70ab..3e8a98330 100644 --- a/service/port.py +++ b/service/port.py @@ -1,4 +1,3 @@ -# coding: utf-8 # ============================================================================= # Copyright (C) 2014 Ryan Holmes # @@ -74,68 +73,77 @@ INV_FLAG_DRONEBAY = 87 INV_FLAG_FIGHTER = 158 # 2017/04/05 NOTE: simple validation, for xml file -RE_XML_START = r'<\?xml\s+version="1.0"\s+\?>' +RE_XML_START = r'<\?xml\s+version="1.0"\s*\?>' # -- 170327 Ignored description -- RE_LTGT = "&(lt|gt);" L_MARK = "<localized hint="" +# <localized hint="([^"]+)">([^\*]+)\*<\/localized> LOCALIZED_PATTERN = re.compile(r'([^\*]+)\*') -def _extract_matche(t): +def _extract_match(t): m = LOCALIZED_PATTERN.match(t) # hint attribute, text content return m.group(1), m.group(2) def _resolve_ship(fitting, sMkt, b_localized): - # type: (xml.dom.minidom.Element, Market, bool) -> eos.saveddata.fit.Fit - - fitobj = Fit() - fitobj.name = fitting.getAttribute("name") - # 2017/03/29 NOTE: - # if fit name contained "<" or ">" then reprace to named html entity by EVE client - # if re.search(RE_LTGT, fitobj.name): - if "<" in fitobj.name or ">" in fitobj.name: - fitobj.name = replace_ltgt(fitobj.name) - + # type: (xml.dom.minidom.Element, service.market.Market, bool) -> eos.saveddata.fit.Fit + """ NOTE: Since it is meaningless unless a correct ship object can be constructed, + process flow changed + """ + # ------ Confirm ship # Maelstrom shipType = fitting.getElementsByTagName("shipType").item(0).getAttribute("value") - # emergency = None + anything = None if b_localized: # expect an official name, emergency cache - shipType, emergency = _extract_matche(shipType) + shipType, anything = _extract_match(shipType) limit = 2 + ship = None while True: must_retry = False try: try: - fitobj.ship = Ship(sMkt.getItem(shipType)) + ship = Ship(sMkt.getItem(shipType)) except ValueError: - fitobj.ship = Citadel(sMkt.getItem(shipType)) + ship = Citadel(sMkt.getItem(shipType)) except Exception as e: pyfalog.warning("Caught exception on _resolve_ship") pyfalog.error(e) limit -= 1 if limit is 0: break - shipType = emergency + shipType = anything must_retry = True if not must_retry: break + if ship is None: + raise Exception("cannot resolve ship type.") + + fitobj = Fit(ship=ship) + # ------ Confirm fit name + anything = fitting.getAttribute("name") + # 2017/03/29 NOTE: + # if fit name contained "<" or ">" then reprace to named html entity by EVE client + # if re.search(RE_LTGT, anything): + if "<" in anything or ">" in anything: + anything = replace_ltgt(anything) + fitobj.name = anything + return fitobj def _resolve_module(hardware, sMkt, b_localized): - # type: (xml.dom.minidom.Element, Market, bool) -> eos.saveddata.module.Module - + # type: (xml.dom.minidom.Element, service.market.Market, bool) -> eos.saveddata.module.Module moduleName = hardware.getAttribute("type") - # emergency = None + emergency = None if b_localized: # expect an official name, emergency cache - moduleName, emergency = _extract_matche(moduleName) + moduleName, emergency = _extract_match(moduleName) item = None limit = 2 @@ -365,6 +373,9 @@ class Port(object): @staticmethod def importFitFromBuffer(bufferStr, activeFit=None): + # type: (basestring, object) -> object + # TODO: catch the exception? + # activeFit is reserved?, bufferStr is unicode? (assume only clipboard string? sFit = svcFit.getInstance() _, fits = Port.importAuto(bufferStr, activeFit=activeFit) for fit in fits: @@ -473,6 +484,7 @@ class Port(object): @classmethod def importAuto(cls, string, path=None, activeFit=None, iportuser=None, encoding=None): + # type: (basestring, basestring, object, IPortUser, basestring) -> object # Get first line and strip space symbols of it to avoid possible detection errors firstLine = re.split("[\n\r]+", string.strip(), maxsplit=1)[0] firstLine = firstLine.strip() @@ -510,9 +522,9 @@ class Port(object): fitobj = Fit() refobj = json.loads(str_) items = refobj['items'] + # "<" and ">" is replace to "<", ">" by EVE client fitobj.name = refobj['name'] # 2017/03/29: read description - # "<" and ">" is replace to "<", ">" by EVE client fitobj.notes = refobj['description'] try: @@ -1008,7 +1020,7 @@ class Port(object): @staticmethod def importXml(text, iportuser=None, encoding="utf-8"): - + # type: (basestring, IPortUser, basestring) -> list[eos.saveddata.fit.Fit] sMkt = Market.getInstance() doc = xml.dom.minidom.parseString(text.encode(encoding)) # NOTE: @@ -1019,7 +1031,7 @@ class Port(object): fittings = fittings.getElementsByTagName("fitting") fit_list = [] - for fitting in (fittings): + for fitting in fittings: fitobj = _resolve_ship(fitting, sMkt, b_localized) # -- 170327 Ignored description -- # read description from exported xml. (EVE client, EFT) @@ -1248,7 +1260,7 @@ class Port(object): try: notes = fit.notes # unicode description.setAttribute( - "value", re.sub("[\r\n]+", "
", notes) if notes is not None else "" + "value", re.sub("(\r|\n|\r\n)+", "
", notes) if notes is not None else "" ) except Exception as e: pyfalog.warning("read description is failed, msg=%s\n" % e.args) diff --git a/utils/strfunctions.py b/utils/strfunctions.py index ec6013284..26bf5f59a 100644 --- a/utils/strfunctions.py +++ b/utils/strfunctions.py @@ -5,7 +5,7 @@ import re def sequential_rep(text_, *args): - # type: (basestring, *list) -> basestring + # type: (basestring, tuple) -> basestring """ :param text_: string content :param args: like , , , , ... From fc7ca56f8bd19cbc16b1d3ece92ed9df59d0ddbf Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Mon, 10 Apr 2017 09:33:54 +0900 Subject: [PATCH 10/11] searching for a way to open db with minimal changes required --- _development/__init__.py | 0 _development/helpers.py | 143 +++ _development/helpers_fits.py | 66 ++ service/settings.py | 11 +- tests/jeffy_ja-en[99].xml | 2116 ++++++++++++++++++++++++++++++++++ tests/test_fitcache.py | 232 ++++ tests/test_unread_desc.py | 69 ++ 7 files changed, 2633 insertions(+), 4 deletions(-) create mode 100644 _development/__init__.py create mode 100644 _development/helpers.py create mode 100644 _development/helpers_fits.py create mode 100644 tests/jeffy_ja-en[99].xml create mode 100644 tests/test_fitcache.py create mode 100644 tests/test_unread_desc.py diff --git a/_development/__init__.py b/_development/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/_development/helpers.py b/_development/helpers.py new file mode 100644 index 000000000..b35ac835f --- /dev/null +++ b/_development/helpers.py @@ -0,0 +1,143 @@ +# noinspection PyPackageRequirements +import pytest + +import os +import sys +import threading + +from sqlalchemy import MetaData, create_engine +from sqlalchemy.orm import sessionmaker + +script_dir = os.path.dirname(os.path.abspath(__file__)) +# Add root folder to python paths +sys.path.append(os.path.realpath(os.path.join(script_dir, '..', '..'))) +sys._called_from_test = True + +# noinspection PyUnresolvedReferences,PyUnusedLocal +@pytest.fixture +def DBInMemory_test(): + def rollback(): + with sd_lock: + saveddata_session.rollback() + + + print("Creating database in memory") + from os.path import realpath, join, dirname, abspath + + debug = False + gamedataCache = True + saveddataCache = True + gamedata_version = "" + gamedata_connectionstring = 'sqlite:///' + realpath(join(dirname(abspath(unicode(__file__))), "..", "eve.db")) + saveddata_connectionstring = 'sqlite:///:memory:' + + class ReadOnlyException(Exception): + pass + + if callable(gamedata_connectionstring): + gamedata_engine = create_engine("sqlite://", creator=gamedata_connectionstring, echo=debug) + else: + gamedata_engine = create_engine(gamedata_connectionstring, echo=debug) + + gamedata_meta = MetaData() + gamedata_meta.bind = gamedata_engine + gamedata_session = sessionmaker(bind=gamedata_engine, autoflush=False, expire_on_commit=False)() + + # This should be moved elsewhere, maybe as an actual query. Current, without try-except, it breaks when making a new + # game db because we haven't reached gamedata_meta.create_all() + try: + gamedata_version = gamedata_session.execute( + "SELECT `field_value` FROM `metadata` WHERE `field_name` LIKE 'client_build'" + ).fetchone()[0] + except Exception as e: + print("Missing gamedata version.") + gamedata_version = None + + if saveddata_connectionstring is not None: + if callable(saveddata_connectionstring): + saveddata_engine = create_engine(creator=saveddata_connectionstring, echo=debug) + else: + saveddata_engine = create_engine(saveddata_connectionstring, echo=debug) + + saveddata_meta = MetaData() + saveddata_meta.bind = saveddata_engine + saveddata_session = sessionmaker(bind=saveddata_engine, autoflush=False, expire_on_commit=False)() + else: + saveddata_meta = None + + # Lock controlling any changes introduced to session + sd_lock = threading.Lock() + + # Import all the definitions for all our database stuff + # noinspection PyPep8 + #from eos.db.gamedata import alphaClones, attribute, category, effect, group, icon, item, marketGroup, metaData, metaGroup, queries, traits, unit + # noinspection PyPep8 + #from eos.db.saveddata import booster, cargo, character, crest, damagePattern, databaseRepair, drone, fighter, fit, implant, implantSet, loadDefaultDatabaseValues, miscData, module, override, price, queries, skill, targetResists, user + + # If using in memory saveddata, you'll want to reflect it so the data structure is good. + if saveddata_connectionstring == "sqlite:///:memory:": + saveddata_meta.create_all() + + # Output debug info to help us troubleshoot Travis + print(saveddata_engine) + print(gamedata_engine) + + helper = { + #'config': eos.config, + 'gamedata_session' : gamedata_session, + 'saveddata_session' : saveddata_session, + } + return helper + +# noinspection PyUnresolvedReferences,PyUnusedLocal +@pytest.fixture +def DBInMemory(): + print("Creating database in memory") + + import eos.config + + import eos + import eos.db + + # Output debug info to help us troubleshoot Travis + print(eos.db.saveddata_engine) + print(eos.db.gamedata_engine) + + helper = { + 'config': eos.config, + 'db' : eos.db, + 'gamedata_session' : eos.db.gamedata_session, + 'saveddata_session' : eos.db.saveddata_session, + } + return helper + + +@pytest.fixture +def Gamedata(): + print("Building Gamedata") + from eos.gamedata import Item + + helper = { + 'Item': Item, + } + return helper + + +@pytest.fixture +def Saveddata(): + print("Building Saveddata") + from eos.saveddata.ship import Ship + from eos.saveddata.fit import Fit + from eos.saveddata.character import Character + from eos.saveddata.module import Module, State + from eos.saveddata.citadel import Citadel + + helper = { + 'Structure': Citadel, + 'Ship' : Ship, + 'Fit' : Fit, + 'Character': Character, + 'Module' : Module, + 'State' : State, + } + return helper diff --git a/_development/helpers_fits.py b/_development/helpers_fits.py new file mode 100644 index 000000000..f164a6bb3 --- /dev/null +++ b/_development/helpers_fits.py @@ -0,0 +1,66 @@ +import pytest + +# noinspection PyPackageRequirements +from _development.helpers import DBInMemory as DB, Gamedata, Saveddata + + +# noinspection PyShadowingNames +@pytest.fixture +def RifterFit(DB, Gamedata, Saveddata): + print("Creating Rifter") + item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Rifter").first() + ship = Saveddata['Ship'](item) + # setup fit + fit = Saveddata['Fit'](ship, "My Rifter Fit") + + return fit + + +# noinspection PyShadowingNames +@pytest.fixture +def KeepstarFit(DB, Gamedata, Saveddata): + print("Creating Keepstar") + item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Keepstar").first() + ship = Saveddata['Structure'](item) + # setup fit + fit = Saveddata['Fit'](ship, "Keepstar Fit") + + return fit + + +# noinspection PyShadowingNames +@pytest.fixture +def CurseFit(DB, Gamedata, Saveddata): + print("Creating Curse - With Neuts") + item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Curse").first() + ship = Saveddata['Ship'](item) + # setup fit + fit = Saveddata['Fit'](ship, "Curse - With Neuts") + + mod = Saveddata['Module'](DB['db'].getItem("Medium Energy Neutralizer II")) + mod.state = Saveddata['State'].ONLINE + + # Add 5 neuts + for _ in xrange(5): + fit.modules.append(mod) + + return fit + + +# noinspection PyShadowingNames +@pytest.fixture +def HeronFit(DB, Gamedata, Saveddata): + print("Creating Heron - RemoteSebo") + item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Heron").first() + ship = Saveddata['Ship'](item) + # setup fit + fit = Saveddata['Fit'](ship, "Heron - RemoteSebo") + + mod = Saveddata['Module'](DB['db'].getItem("Remote Sensor Booster II")) + mod.state = Saveddata['State'].ONLINE + + # Add 5 neuts + for _ in xrange(4): + fit.modules.append(mod) + + return fit diff --git a/service/settings.py b/service/settings.py index cfc48fc4a..3da94b084 100644 --- a/service/settings.py +++ b/service/settings.py @@ -29,7 +29,8 @@ pyfalog = Logger(__name__) class SettingsProvider(object): - BASE_PATH = os.path.join(config.savePath, 'settings') + if config.savePath: + BASE_PATH = os.path.join(config.savePath, 'settings') settings = {} _instance = None @@ -41,13 +42,15 @@ class SettingsProvider(object): return cls._instance def __init__(self): - if not os.path.exists(self.BASE_PATH): - os.mkdir(self.BASE_PATH) + if hasattr(self, 'BASE_PATH'): + if not os.path.exists(self.BASE_PATH): + os.mkdir(self.BASE_PATH) def getSettings(self, area, defaults=None): s = self.settings.get(area) - if s is None: + + if s is None and hasattr(self, 'BASE_PATH'): p = os.path.join(self.BASE_PATH, area) if not os.path.exists(p): diff --git a/tests/jeffy_ja-en[99].xml b/tests/jeffy_ja-en[99].xml new file mode 100644 index 000000000..8ec178b16 --- /dev/null +++ b/tests/jeffy_ja-en[99].xml @@ -0,0 +1,2116 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test_fitcache.py b/tests/test_fitcache.py new file mode 100644 index 000000000..9de98d097 --- /dev/null +++ b/tests/test_fitcache.py @@ -0,0 +1,232 @@ +# Add root folder to python paths +# This must be done on every test in order to pass in Travis +import os +import sys +from time import time +script_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.realpath(os.path.join(script_dir, '..'))) + +# +# noinspection PyPackageRequirements +from _development.helpers import DBInMemory as DB, Gamedata, Saveddata +# noinspection PyPackageRequirements +from _development.helpers_fits import RifterFit, KeepstarFit, HeronFit, CurseFit +from service.fit import Fit +# +# # Fake import wx +# # todo: fix this +# # from types import ModuleType +# # wx = ModuleType("fake_module") +# # sys.modules[wx.__name__] = wx +# +# def test_getAllFits(DB, RifterFit, KeepstarFit): +# assert len(Fit.getAllFits()) == 0 +# DB['db'].save(RifterFit) +# assert len(Fit.getAllFits()) == 1 +# DB['db'].save(KeepstarFit) +# assert len(Fit.getAllFits()) == 2 +# +# # Cleanup after ourselves +# DB['db'].remove(RifterFit) +# DB['db'].remove(KeepstarFit) +# +# +# def test_getFitsWithShip_RifterFit(DB, RifterFit): +# DB['db'].save(RifterFit) +# +# assert Fit.getFitsWithShip(587)[0][1] == 'My Rifter Fit' +# +# DB['db'].remove(RifterFit) + + +def test_RifterSingleNew(DB, RifterFit, KeepstarFit, HeronFit, CurseFit): + DB['db'].save(RifterFit) + DB['db'].save(KeepstarFit) + DB['db'].save(HeronFit) + DB['db'].save(CurseFit) + sFit = Fit.getInstance() + sFit.serviceFittingOptions = { + "useGlobalCharacter" : False, + "useGlobalDamagePattern": False, + "useGlobalForceReload" : False, + "colorFitBySlot" : False, + "rackSlots" : True, + "rackLabels" : True, + "compactSkills" : True, + "showTooltip" : True, + "showMarketShortcuts" : False, + "enableGaugeAnimation" : True, + "exportCharges" : True, + "openFitInNew" : False, + "priceSystem" : "Jita", + "showShipBrowserTooltip": True, + } + + cached_fits = [] + fit = DB["db"].getFit(1) + cached_fits.append(fit) + fit = None + + time_start = time() + + for _ in xrange(1000000): + + fit = next((x for x in cached_fits if x.ID == 1), None) + + fit = None + + print("1000000 of the Rifter fit (new): " + str(time()-time_start)) + + # fit = DB["db"].getFit(1) + + # Cleanup after ourselves + DB['db'].remove(RifterFit) + DB['db'].remove(KeepstarFit) + DB['db'].remove(HeronFit) + DB['db'].remove(CurseFit) + +def test_RifterSingleOld(DB, RifterFit, KeepstarFit, HeronFit, CurseFit): + DB['db'].save(RifterFit) + DB['db'].save(KeepstarFit) + DB['db'].save(HeronFit) + DB['db'].save(CurseFit) + sFit = Fit.getInstance() + sFit.serviceFittingOptions = { + "useGlobalCharacter" : False, + "useGlobalDamagePattern": False, + "useGlobalForceReload" : False, + "colorFitBySlot" : False, + "rackSlots" : True, + "rackLabels" : True, + "compactSkills" : True, + "showTooltip" : True, + "showMarketShortcuts" : False, + "enableGaugeAnimation" : True, + "exportCharges" : True, + "openFitInNew" : False, + "priceSystem" : "Jita", + "showShipBrowserTooltip": True, + } + + cached_fits = [] + fit = DB["db"].getFit(1) + cached_fits.append(fit) + fit = None + + time_start = time() + + for _ in xrange(1000000): + + fit = DB["db"].getFit(1) + + fit = None + + print("1000000 of the Rifter fit (old): " + str(time()-time_start)) + + # Cleanup after ourselves + DB['db'].remove(RifterFit) + DB['db'].remove(KeepstarFit) + DB['db'].remove(HeronFit) + DB['db'].remove(CurseFit) + +def test_FourNew(DB, RifterFit, KeepstarFit, HeronFit, CurseFit): + DB['db'].save(RifterFit) + DB['db'].save(KeepstarFit) + DB['db'].save(HeronFit) + DB['db'].save(CurseFit) + sFit = Fit.getInstance() + sFit.serviceFittingOptions = { + "useGlobalCharacter" : False, + "useGlobalDamagePattern": False, + "useGlobalForceReload" : False, + "colorFitBySlot" : False, + "rackSlots" : True, + "rackLabels" : True, + "compactSkills" : True, + "showTooltip" : True, + "showMarketShortcuts" : False, + "enableGaugeAnimation" : True, + "exportCharges" : True, + "openFitInNew" : False, + "priceSystem" : "Jita", + "showShipBrowserTooltip": True, + } + + cached_fits = [] + fit = DB["db"].getFit(1) + cached_fits.append(fit) + fit = None + + time_start = time() + + for _ in xrange(250000): + + fit = next((x for x in cached_fits if x.ID == 1), None) + fit = None + fit = next((x for x in cached_fits if x.ID == 2), None) + fit = None + fit = next((x for x in cached_fits if x.ID == 3), None) + fit = None + fit = next((x for x in cached_fits if x.ID == 4), None) + fit = None + + + print("1000000 of the four fits (new): " + str(time()-time_start)) + + # fit = DB["db"].getFit(1) + + # Cleanup after ourselves + DB['db'].remove(RifterFit) + DB['db'].remove(KeepstarFit) + DB['db'].remove(HeronFit) + DB['db'].remove(CurseFit) + +def test_FourOld(DB, RifterFit, KeepstarFit, HeronFit, CurseFit): + DB['db'].save(RifterFit) + DB['db'].save(KeepstarFit) + DB['db'].save(HeronFit) + DB['db'].save(CurseFit) + sFit = Fit.getInstance() + sFit.serviceFittingOptions = { + "useGlobalCharacter" : False, + "useGlobalDamagePattern": False, + "useGlobalForceReload" : False, + "colorFitBySlot" : False, + "rackSlots" : True, + "rackLabels" : True, + "compactSkills" : True, + "showTooltip" : True, + "showMarketShortcuts" : False, + "enableGaugeAnimation" : True, + "exportCharges" : True, + "openFitInNew" : False, + "priceSystem" : "Jita", + "showShipBrowserTooltip": True, + } + + cached_fits = [] + fit = DB["db"].getFit(1) + cached_fits.append(fit) + fit = None + + time_start = time() + + for _ in xrange(250000): + + fit = DB["db"].getFit(1) + fit = None + fit = DB["db"].getFit(2) + fit = None + fit = DB["db"].getFit(3) + fit = None + fit = DB["db"].getFit(4) + fit = None + + print("1000000 of the four fits (old): " + str(time()-time_start)) + + # Cleanup after ourselves + DB['db'].remove(RifterFit) + DB['db'].remove(KeepstarFit) + DB['db'].remove(HeronFit) + DB['db'].remove(CurseFit) + diff --git a/tests/test_unread_desc.py b/tests/test_unread_desc.py new file mode 100644 index 000000000..8cd2d40a6 --- /dev/null +++ b/tests/test_unread_desc.py @@ -0,0 +1,69 @@ +""" + 2017/04/05: unread description tests module. +""" +# Add root folder to python paths +# This must be done on every test in order to pass in Travis +import os +import sys +script_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.realpath(os.path.join(script_dir, '..'))) + +import re +# +# noinspection PyPackageRequirements +from _development.helpers import DBInMemory as DB, Gamedata, Saveddata +# noinspection PyPep8 +from service.port import Port, IPortUser + +# from utils.strfunctions import sequential_rep, replace_ltgt +from utils.stopwatch import Stopwatch + +""" +NOTE: + description character length is restricted 4hundred by EVE client. + these things apply to multi byte environment too. + + + o read xml fit data (and encode to utf-8 if need. + + o construct xml dom object, and extract "fitting" elements. + + o apply _resolve_ship method to each "fitting" elements. (time measurement + + o extract "hardware" elements from "fitting" element. + + o apply _resolve_module method to each "hardware" elements. (time measurement + +xml files: + "jeffy_ja-en[99].xml" + +NOTE of @decorator: + o Function to receive arguments of function to be decorated + o A function that accepts the decorate target function itself as an argument + o A function that accepts arguments of the decorator itself +""" + +class PortUser(IPortUser): + + def on_port_processing(self, action, data=None): + print(data) + return True + + +stpw = Stopwatch('test measurementer') + +def test_import_xml(): + usr = PortUser() +# for path in XML_FILES: + xml_file = "jeffy_ja-en[99].xml" + fit_count = int(re.search(r"\[(\d+)\]", xml_file).group(1)) + with open(os.path.join(script_dir, xml_file), "r") as file_: + srcString = file_.read() + srcString = unicode(srcString, "utf-8") + # (basestring, IPortUser, basestring) -> list[eos.saveddata.fit.Fit] + usr.on_port_process_start() + stpw.reset() + with stpw: + fits = Port.importXml(srcString, usr) + + assert fits is not None and len(fits) is fit_count From c07bcf6a2985006b79aee9d239dfe1653f909cef Mon Sep 17 00:00:00 2001 From: jeffy-g Date: Mon, 10 Apr 2017 14:15:24 +0900 Subject: [PATCH 11/11] Determine the necessary changes to use db with test code, I made necessary edits .travis.yml necessary to depend on wx mod indirectly when running test code eos\config.py copy from development branch, This change was necessary when using data base in test code. service\settings.py copy from development branch. and modified SettingsProvider.getSettings and Settings.save. After that, we made the same as master branch except for necessary code. This change was necessary when using data base in test code. and other improvement. --- .travis.yml | 32 +++-- _development/__init__.py | 0 _development/helpers.py | 143 --------------------- _development/helpers_fits.py | 66 ---------- eos/config.py | 22 +++- eos/saveddata/module.py | 9 ++ service/settings.py | 77 ++++++++---- tests/test_fitcache.py | 232 ----------------------------------- tests/test_unread_desc.py | 39 ++++-- 9 files changed, 134 insertions(+), 486 deletions(-) delete mode 100644 _development/__init__.py delete mode 100644 _development/helpers.py delete mode 100644 _development/helpers_fits.py delete mode 100644 tests/test_fitcache.py diff --git a/.travis.yml b/.travis.yml index 497b67b08..2b4dbf181 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: python +cache: pip python: - '2.7' env: @@ -6,20 +7,33 @@ env: addons: apt: packages: - # for wxPython: - - python-wxgtk2.8 - - python-wxtools - - wx2.8-doc - - wx2.8-examples - - wx2.8-headers - - wx2.8-i18n before_install: - - pip install -U tox + - sudo apt-get update && sudo apt-get --reinstall install -qq language-pack-en language-pack-ru language-pack-he language-pack-zh-hans + - pip install tox + # We're not actually installing Tox, but have to run it before we install wxPython via Conda. This is fugly but vOv + - tox + # get Conda + - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then + wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; + else + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + fi + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda update -q conda + # Useful for debugging any issues with conda + - conda info -a install: + # install wxPython 3.0.0.0 + - conda install -c https://conda.anaconda.org/travis wxpython +before_script: - pip install -r requirements.txt - pip install -r requirements_test.txt script: - - tox - py.test --cov=./ after_success: - bash <(curl -s https://codecov.io/bash) +before_deploy: + - pip install -r requirements_build_linux.txt diff --git a/_development/__init__.py b/_development/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/_development/helpers.py b/_development/helpers.py deleted file mode 100644 index b35ac835f..000000000 --- a/_development/helpers.py +++ /dev/null @@ -1,143 +0,0 @@ -# noinspection PyPackageRequirements -import pytest - -import os -import sys -import threading - -from sqlalchemy import MetaData, create_engine -from sqlalchemy.orm import sessionmaker - -script_dir = os.path.dirname(os.path.abspath(__file__)) -# Add root folder to python paths -sys.path.append(os.path.realpath(os.path.join(script_dir, '..', '..'))) -sys._called_from_test = True - -# noinspection PyUnresolvedReferences,PyUnusedLocal -@pytest.fixture -def DBInMemory_test(): - def rollback(): - with sd_lock: - saveddata_session.rollback() - - - print("Creating database in memory") - from os.path import realpath, join, dirname, abspath - - debug = False - gamedataCache = True - saveddataCache = True - gamedata_version = "" - gamedata_connectionstring = 'sqlite:///' + realpath(join(dirname(abspath(unicode(__file__))), "..", "eve.db")) - saveddata_connectionstring = 'sqlite:///:memory:' - - class ReadOnlyException(Exception): - pass - - if callable(gamedata_connectionstring): - gamedata_engine = create_engine("sqlite://", creator=gamedata_connectionstring, echo=debug) - else: - gamedata_engine = create_engine(gamedata_connectionstring, echo=debug) - - gamedata_meta = MetaData() - gamedata_meta.bind = gamedata_engine - gamedata_session = sessionmaker(bind=gamedata_engine, autoflush=False, expire_on_commit=False)() - - # This should be moved elsewhere, maybe as an actual query. Current, without try-except, it breaks when making a new - # game db because we haven't reached gamedata_meta.create_all() - try: - gamedata_version = gamedata_session.execute( - "SELECT `field_value` FROM `metadata` WHERE `field_name` LIKE 'client_build'" - ).fetchone()[0] - except Exception as e: - print("Missing gamedata version.") - gamedata_version = None - - if saveddata_connectionstring is not None: - if callable(saveddata_connectionstring): - saveddata_engine = create_engine(creator=saveddata_connectionstring, echo=debug) - else: - saveddata_engine = create_engine(saveddata_connectionstring, echo=debug) - - saveddata_meta = MetaData() - saveddata_meta.bind = saveddata_engine - saveddata_session = sessionmaker(bind=saveddata_engine, autoflush=False, expire_on_commit=False)() - else: - saveddata_meta = None - - # Lock controlling any changes introduced to session - sd_lock = threading.Lock() - - # Import all the definitions for all our database stuff - # noinspection PyPep8 - #from eos.db.gamedata import alphaClones, attribute, category, effect, group, icon, item, marketGroup, metaData, metaGroup, queries, traits, unit - # noinspection PyPep8 - #from eos.db.saveddata import booster, cargo, character, crest, damagePattern, databaseRepair, drone, fighter, fit, implant, implantSet, loadDefaultDatabaseValues, miscData, module, override, price, queries, skill, targetResists, user - - # If using in memory saveddata, you'll want to reflect it so the data structure is good. - if saveddata_connectionstring == "sqlite:///:memory:": - saveddata_meta.create_all() - - # Output debug info to help us troubleshoot Travis - print(saveddata_engine) - print(gamedata_engine) - - helper = { - #'config': eos.config, - 'gamedata_session' : gamedata_session, - 'saveddata_session' : saveddata_session, - } - return helper - -# noinspection PyUnresolvedReferences,PyUnusedLocal -@pytest.fixture -def DBInMemory(): - print("Creating database in memory") - - import eos.config - - import eos - import eos.db - - # Output debug info to help us troubleshoot Travis - print(eos.db.saveddata_engine) - print(eos.db.gamedata_engine) - - helper = { - 'config': eos.config, - 'db' : eos.db, - 'gamedata_session' : eos.db.gamedata_session, - 'saveddata_session' : eos.db.saveddata_session, - } - return helper - - -@pytest.fixture -def Gamedata(): - print("Building Gamedata") - from eos.gamedata import Item - - helper = { - 'Item': Item, - } - return helper - - -@pytest.fixture -def Saveddata(): - print("Building Saveddata") - from eos.saveddata.ship import Ship - from eos.saveddata.fit import Fit - from eos.saveddata.character import Character - from eos.saveddata.module import Module, State - from eos.saveddata.citadel import Citadel - - helper = { - 'Structure': Citadel, - 'Ship' : Ship, - 'Fit' : Fit, - 'Character': Character, - 'Module' : Module, - 'State' : State, - } - return helper diff --git a/_development/helpers_fits.py b/_development/helpers_fits.py deleted file mode 100644 index f164a6bb3..000000000 --- a/_development/helpers_fits.py +++ /dev/null @@ -1,66 +0,0 @@ -import pytest - -# noinspection PyPackageRequirements -from _development.helpers import DBInMemory as DB, Gamedata, Saveddata - - -# noinspection PyShadowingNames -@pytest.fixture -def RifterFit(DB, Gamedata, Saveddata): - print("Creating Rifter") - item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Rifter").first() - ship = Saveddata['Ship'](item) - # setup fit - fit = Saveddata['Fit'](ship, "My Rifter Fit") - - return fit - - -# noinspection PyShadowingNames -@pytest.fixture -def KeepstarFit(DB, Gamedata, Saveddata): - print("Creating Keepstar") - item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Keepstar").first() - ship = Saveddata['Structure'](item) - # setup fit - fit = Saveddata['Fit'](ship, "Keepstar Fit") - - return fit - - -# noinspection PyShadowingNames -@pytest.fixture -def CurseFit(DB, Gamedata, Saveddata): - print("Creating Curse - With Neuts") - item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Curse").first() - ship = Saveddata['Ship'](item) - # setup fit - fit = Saveddata['Fit'](ship, "Curse - With Neuts") - - mod = Saveddata['Module'](DB['db'].getItem("Medium Energy Neutralizer II")) - mod.state = Saveddata['State'].ONLINE - - # Add 5 neuts - for _ in xrange(5): - fit.modules.append(mod) - - return fit - - -# noinspection PyShadowingNames -@pytest.fixture -def HeronFit(DB, Gamedata, Saveddata): - print("Creating Heron - RemoteSebo") - item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Heron").first() - ship = Saveddata['Ship'](item) - # setup fit - fit = Saveddata['Fit'](ship, "Heron - RemoteSebo") - - mod = Saveddata['Module'](DB['db'].getItem("Remote Sensor Booster II")) - mod.state = Saveddata['State'].ONLINE - - # Add 5 neuts - for _ in xrange(4): - fit.modules.append(mod) - - return fit diff --git a/eos/config.py b/eos/config.py index 38371e299..75f797d9c 100644 --- a/eos/config.py +++ b/eos/config.py @@ -1,17 +1,29 @@ import sys from os.path import realpath, join, dirname, abspath +from logbook import Logger +import os + +istravis = os.environ.get('TRAVIS') == 'true' +pyfalog = Logger(__name__) + debug = False gamedataCache = True saveddataCache = True gamedata_version = "" -gamedata_connectionstring = 'sqlite:///' + unicode(realpath(join(dirname(abspath(__file__)), "..", "eve.db")), - sys.getfilesystemencoding()) -saveddata_connectionstring = 'sqlite:///' + unicode( - realpath(join(dirname(abspath(__file__)), "..", "saveddata", "saveddata.db")), sys.getfilesystemencoding()) +gamedata_connectionstring = 'sqlite:///' + unicode(realpath(join(dirname(abspath(__file__)), "..", "eve.db")), sys.getfilesystemencoding()) +pyfalog.debug("Gamedata connection string: {0}", gamedata_connectionstring) + +if istravis is True or hasattr(sys, '_called_from_test'): + # Running in Travis. Run saveddata database in memory. + saveddata_connectionstring = 'sqlite:///:memory:' +else: + saveddata_connectionstring = 'sqlite:///' + unicode(realpath(join(dirname(abspath(__file__)), "..", "saveddata", "saveddata.db")), sys.getfilesystemencoding()) + +pyfalog.debug("Saveddata connection string: {0}", saveddata_connectionstring) settings = { - "setting1": True + "useStaticAdaptiveArmorHardener": False } # Autodetect path, only change if the autodetection bugs out. diff --git a/eos/saveddata/module.py b/eos/saveddata/module.py index 8287f349a..919b8c7b7 100644 --- a/eos/saveddata/module.py +++ b/eos/saveddata/module.py @@ -199,6 +199,10 @@ class Module(HandledItem, HandledCharge, ItemAttrShortcut, ChargeAttrShortcut): if self.owner: return self.owner.modules.index(self) + @property + def isCapitalSize(self): + return self.getModifiedItemAttr("volume", 0) >= 4000 + @property def hpBeforeReload(self): """ @@ -418,6 +422,11 @@ class Module(HandledItem, HandledCharge, ItemAttrShortcut, ChargeAttrShortcut): if isinstance(fit.ship, Citadel) and len(fitsOnGroup) == 0 and len(fitsOnType) == 0: return False + # EVE doesn't let capital modules be fit onto subcapital hulls. Confirmed by CCP Larrikin that this is dictated + # by the modules volume. See GH issue #1096 + if (fit.ship.getModifiedItemAttr("isCapitalSize", 0) != 1 and self.isCapitalSize): + return False + # If the mod is a subsystem, don't let two subs in the same slot fit if self.slot == Slot.SUBSYSTEM: subSlot = self.getModifiedItemAttr("subSystemSlot") diff --git a/service/settings.py b/service/settings.py index 3da94b084..a45d58980 100644 --- a/service/settings.py +++ b/service/settings.py @@ -46,36 +46,61 @@ class SettingsProvider(object): if not os.path.exists(self.BASE_PATH): os.mkdir(self.BASE_PATH) + # def getSettings(self, area, defaults=None): + # # type: (basestring, dict) -> service.Settings + # # NOTE: needed to change for tests + # settings_obj = self.settings.get(area) + # + # if settings_obj is None and hasattr(self, 'BASE_PATH'): + # canonical_path = os.path.join(self.BASE_PATH, area) + # + # if not os.path.exists(canonical_path): + # info = {} + # if defaults: + # for item in defaults: + # info[item] = defaults[item] + # + # else: + # try: + # f = open(canonical_path, "rb") + # info = cPickle.load(f) + # for item in defaults: + # if item not in info: + # info[item] = defaults[item] + # + # except: + # info = {} + # if defaults: + # for item in defaults: + # info[item] = defaults[item] + # + # self.settings[area] = settings_obj = Settings(canonical_path, info) + # + # return settings_obj def getSettings(self, area, defaults=None): - - s = self.settings.get(area) - - if s is None and hasattr(self, 'BASE_PATH'): - p = os.path.join(self.BASE_PATH, area) - - if not os.path.exists(p): + # type: (basestring, dict) -> service.Settings + # NOTE: needed to change for tests + # TODO: Write to memory with mmap -> https://docs.python.org/2/library/mmap.html + settings_obj = self.settings.get(area) + if settings_obj is None: # and hasattr(self, 'BASE_PATH'): + canonical_path = os.path.join(self.BASE_PATH, area) if hasattr(self, 'BASE_PATH') else "" + if not os.path.exists(canonical_path): # path string or empty string. info = {} if defaults: - for item in defaults: - info[item] = defaults[item] - + info.update(defaults) else: try: - f = open(p, "rb") - info = cPickle.load(f) + with open(canonical_path, "rb") as f: + info = cPickle.load(f) for item in defaults: if item not in info: info[item] = defaults[item] - except: info = {} - if defaults: - for item in defaults: - info[item] = defaults[item] + info.update(defaults) - self.settings[area] = s = Settings(p, info) - - return s + self.settings[area] = settings_obj = Settings(canonical_path, info) + return settings_obj def saveAll(self): for settings in self.settings.itervalues(): @@ -84,12 +109,22 @@ class SettingsProvider(object): class Settings(object): def __init__(self, location, info): + # type: (basestring, dict) -> None + # path string or empty string. self.location = location self.info = info + # def save(self): + # f = open(self.location, "wb") + # cPickle.dump(self.info, f, cPickle.HIGHEST_PROTOCOL) + def save(self): - f = open(self.location, "wb") - cPickle.dump(self.info, f, cPickle.HIGHEST_PROTOCOL) + # NOTE: needed to change for tests + if self.location is None or not self.location: + return + # NOTE: with + open -> file handle auto close + with open(self.location, "wb") as f: + cPickle.dump(self.info, f, cPickle.HIGHEST_PROTOCOL) def __getitem__(self, k): try: diff --git a/tests/test_fitcache.py b/tests/test_fitcache.py deleted file mode 100644 index 9de98d097..000000000 --- a/tests/test_fitcache.py +++ /dev/null @@ -1,232 +0,0 @@ -# Add root folder to python paths -# This must be done on every test in order to pass in Travis -import os -import sys -from time import time -script_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.realpath(os.path.join(script_dir, '..'))) - -# -# noinspection PyPackageRequirements -from _development.helpers import DBInMemory as DB, Gamedata, Saveddata -# noinspection PyPackageRequirements -from _development.helpers_fits import RifterFit, KeepstarFit, HeronFit, CurseFit -from service.fit import Fit -# -# # Fake import wx -# # todo: fix this -# # from types import ModuleType -# # wx = ModuleType("fake_module") -# # sys.modules[wx.__name__] = wx -# -# def test_getAllFits(DB, RifterFit, KeepstarFit): -# assert len(Fit.getAllFits()) == 0 -# DB['db'].save(RifterFit) -# assert len(Fit.getAllFits()) == 1 -# DB['db'].save(KeepstarFit) -# assert len(Fit.getAllFits()) == 2 -# -# # Cleanup after ourselves -# DB['db'].remove(RifterFit) -# DB['db'].remove(KeepstarFit) -# -# -# def test_getFitsWithShip_RifterFit(DB, RifterFit): -# DB['db'].save(RifterFit) -# -# assert Fit.getFitsWithShip(587)[0][1] == 'My Rifter Fit' -# -# DB['db'].remove(RifterFit) - - -def test_RifterSingleNew(DB, RifterFit, KeepstarFit, HeronFit, CurseFit): - DB['db'].save(RifterFit) - DB['db'].save(KeepstarFit) - DB['db'].save(HeronFit) - DB['db'].save(CurseFit) - sFit = Fit.getInstance() - sFit.serviceFittingOptions = { - "useGlobalCharacter" : False, - "useGlobalDamagePattern": False, - "useGlobalForceReload" : False, - "colorFitBySlot" : False, - "rackSlots" : True, - "rackLabels" : True, - "compactSkills" : True, - "showTooltip" : True, - "showMarketShortcuts" : False, - "enableGaugeAnimation" : True, - "exportCharges" : True, - "openFitInNew" : False, - "priceSystem" : "Jita", - "showShipBrowserTooltip": True, - } - - cached_fits = [] - fit = DB["db"].getFit(1) - cached_fits.append(fit) - fit = None - - time_start = time() - - for _ in xrange(1000000): - - fit = next((x for x in cached_fits if x.ID == 1), None) - - fit = None - - print("1000000 of the Rifter fit (new): " + str(time()-time_start)) - - # fit = DB["db"].getFit(1) - - # Cleanup after ourselves - DB['db'].remove(RifterFit) - DB['db'].remove(KeepstarFit) - DB['db'].remove(HeronFit) - DB['db'].remove(CurseFit) - -def test_RifterSingleOld(DB, RifterFit, KeepstarFit, HeronFit, CurseFit): - DB['db'].save(RifterFit) - DB['db'].save(KeepstarFit) - DB['db'].save(HeronFit) - DB['db'].save(CurseFit) - sFit = Fit.getInstance() - sFit.serviceFittingOptions = { - "useGlobalCharacter" : False, - "useGlobalDamagePattern": False, - "useGlobalForceReload" : False, - "colorFitBySlot" : False, - "rackSlots" : True, - "rackLabels" : True, - "compactSkills" : True, - "showTooltip" : True, - "showMarketShortcuts" : False, - "enableGaugeAnimation" : True, - "exportCharges" : True, - "openFitInNew" : False, - "priceSystem" : "Jita", - "showShipBrowserTooltip": True, - } - - cached_fits = [] - fit = DB["db"].getFit(1) - cached_fits.append(fit) - fit = None - - time_start = time() - - for _ in xrange(1000000): - - fit = DB["db"].getFit(1) - - fit = None - - print("1000000 of the Rifter fit (old): " + str(time()-time_start)) - - # Cleanup after ourselves - DB['db'].remove(RifterFit) - DB['db'].remove(KeepstarFit) - DB['db'].remove(HeronFit) - DB['db'].remove(CurseFit) - -def test_FourNew(DB, RifterFit, KeepstarFit, HeronFit, CurseFit): - DB['db'].save(RifterFit) - DB['db'].save(KeepstarFit) - DB['db'].save(HeronFit) - DB['db'].save(CurseFit) - sFit = Fit.getInstance() - sFit.serviceFittingOptions = { - "useGlobalCharacter" : False, - "useGlobalDamagePattern": False, - "useGlobalForceReload" : False, - "colorFitBySlot" : False, - "rackSlots" : True, - "rackLabels" : True, - "compactSkills" : True, - "showTooltip" : True, - "showMarketShortcuts" : False, - "enableGaugeAnimation" : True, - "exportCharges" : True, - "openFitInNew" : False, - "priceSystem" : "Jita", - "showShipBrowserTooltip": True, - } - - cached_fits = [] - fit = DB["db"].getFit(1) - cached_fits.append(fit) - fit = None - - time_start = time() - - for _ in xrange(250000): - - fit = next((x for x in cached_fits if x.ID == 1), None) - fit = None - fit = next((x for x in cached_fits if x.ID == 2), None) - fit = None - fit = next((x for x in cached_fits if x.ID == 3), None) - fit = None - fit = next((x for x in cached_fits if x.ID == 4), None) - fit = None - - - print("1000000 of the four fits (new): " + str(time()-time_start)) - - # fit = DB["db"].getFit(1) - - # Cleanup after ourselves - DB['db'].remove(RifterFit) - DB['db'].remove(KeepstarFit) - DB['db'].remove(HeronFit) - DB['db'].remove(CurseFit) - -def test_FourOld(DB, RifterFit, KeepstarFit, HeronFit, CurseFit): - DB['db'].save(RifterFit) - DB['db'].save(KeepstarFit) - DB['db'].save(HeronFit) - DB['db'].save(CurseFit) - sFit = Fit.getInstance() - sFit.serviceFittingOptions = { - "useGlobalCharacter" : False, - "useGlobalDamagePattern": False, - "useGlobalForceReload" : False, - "colorFitBySlot" : False, - "rackSlots" : True, - "rackLabels" : True, - "compactSkills" : True, - "showTooltip" : True, - "showMarketShortcuts" : False, - "enableGaugeAnimation" : True, - "exportCharges" : True, - "openFitInNew" : False, - "priceSystem" : "Jita", - "showShipBrowserTooltip": True, - } - - cached_fits = [] - fit = DB["db"].getFit(1) - cached_fits.append(fit) - fit = None - - time_start = time() - - for _ in xrange(250000): - - fit = DB["db"].getFit(1) - fit = None - fit = DB["db"].getFit(2) - fit = None - fit = DB["db"].getFit(3) - fit = None - fit = DB["db"].getFit(4) - fit = None - - print("1000000 of the four fits (old): " + str(time()-time_start)) - - # Cleanup after ourselves - DB['db'].remove(RifterFit) - DB['db'].remove(KeepstarFit) - DB['db'].remove(HeronFit) - DB['db'].remove(CurseFit) - diff --git a/tests/test_unread_desc.py b/tests/test_unread_desc.py index 8cd2d40a6..abe30af4f 100644 --- a/tests/test_unread_desc.py +++ b/tests/test_unread_desc.py @@ -1,23 +1,26 @@ """ 2017/04/05: unread description tests module. """ +# noinspection PyPackageRequirements +import pytest # Add root folder to python paths # This must be done on every test in order to pass in Travis import os import sys -script_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.realpath(os.path.join(script_dir, '..'))) - +# nopep8 import re -# -# noinspection PyPackageRequirements -from _development.helpers import DBInMemory as DB, Gamedata, Saveddata -# noinspection PyPep8 -from service.port import Port, IPortUser - # from utils.strfunctions import sequential_rep, replace_ltgt from utils.stopwatch import Stopwatch +script_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.realpath(os.path.join(script_dir, '..'))) +sys._called_from_test = True # need db open for tests. (see eos/config.py#17 +# noinspection PyPep8 +from service.port import Port, IPortUser +# +# noinspection PyPackageRequirements +# from _development.helpers import DBInMemory as DB + """ NOTE: description character length is restricted 4hundred by EVE client. @@ -41,6 +44,9 @@ NOTE of @decorator: o Function to receive arguments of function to be decorated o A function that accepts the decorate target function itself as an argument o A function that accepts arguments of the decorator itself + +for local coverage: + py.test --cov=./ --cov-report=html """ class PortUser(IPortUser): @@ -52,11 +58,24 @@ class PortUser(IPortUser): stpw = Stopwatch('test measurementer') -def test_import_xml(): +@pytest.fixture() +def print_db_info(): + # Output debug info + import eos + print + print "------------ data base connection info ------------" + print(eos.db.saveddata_engine) + print(eos.db.gamedata_engine) + print + + +# noinspection PyUnusedLocal +def test_import_xml(print_db_info): usr = PortUser() # for path in XML_FILES: xml_file = "jeffy_ja-en[99].xml" fit_count = int(re.search(r"\[(\d+)\]", xml_file).group(1)) + fits = None with open(os.path.join(script_dir, xml_file), "r") as file_: srcString = file_.read() srcString = unicode(srcString, "utf-8")