Merge remote-tracking branch 'jeffy/fix_unread_description' into dev, see #1099

Conflicts:
	eos/config.py
	gui/notesView.py
	service/settings.py

eos/saveddata/ship.py

@@ -71,6 +71,11 @@ class Ship(ItemAttrShortcut, HandledItem):
     def item(self):
         return self.__item
 
+    @property
+    def name(self):
+        # NOTE: add name property
+        return self.__item.name
+
     @property
     def itemModifiedAttributes(self):
         return self.__itemModifiedAttributes

gui/mainFrame.py

@@ -73,7 +73,7 @@ from service.update import Update
 from eos.modifiedAttributeDict import ModifiedAttributeDict
 from eos.db.saveddata.loadDefaultDatabaseValues import DefaultDatabaseValues
 from eos.db.saveddata.queries import getFit as db_getFit
-from service.port import Port
+from service.port import Port, IPortUser
 from service.settings import HTMLExportSettings
 
 from time import gmtime, strftime
@@ -138,7 +138,7 @@ class OpenFitsThread(threading.Thread):
         wx.CallAfter(self.callback)
 
 
-class MainFrame(wx.Frame):
+class MainFrame(wx.Frame, IPortUser):
     __instance = None
 
     @classmethod
@@ -422,8 +422,7 @@ class MainFrame(wx.Frame):
             format_ = dlg.GetFilterIndex()
             path = dlg.GetPath()
             if format_ == 0:
-                sPort = Port.getInstance()
-                output = sPort.exportXml(None, fit)
+                output = Port.exportXml(None, fit)
                 if '.' not in os.path.basename(path):
                     path += ".xml"
             else:
@@ -814,7 +813,6 @@ class MainFrame(wx.Frame):
 
     def fileImportDialog(self, event):
         """Handles importing single/multiple EVE XML / EFT cfg fit files"""
-        sPort = Port.getInstance()
         dlg = wx.FileDialog(
             self,
             "Open One Or More Fitting Files",
@@ -828,10 +826,10 @@ class MainFrame(wx.Frame):
                 "Importing fits",
                 " " * 100,  # set some arbitrary spacing to create width in window
                 parent=self,
-                style=wx.PD_APP_MODAL | wx.PD_ELAPSED_TIME
+                style=wx.PD_CAN_ABORT | wx.PD_SMOOTH | wx.PD_ELAPSED_TIME | wx.PD_APP_MODAL
             )
-            self.progressDialog.message = None
-            sPort.importFitsThreaded(dlg.GetPaths(), self.fileImportCallback)
+            # self.progressDialog.message = None
+            Port.importFitsThreaded(dlg.GetPaths(), self)
             self.progressDialog.ShowModal()
             try:
                 dlg.Destroy()
@@ -863,9 +861,9 @@ class MainFrame(wx.Frame):
                 "Backing up %d fits to: %s" % (max_, filePath),
                 maximum=max_,
                 parent=self,
-                style=wx.PD_APP_MODAL | wx.PD_ELAPSED_TIME,
+                style=wx.PD_CAN_ABORT | wx.PD_SMOOTH | wx.PD_ELAPSED_TIME | wx.PD_APP_MODAL
             )
-            Port().backupFits(filePath, self.backupCallback)
+            Port.backupFits(filePath, self)
             self.progressDialog.ShowModal()
 
     def exportHtml(self, event):
@@ -903,7 +901,19 @@ class MainFrame(wx.Frame):
         else:
             self.progressDialog.Update(info)
 
-    def fileImportCallback(self, action, data=None):
+    def on_port_process_start(self):
+        # flag for progress dialog.
+        self.__progress_flag = True
+
+    def on_port_processing(self, action, data=None):
+        # 2017/03/29 NOTE: implementation like interface
+        wx.CallAfter(
+            self._on_port_processing, action, data
+        )
+
+        return self.__progress_flag
+
+    def _on_port_processing(self, action, data):
         """
         While importing fits from file, the logic calls back to this function to
         update progress bar to show activity. XML files can contain multiple
@@ -917,22 +927,38 @@ class MainFrame(wx.Frame):
                 1: Replace message with data
                 other: Close dialog and handle based on :action (-1 open fits, -2 display error)
         """
-
-        if action is None:
-            self.progressDialog.Pulse()
-        elif action == 1 and data != self.progressDialog.message:
-            self.progressDialog.message = data
-            self.progressDialog.Pulse(data)
-        else:
+        _message = None
+        if action & IPortUser.ID_ERROR:
             self.closeProgressDialog()
-            if action == -1:
-                self._openAfterImport(data)
-            elif action == -2:
-                dlg = wx.MessageDialog(self,
-                                       "The following error was generated\n\n%s\n\nBe aware that already processed fits were not saved" % data,
-                                       "Import Error", wx.OK | wx.ICON_ERROR)
-                if dlg.ShowModal() == wx.ID_OK:
-                    return
+            _message = "Import Error" if action & IPortUser.PROCESS_IMPORT else "Export Error"
+            dlg = wx.MessageDialog(self,
+                                   "The following error was generated\n\n%s\n\nBe aware that already processed fits were not saved" % data,
+                                   _message, wx.OK | wx.ICON_ERROR)
+            # if dlg.ShowModal() == wx.ID_OK:
+            #     return
+            dlg.ShowModal()
+            return
+
+        # data is str
+        if action & IPortUser.PROCESS_IMPORT:
+            if action & IPortUser.ID_PULSE:
+                _message = ()
+            # update message
+            elif action & IPortUser.ID_UPDATE:  # and data != self.progressDialog.message:
+                _message = data
+
+            if _message is not None:
+                self.__progress_flag, _unuse = self.progressDialog.Pulse(_message)
+            else:
+                self.closeProgressDialog()
+                if action & IPortUser.ID_DONE:
+                    self._openAfterImport(data)
+        # data is tuple(int, str)
+        elif action & IPortUser.PROCESS_EXPORT:
+            if action & IPortUser.ID_DONE:
+                self.closeProgressDialog()
+            else:
+                self.__progress_flag, _unuse = self.progressDialog.Update(data[0], data[1])
 
     def _openAfterImport(self, fits):
         if len(fits) > 0:

service/port.py

@@ -46,6 +46,8 @@ from eos.saveddata.ship import Ship
 from eos.saveddata.citadel import Citadel
 from eos.saveddata.fit import Fit
 from service.market import Market
+from utils.strfunctions import sequential_rep, replace_ltgt
+from abc import ABCMeta, abstractmethod
 
 if 'wxMac' not in wx.PlatformInfo or ('wxMac' in wx.PlatformInfo and wx.VERSION >= (3, 0)):
     from service.crest import Crest
@@ -70,9 +72,152 @@ INV_FLAG_CARGOBAY = 5
 INV_FLAG_DRONEBAY = 87
 INV_FLAG_FIGHTER = 158
 
+# 2017/04/05 NOTE: simple validation, for xml file
+RE_XML_START = r'<\?xml\s+version="1.0"\s*\?>'
+
+# -- 170327 Ignored description --
+RE_LTGT = "&(lt|gt);"
+L_MARK = "&lt;localized hint=&quot;"
+# &lt;localized hint=&quot;([^"]+)&quot;&gt;([^\*]+)\*&lt;\/localized&gt;
+LOCALIZED_PATTERN = re.compile(r'<localized hint="([^"]+)">([^\*]+)\*</localized>')
+
+
+def _extract_match(t):
+    m = LOCALIZED_PATTERN.match(t)
+    # hint attribute, text content
+    return m.group(1), m.group(2)
+
+
+def _resolve_ship(fitting, sMkt, b_localized):
+    # type: (xml.dom.minidom.Element, service.market.Market, bool) -> eos.saveddata.fit.Fit
+    """ NOTE: Since it is meaningless unless a correct ship object can be constructed,
+        process flow changed
+    """
+    # ------ Confirm ship
+    # <localized hint="Maelstrom">Maelstrom</localized>
+    shipType = fitting.getElementsByTagName("shipType").item(0).getAttribute("value")
+    anything = None
+    if b_localized:
+        # expect an official name, emergency cache
+        shipType, anything = _extract_match(shipType)
+
+    limit = 2
+    ship = None
+    while True:
+        must_retry = False
+        try:
+            try:
+                ship = Ship(sMkt.getItem(shipType))
+            except ValueError:
+                ship = Citadel(sMkt.getItem(shipType))
+        except Exception as e:
+            pyfalog.warning("Caught exception on _resolve_ship")
+            pyfalog.error(e)
+            limit -= 1
+            if limit is 0:
+                break
+            shipType = anything
+            must_retry = True
+        if not must_retry:
+            break
+
+    if ship is None:
+        raise Exception("cannot resolve ship type.")
+
+    fitobj = Fit(ship=ship)
+    # ------ Confirm fit name
+    anything = fitting.getAttribute("name")
+    # 2017/03/29 NOTE:
+    #    if fit name contained "<" or ">" then reprace to named html entity by EVE client
+    # if re.search(RE_LTGT, anything):
+    if "&lt;" in anything or "&gt;" in anything:
+        anything = replace_ltgt(anything)
+    fitobj.name = anything
+
+    return fitobj
+
+
+def _resolve_module(hardware, sMkt, b_localized):
+    # type: (xml.dom.minidom.Element, service.market.Market, bool) -> eos.saveddata.module.Module
+    moduleName = hardware.getAttribute("type")
+    emergency = None
+    if b_localized:
+        # expect an official name, emergency cache
+        moduleName, emergency = _extract_match(moduleName)
+
+    item = None
+    limit = 2
+    while True:
+        must_retry = False
+        try:
+            item = sMkt.getItem(moduleName, eager="group.category")
+        except Exception as e:
+            pyfalog.warning("Caught exception on _resolve_module")
+            pyfalog.error(e)
+            limit -= 1
+            if limit is 0:
+                break
+            moduleName = emergency
+            must_retry = True
+        if not must_retry:
+            break
+    return item
+
+
+class UserCancelException(Exception):
+    """when user cancel on port processing."""
+    pass
+
+
+class IPortUser:
+
+    __metaclass__ = ABCMeta
+
+    ID_PULSE = 1
+    # Pulse the progress bar
+    ID_UPDATE = ID_PULSE << 1
+    # Replace message with data: update messate
+    ID_DONE = ID_PULSE << 2
+    # open fits: import process done
+    ID_ERROR = ID_PULSE << 3
+    # display error: raise some error
+
+    PROCESS_IMPORT = ID_PULSE << 4
+    # means import process.
+    PROCESS_EXPORT = ID_PULSE << 5
+    # means import process.
+
+    @abstractmethod
+    def on_port_processing(self, action, data=None):
+        """
+        While importing fits from file, the logic calls back to this function to
+        update progress bar to show activity. XML files can contain multiple
+        ships with multiple fits, whereas EFT cfg files contain many fits of
+        a single ship. When iterating through the files, we update the message
+        when we start a new file, and then Pulse the progress bar with every fit
+        that is processed.
+
+        action : a flag that lets us know how to deal with :data
+                None: Pulse the progress bar
+                1: Replace message with data
+                other: Close dialog and handle based on :action (-1 open fits, -2 display error)
+        """
+
+        """return: True is continue process, False is cancel."""
+        pass
+
+    def on_port_process_start(self):
+        pass
+
 
 class Port(object):
+    """
+    2017/03/31 NOTE: About change
+        1. want to keep the description recorded in fit
+        2. i think should not write wx.CallAfter in here
+    """
     instance = None
+    __tag_replace_flag = True
 
     @classmethod
     def getInstance(cls):
@@ -81,20 +226,44 @@ class Port(object):
 
         return cls.instance
 
+    @classmethod
+    def set_tag_replace(cls, b):
+        cls.__tag_replace_flag = b
+
+    @classmethod
+    def is_tag_replace(cls):
+        # might there is a person who wants to hold tags.
+        # (item link in EVE client etc. When importing again to EVE)
+        return cls.__tag_replace_flag
+
     @staticmethod
-    def backupFits(path, callback):
+    def backupFits(path, iportuser):
         pyfalog.debug("Starting backup fits thread.")
-        thread = FitBackupThread(path, callback)
-        thread.start()
+#         thread = FitBackupThread(path, callback)
+#         thread.start()
+        threading.Thread(
+            target=PortProcessing.backupFits,
+            args=(path, iportuser)
+        ).start()
 
     @staticmethod
-    def importFitsThreaded(paths, callback):
+    def importFitsThreaded(paths, iportuser):
+        # type: (tuple, IPortUser) -> None
+        """
+        :param paths: fits data file path list.
+        :param iportuser:  IPortUser implemented class.
+        :rtype: None
+        """
         pyfalog.debug("Starting import fits thread.")
-        thread = FitImportThread(paths, callback)
-        thread.start()
+#         thread = FitImportThread(paths, iportuser)
+#         thread.start()
+        threading.Thread(
+            target=PortProcessing.importFitsFromFile,
+            args=(paths, iportuser)
+        ).start()
 
     @staticmethod
-    def importFitFromFiles(paths, callback=None):
+    def importFitFromFiles(paths, iportuser=None):
         """
         Imports fits from file(s). First processes all provided paths and stores
         assembled fits into a list. This allows us to call back to the GUI as
@@ -104,99 +273,109 @@ class Port(object):
         defcodepage = locale.getpreferredencoding()
         sFit = svcFit.getInstance()
 
-        fits = []
-        for path in paths:
-            if callback:  # Pulse
-                pyfalog.debug("Processing file:\n{0}", path)
-                wx.CallAfter(callback, 1, "Processing file:\n%s" % path)
+        fit_list = []
+        try:
+            for path in paths:
+                if iportuser:  # Pulse
+                    msg = "Processing file:\n%s" % path
+                    pyfalog.debug(msg)
+                    PortProcessing.notify(iportuser, IPortUser.PROCESS_IMPORT | IPortUser.ID_UPDATE, msg)
+                    # wx.CallAfter(callback, 1, msg)
 
-            file_ = open(path, "r")
-            srcString = file_.read()
+                with open(path, "r") as file_:
+                    srcString = file_.read()
 
-            if len(srcString) == 0:  # ignore blank files
-                pyfalog.debug("File is blank.")
-                continue
+                if len(srcString) == 0:  # ignore blank files
+                    pyfalog.debug("File is blank.")
+                    continue
 
-            codec_found = None
-            # If file had ANSI encoding, decode it to unicode using detection
-            # of BOM header or if there is no header try default
-            # codepage then fallback to utf-16, cp1252
+                codec_found = None
+                # If file had ANSI encoding, decode it to unicode using detection
+                # of BOM header or if there is no header try default
+                # codepage then fallback to utf-16, cp1252
 
-            if isinstance(srcString, str):
-                savebom = None
+                if isinstance(srcString, str):
+                    savebom = None
 
-                encoding_map = (
-                    ('\xef\xbb\xbf', 'utf-8'),
-                    ('\xff\xfe\0\0', 'utf-32'),
-                    ('\0\0\xfe\xff', 'UTF-32BE'),
-                    ('\xff\xfe', 'utf-16'),
-                    ('\xfe\xff', 'UTF-16BE'))
+                    encoding_map = (
+                        ('\xef\xbb\xbf', 'utf-8'),
+                        ('\xff\xfe\0\0', 'utf-32'),
+                        ('\0\0\xfe\xff', 'UTF-32BE'),
+                        ('\xff\xfe', 'utf-16'),
+                        ('\xfe\xff', 'UTF-16BE'))
 
-                for bom, encoding in encoding_map:
-                    if srcString.startswith(bom):
-                        codec_found = encoding
-                        savebom = bom
+                    for bom, encoding in encoding_map:
+                        if srcString.startswith(bom):
+                            codec_found = encoding
+                            savebom = bom
+
+                    if codec_found is None:
+                        pyfalog.info("Unicode BOM not found in file {0}.", path)
+                        attempt_codecs = (defcodepage, "utf-8", "utf-16", "cp1252")
+
+                        for page in attempt_codecs:
+                            try:
+                                pyfalog.info("Attempting to decode file {0} using {1} page.", path, page)
+                                srcString = unicode(srcString, page)
+                                codec_found = page
+                                pyfalog.info("File {0} decoded using {1} page.", path, page)
+                            except UnicodeDecodeError:
+                                pyfalog.info("Error unicode decoding {0} from page {1}, trying next codec", path, page)
+                            else:
+                                break
+                    else:
+                        pyfalog.info("Unicode BOM detected in {0}, using {1} page.", path, codec_found)
+                        srcString = unicode(srcString[len(savebom):], codec_found)
+
+                else:
+                    # nasty hack to detect other transparent utf-16 loading
+                    if srcString[0] == '<' and 'utf-16' in srcString[:128].lower():
+                        codec_found = "utf-16"
+                    else:
+                        codec_found = "utf-8"
 
                 if codec_found is None:
-                    pyfalog.info("Unicode BOM not found in file {0}.", path)
-                    attempt_codecs = (defcodepage, "utf-8", "utf-16", "cp1252")
+                    return False, "Proper codec could not be established for %s" % path
 
-                    for page in attempt_codecs:
-                        try:
-                            pyfalog.info("Attempting to decode file {0} using {1} page.", path, page)
-                            srcString = unicode(srcString, page)
-                            codec_found = page
-                            pyfalog.info("File {0} decoded using {1} page.", path, page)
-                        except UnicodeDecodeError:
-                            pyfalog.info("Error unicode decoding {0} from page {1}, trying next codec", path, page)
-                        else:
-                            break
-                else:
-                    pyfalog.info("Unicode BOM detected in {0}, using {1} page.", path, codec_found)
-                    srcString = unicode(srcString[len(savebom):], codec_found)
+                try:
+                    _, fitsImport = Port.importAuto(srcString, path, iportuser=iportuser, encoding=codec_found)
+                    fit_list += fitsImport
+                except xml.parsers.expat.ExpatError:
+                    pyfalog.warning("Malformed XML in:\n{0}", path)
+                    return False, "Malformed XML in %s" % path
 
-            else:
-                # nasty hack to detect other transparent utf-16 loading
-                if srcString[0] == '<' and 'utf-16' in srcString[:128].lower():
-                    codec_found = "utf-16"
-                else:
-                    codec_found = "utf-8"
+            # IDs = []  # NOTE: what use for IDs?
+            numFits = len(fit_list)
+            for idx, fit in enumerate(fit_list):
+                # Set some more fit attributes and save
+                fit.character = sFit.character
+                fit.damagePattern = sFit.pattern
+                fit.targetResists = sFit.targetResists
+                db.save(fit)
+                # IDs.append(fit.ID)
+                if iportuser:  # Pulse
+                    pyfalog.debug("Processing complete, saving fits to database: {0}/{1}", idx + 1, numFits)
+                    PortProcessing.notify(
+                        iportuser, IPortUser.PROCESS_IMPORT | IPortUser.ID_UPDATE,
+                        "Processing complete, saving fits to database\n(%d/%d) %s" % (idx + 1, numFits, fit.ship.name)
+                    )
 
-            if codec_found is None:
-                return False, "Proper codec could not be established for %s" % path
+        except UserCancelException:
+            return False, "Processing has been canceled.\n"
+        except Exception as e:
+            pyfalog.critical("Unknown exception processing: {0}", path)
+            pyfalog.critical(e)
+            # TypeError: not all arguments converted during string formatting
+#                 return False, "Unknown Error while processing {0}" % path
+            return False, "Unknown Error while processing %s\n\n message: %s" % (path, e.message)
 
-            try:
-                _, fitsImport = Port.importAuto(srcString, path, callback=callback, encoding=codec_found)
-                fits += fitsImport
-            except xml.parsers.expat.ExpatError:
-                pyfalog.warning("Malformed XML in:\n{0}", path)
-                return False, "Malformed XML in %s" % path
-            except Exception as e:
-                pyfalog.critical("Unknown exception processing: {0}", path)
-                pyfalog.critical(e)
-                return False, "Unknown Error while processing {0}" % path
-
-        IDs = []
-        numFits = len(fits)
-        for i, fit in enumerate(fits):
-            # Set some more fit attributes and save
-            fit.character = sFit.character
-            fit.damagePattern = sFit.pattern
-            fit.targetResists = sFit.targetResists
-            db.save(fit)
-            IDs.append(fit.ID)
-            if callback:  # Pulse
-                pyfalog.debug("Processing complete, saving fits to database: {0}/{1}", i + 1, numFits)
-                wx.CallAfter(
-                    callback, 1,
-                    "Processing complete, saving fits to database\n(%d/%d)" %
-                    (i + 1, numFits)
-                )
-
-        return True, fits
+        return True, fit_list
 
     @staticmethod
     def importFitFromBuffer(bufferStr, activeFit=None):
+        # type: (basestring, object) -> object
+        # TODO: catch the exception?
+        # activeFit is reserved?, bufferStr is unicode? (assume only clipboard string?
         sFit = svcFit.getInstance()
         _, fits = Port.importAuto(bufferStr, activeFit=activeFit)
         for fit in fits:
@@ -228,7 +407,9 @@ class Port(object):
         fit['ship']['id'] = ofit.ship.item.ID
         fit['ship']['name'] = ''
 
-        fit['description'] = "<pyfa:%d />" % ofit.ID
+        # 2017/03/29 NOTE: "<" or "&lt;" is Ignored
+        # fit['description'] = "<pyfa:%d />" % ofit.ID
+        fit['description'] = ofit.notes if ofit.notes is not None else ""
         fit['items'] = []
 
         slotNum = {}
@@ -302,17 +483,18 @@ class Port(object):
         return json.dumps(fit)
 
     @classmethod
-    def importAuto(cls, string, path=None, activeFit=None, callback=None, encoding=None):
+    def importAuto(cls, string, path=None, activeFit=None, iportuser=None, encoding=None):
+        # type: (basestring, basestring, object, IPortUser, basestring) -> object
         # Get first line and strip space symbols of it to avoid possible detection errors
         firstLine = re.split("[\n\r]+", string.strip(), maxsplit=1)[0]
         firstLine = firstLine.strip()
 
         # If XML-style start of tag encountered, detect as XML
-        if re.match("<", firstLine):
+        if re.search(RE_XML_START, firstLine):
             if encoding:
-                return "XML", cls.importXml(string, callback, encoding)
+                return "XML", cls.importXml(string, iportuser, encoding)
             else:
-                return "XML", cls.importXml(string, callback)
+                return "XML", cls.importXml(string, iportuser)
 
         # If JSON-style start, parse as CREST/JSON
         if firstLine[0] == '{':
@@ -323,7 +505,7 @@ class Port(object):
         if re.match("\[.*\]", firstLine) and path is not None:
             filename = os.path.split(path)[1]
             shipName = filename.rsplit('.')[0]
-            return "EFT Config", cls.importEftCfg(shipName, string, callback)
+            return "EFT Config", cls.importEftCfg(shipName, string, iportuser)
 
         # If no file is specified and there's comma between brackets,
         # consider that we have [ship, setup name] and detect like eft export format
@@ -335,22 +517,26 @@ class Port(object):
 
     @staticmethod
     def importCrest(str_):
-        fit = json.loads(str_)
-        sMkt = Market.getInstance()
 
-        f = Fit()
-        f.name = fit['name']
+        sMkt = Market.getInstance()
+        fitobj = Fit()
+        refobj = json.loads(str_)
+        items = refobj['items']
+        # "<" and ">" is replace to "&lt;", "&gt;" by EVE client
+        fitobj.name = refobj['name']
+        # 2017/03/29: read description
+        fitobj.notes = refobj['description']
 
         try:
+            refobj = refobj['ship']['id']
             try:
-                f.ship = Ship(sMkt.getItem(fit['ship']['id']))
+                fitobj.ship = Ship(sMkt.getItem(refobj))
             except ValueError:
-                f.ship = Citadel(sMkt.getItem(fit['ship']['id']))
+                fitobj.ship = Citadel(sMkt.getItem(refobj))
         except:
             pyfalog.warning("Caught exception in importCrest")
             return None
 
-        items = fit['items']
         items.sort(key=lambda k: k['flag'])
 
         moduleList = []
@@ -360,14 +546,14 @@ class Port(object):
                 if module['flag'] == INV_FLAG_DRONEBAY:
                     d = Drone(item)
                     d.amount = module['quantity']
-                    f.drones.append(d)
+                    fitobj.drones.append(d)
                 elif module['flag'] == INV_FLAG_CARGOBAY:
                     c = Cargo(item)
                     c.amount = module['quantity']
-                    f.cargo.append(c)
+                    fitobj.cargo.append(c)
                 elif module['flag'] == INV_FLAG_FIGHTER:
                     fighter = Fighter(item)
-                    f.fighters.append(fighter)
+                    fitobj.fighters.append(fighter)
                 else:
                     try:
                         m = Module(item)
@@ -377,8 +563,8 @@ class Port(object):
                         continue
                     # Add subsystems before modules to make sure T3 cruisers have subsystems installed
                     if item.category.name == "Subsystem":
-                        if m.fits(f):
-                            f.modules.append(m)
+                        if m.fits(fitobj):
+                            fitobj.modules.append(m)
                     else:
                         if m.isValidState(State.ACTIVE):
                             m.state = State.ACTIVE
@@ -390,13 +576,13 @@ class Port(object):
                 continue
 
         # Recalc to get slot numbers correct for T3 cruisers
-        svcFit.getInstance().recalc(f)
+        svcFit.getInstance().recalc(fitobj)
 
         for module in moduleList:
-            if module.fits(f):
-                f.modules.append(module)
+            if module.fits(fitobj):
+                fitobj.modules.append(module)
 
-        return f
+        return fitobj
 
     @staticmethod
     def importDna(string):
@@ -635,7 +821,7 @@ class Port(object):
         return fit
 
     @staticmethod
-    def importEftCfg(shipname, contents, callback=None):
+    def importEftCfg(shipname, contents, iportuser=None):
         """Handle import from EFT config store file"""
 
         # Check if we have such ship in database, bail if we don't
@@ -648,7 +834,7 @@ class Port(object):
         # If client didn't take care of encoding file contents into Unicode,
         # do it using fallback encoding ourselves
         if isinstance(contents, str):
-            contents = unicode(contents, "cp1252")
+            contents = unicode(contents, locale.getpreferredencoding())
 
         fits = []  # List for fits
         fitIndices = []  # List for starting line numbers for each fit
@@ -671,14 +857,14 @@ class Port(object):
 
             try:
                 # Create fit object
-                f = Fit()
+                fitobj = Fit()
                 # Strip square brackets and pull out a fit name
-                f.name = fitLines[0][1:-1]
+                fitobj.name = fitLines[0][1:-1]
                 # Assign ship to fitting
                 try:
-                    f.ship = Ship(sMkt.getItem(shipname))
+                    fitobj.ship = Ship(sMkt.getItem(shipname))
                 except ValueError:
-                    f.ship = Citadel(sMkt.getItem(shipname))
+                    fitobj.ship = Citadel(sMkt.getItem(shipname))
 
                 moduleList = []
                 for x in range(1, len(fitLines)):
@@ -689,6 +875,8 @@ class Port(object):
                     # Parse line into some data we will need
                     misc = re.match("(Drones|Implant|Booster)_(Active|Inactive)=(.+)", line)
                     cargo = re.match("Cargohold=(.+)", line)
+                    # 2017/03/27 NOTE: store description from EFT
+                    description = re.match("Description=(.+)", line)
 
                     if misc:
                         entityType = misc.group(1)
@@ -713,11 +901,11 @@ class Port(object):
                                     d.amountActive = droneAmount
                                 elif entityState == "Inactive":
                                     d.amountActive = 0
-                                f.drones.append(d)
+                                fitobj.drones.append(d)
                             elif droneItem.category.name == "Fighter":  # EFT saves fighter as drones
                                 ft = Fighter(droneItem)
                                 ft.amount = int(droneAmount) if ft.amount <= ft.fighterSquadronMaxSize else ft.fighterSquadronMaxSize
-                                f.fighters.append(ft)
+                                fitobj.fighters.append(ft)
                             else:
                                 continue
                         elif entityType == "Implant":
@@ -735,7 +923,7 @@ class Port(object):
                                 imp.active = True
                             elif entityState == "Inactive":
                                 imp.active = False
-                            f.implants.append(imp)
+                            fitobj.implants.append(imp)
                         elif entityType == "Booster":
                             # Bail if we can't get item or it's not from implant category
                             try:
@@ -752,7 +940,7 @@ class Port(object):
                                 b.active = True
                             elif entityState == "Inactive":
                                 b.active = False
-                            f.boosters.append(b)
+                            fitobj.boosters.append(b)
                     # If we don't have any prefixes, then it's a module
                     elif cargo:
                         cargoData = re.match("(.+),([0-9]+)", cargo.group(1))
@@ -767,7 +955,10 @@ class Port(object):
                         # Add Cargo to the fitting
                         c = Cargo(item)
                         c.amount = cargoAmount
-                        f.cargo.append(c)
+                        fitobj.cargo.append(c)
+                    # 2017/03/27 NOTE: store description from EFT
+                    elif description:
+                        fitobj.notes = description.group(1).replace("|", "\n")
                     else:
                         withCharge = re.match("(.+),(.+)", line)
                         modName = withCharge.group(1) if withCharge else line
@@ -784,10 +975,10 @@ class Port(object):
 
                         # Add subsystems before modules to make sure T3 cruisers have subsystems installed
                         if modItem.category.name == "Subsystem":
-                            if m.fits(f):
-                                f.modules.append(m)
+                            if m.fits(fitobj):
+                                fitobj.modules.append(m)
                         else:
-                            m.owner = f
+                            m.owner = fitobj
                             # Activate mod if it is activable
                             if m.isValidState(State.ACTIVE):
                                 m.state = State.ACTIVE
@@ -804,17 +995,21 @@ class Port(object):
                             moduleList.append(m)
 
                 # Recalc to get slot numbers correct for T3 cruisers
-                svcFit.getInstance().recalc(f)
+                svcFit.getInstance().recalc(fitobj)
 
                 for module in moduleList:
-                    if module.fits(f):
-                        f.modules.append(module)
+                    if module.fits(fitobj):
+                        fitobj.modules.append(module)
 
                 # Append fit to list of fits
-                fits.append(f)
+                fits.append(fitobj)
+
+                if iportuser:  # NOTE: Send current processing status
+                    PortProcessing.notify(
+                        iportuser, IPortUser.PROCESS_IMPORT | IPortUser.ID_UPDATE,
+                        "%s:\n%s" % (fitobj.ship.name, fitobj.name)
+                    )
 
-                if callback:
-                    wx.CallAfter(callback, None)
             # Skip fit silently if we get an exception
             except Exception as e:
                 pyfalog.error("Caught exception on fit.")
@@ -824,93 +1019,100 @@ class Port(object):
         return fits
 
     @staticmethod
-    def importXml(text, callback=None, encoding="utf-8"):
+    def importXml(text, iportuser=None, encoding="utf-8"):
+        # type: (basestring, IPortUser, basestring) -> list[eos.saveddata.fit.Fit]
         sMkt = Market.getInstance()
-
         doc = xml.dom.minidom.parseString(text.encode(encoding))
+        # NOTE:
+        #   When L_MARK is included at this point,
+        #   Decided to be localized data
+        b_localized = L_MARK in text
         fittings = doc.getElementsByTagName("fittings").item(0)
         fittings = fittings.getElementsByTagName("fitting")
-        fits = []
+        fit_list = []
+
+        for fitting in fittings:
+            fitobj = _resolve_ship(fitting, sMkt, b_localized)
+            # -- 170327 Ignored description --
+            # read description from exported xml. (EVE client, EFT)
+            description = fitting.getElementsByTagName("description").item(0).getAttribute("value")
+            if description is None:
+                description = ""
+            elif len(description):
+                # convert <br> to "\n" and remove html tags.
+                if Port.is_tag_replace():
+                    description = replace_ltgt(
+                        sequential_rep(description, r"<(br|BR)>", "\n", r"<[^<>]+>", "")
+                    )
+            fitobj.notes = description
 
-        for i, fitting in enumerate(fittings):
-            f = Fit()
-            f.name = fitting.getAttribute("name")
-            # <localized hint="Maelstrom">Maelstrom</localized>
-            shipType = fitting.getElementsByTagName("shipType").item(0).getAttribute("value")
-            try:
-                try:
-                    f.ship = Ship(sMkt.getItem(shipType))
-                except ValueError:
-                    f.ship = Citadel(sMkt.getItem(shipType))
-            except Exception as e:
-                pyfalog.warning("Caught exception on importXml")
-                pyfalog.error(e)
-                continue
             hardwares = fitting.getElementsByTagName("hardware")
             moduleList = []
             for hardware in hardwares:
                 try:
-                    moduleName = hardware.getAttribute("type")
-                    try:
-                        item = sMkt.getItem(moduleName, eager="group.category")
-                    except Exception as e:
-                        pyfalog.warning("Caught exception on importXml")
-                        pyfalog.error(e)
+                    item = _resolve_module(hardware, sMkt, b_localized)
+                    if not item:
                         continue
-                    if item:
-                        if item.category.name == "Drone":
-                            d = Drone(item)
-                            d.amount = int(hardware.getAttribute("qty"))
-                            f.drones.append(d)
-                        elif item.category.name == "Fighter":
-                            ft = Fighter(item)
-                            ft.amount = int(hardware.getAttribute("qty")) if ft.amount <= ft.fighterSquadronMaxSize else ft.fighterSquadronMaxSize
-                            f.fighters.append(ft)
-                        elif hardware.getAttribute("slot").lower() == "cargo":
-                            # although the eve client only support charges in cargo, third-party programs
-                            # may support items or "refits" in cargo. Support these by blindly adding all
-                            # cargo, not just charges
-                            c = Cargo(item)
-                            c.amount = int(hardware.getAttribute("qty"))
-                            f.cargo.append(c)
-                        else:
-                            try:
-                                m = Module(item)
-                            # When item can't be added to any slot (unknown item or just charge), ignore it
-                            except ValueError:
-                                pyfalog.warning("item can't be added to any slot (unknown item or just charge), ignore it")
-                                continue
-                            # Add subsystems before modules to make sure T3 cruisers have subsystems installed
-                            if item.category.name == "Subsystem":
-                                if m.fits(f):
-                                    m.owner = f
-                                    f.modules.append(m)
-                            else:
-                                if m.isValidState(State.ACTIVE):
-                                    m.state = State.ACTIVE
 
-                                moduleList.append(m)
+                    if item.category.name == "Drone":
+                        d = Drone(item)
+                        d.amount = int(hardware.getAttribute("qty"))
+                        fitobj.drones.append(d)
+                    elif item.category.name == "Fighter":
+                        ft = Fighter(item)
+                        ft.amount = int(hardware.getAttribute("qty")) if ft.amount <= ft.fighterSquadronMaxSize else ft.fighterSquadronMaxSize
+                        fitobj.fighters.append(ft)
+                    elif hardware.getAttribute("slot").lower() == "cargo":
+                        # although the eve client only support charges in cargo, third-party programs
+                        # may support items or "refits" in cargo. Support these by blindly adding all
+                        # cargo, not just charges
+                        c = Cargo(item)
+                        c.amount = int(hardware.getAttribute("qty"))
+                        fitobj.cargo.append(c)
+                    else:
+                        try:
+                            m = Module(item)
+                        # When item can't be added to any slot (unknown item or just charge), ignore it
+                        except ValueError:
+                            pyfalog.warning("item can't be added to any slot (unknown item or just charge), ignore it")
+                            continue
+                        # Add subsystems before modules to make sure T3 cruisers have subsystems installed
+                        if item.category.name == "Subsystem":
+                            if m.fits(fitobj):
+                                m.owner = fitobj
+                                fitobj.modules.append(m)
+                        else:
+                            if m.isValidState(State.ACTIVE):
+                                m.state = State.ACTIVE
+
+                            moduleList.append(m)
 
                 except KeyboardInterrupt:
                     pyfalog.warning("Keyboard Interrupt")
                     continue
 
             # Recalc to get slot numbers correct for T3 cruisers
-            svcFit.getInstance().recalc(f)
+            svcFit.getInstance().recalc(fitobj)
 
             for module in moduleList:
-                if module.fits(f):
-                    module.owner = f
-                    f.modules.append(module)
+                if module.fits(fitobj):
+                    module.owner = fitobj
+                    fitobj.modules.append(module)
 
-            fits.append(f)
-            if callback:
-                wx.CallAfter(callback, None)
+            fit_list.append(fitobj)
+            if iportuser:  # NOTE: Send current processing status
+                PortProcessing.notify(
+                    iportuser, IPortUser.PROCESS_IMPORT | IPortUser.ID_UPDATE,
+                    "analysis :%s\n%s" % (fitobj.ship.name, fitobj.name)
+                )
 
-        return fits
+        return fit_list
 
     @staticmethod
     def _exportEftBase(fit):
+        """Basically EFT format does not require blank lines
+            also, it's OK to arrange modules randomly?
+        """
         offineSuffix = " /OFFLINE"
         export = "[%s, %s]\n" % (fit.ship.item.name, fit.name)
         stuff = {}
@@ -1038,10 +1240,13 @@ class Port(object):
 
         return dna + "::"
 
-    @classmethod
-    def exportXml(cls, callback=None, *fits):
+    @staticmethod
+    def exportXml(iportuser=None, *fits):
         doc = xml.dom.minidom.Document()
         fittings = doc.createElement("fittings")
+        # fit count
+        fit_count = len(fits)
+        fittings.setAttribute("count", "%s" % fit_count)
         doc.appendChild(fittings)
         sFit = svcFit.getInstance()
 
@@ -1051,10 +1256,18 @@ class Port(object):
                 fitting.setAttribute("name", fit.name)
                 fittings.appendChild(fitting)
                 description = doc.createElement("description")
-                description.setAttribute("value", "")
+                # -- 170327 Ignored description --
+                try:
+                    notes = fit.notes  # unicode
+                    description.setAttribute(
+                        "value", re.sub("(\r|\n|\r\n)+", "<br>", notes) if notes is not None else ""
+                    )
+                except Exception as e:
+                    pyfalog.warning("read description is failed, msg=%s\n" % e.args)
+
                 fitting.appendChild(description)
                 shipType = doc.createElement("shipType")
-                shipType.setAttribute("value", fit.ship.item.name)
+                shipType.setAttribute("value", fit.ship.name)
                 fitting.appendChild(shipType)
 
                 charges = {}
@@ -1113,12 +1326,17 @@ class Port(object):
                     hardware.setAttribute("slot", "cargo")
                     hardware.setAttribute("type", name)
                     fitting.appendChild(hardware)
-            except:
-                print("Failed on fitID: %d" % fit.ID)
+            except Exception as e:
+                # print("Failed on fitID: %d" % fit.ID)
+                pyfalog.error("Failed on fitID: %d, message: %s" % e.message)
                 continue
             finally:
-                if callback:
-                    wx.CallAfter(callback, i)
+                if iportuser:
+                    PortProcessing.notify(
+                        iportuser, IPortUser.PROCESS_EXPORT | IPortUser.ID_UPDATE,
+                        (i, "convert to xml (%s/%s) %s" % (i + 1, fit_count, fit.ship.name))
+                    )
+#                     wx.CallAfter(callback, i, "(%s/%s) %s" % (i, fit_count, fit.ship.name))
 
         return doc.toprettyxml()
 
@@ -1169,37 +1387,33 @@ class Port(object):
         return export
 
 
-class FitBackupThread(threading.Thread):
-    def __init__(self, path, callback):
-        threading.Thread.__init__(self)
-        self.path = path
-        self.callback = callback
-
-    def run(self):
-        path = self.path
-        sFit = svcFit.getInstance()
-        sPort = Port.getInstance()
-        backedUpFits = sPort.exportXml(self.callback, *sFit.getAllFits())
-        backupFile = open(path, "w", encoding="utf-8")
-        backupFile.write(backedUpFits)
-        backupFile.close()
-
+class PortProcessing(object):
+    """Port Processing class """
+    @staticmethod
+    def backupFits(path, iportuser):
+        success = True
+        try:
+            iportuser.on_port_process_start()
+            backedUpFits = Port.exportXml(iportuser, *svcFit.getInstance().getAllFits())
+            backupFile = open(path, "w", encoding="utf-8")
+            backupFile.write(backedUpFits)
+            backupFile.close()
+        except UserCancelException:
+            success = False
         # Send done signal to GUI
-        wx.CallAfter(self.callback, -1)
+#         wx.CallAfter(callback, -1, "Done.")
+        flag = IPortUser.ID_ERROR if not success else IPortUser.ID_DONE
+        iportuser.on_port_processing(IPortUser.PROCESS_EXPORT | flag,
+                                   "User canceled or some error occurrence." if not success else "Done.")
 
+    @staticmethod
+    def importFitsFromFile(paths, iportuser):
+        iportuser.on_port_process_start()
+        success, result = Port.importFitFromFiles(paths, iportuser)
+        flag = IPortUser.ID_ERROR if not success else IPortUser.ID_DONE
+        iportuser.on_port_processing(IPortUser.PROCESS_IMPORT | flag, result)
 
-class FitImportThread(threading.Thread):
-    def __init__(self, paths, callback):
-        threading.Thread.__init__(self)
-        self.paths = paths
-        self.callback = callback
-
-    def run(self):
-        sPort = Port.getInstance()
-        success, result = sPort.importFitFromFiles(self.paths, self.callback)
-
-        if not success:  # there was an error during processing
-            pyfalog.error("Error while processing file import: {0}", result)
-            wx.CallAfter(self.callback, -2, result)
-        else:  # Send done signal to GUI
-            wx.CallAfter(self.callback, -1, result)
+    @staticmethod
+    def notify(iportuser, flag, data):
+        if not iportuser.on_port_processing(flag, data):
+            raise UserCancelException

service/settings.py

@@ -46,36 +46,61 @@ class SettingsProvider(object):
             if not os.path.exists(self.BASE_PATH):
                 os.mkdir(self.BASE_PATH)
 
+    # def getSettings(self, area, defaults=None):
+    #     # type: (basestring, dict) -> service.Settings
+    #     # NOTE: needed to change for tests
+    #     settings_obj = self.settings.get(area)
+    #
+    #     if settings_obj is None and hasattr(self, 'BASE_PATH'):
+    #         canonical_path = os.path.join(self.BASE_PATH, area)
+    #
+    #         if not os.path.exists(canonical_path):
+    #             info = {}
+    #             if defaults:
+    #                 for item in defaults:
+    #                     info[item] = defaults[item]
+    #
+    #         else:
+    #             try:
+    #                 f = open(canonical_path, "rb")
+    #                 info = cPickle.load(f)
+    #                 for item in defaults:
+    #                     if item not in info:
+    #                         info[item] = defaults[item]
+    #
+    #             except:
+    #                 info = {}
+    #                 if defaults:
+    #                     for item in defaults:
+    #                         info[item] = defaults[item]
+    #
+    #         self.settings[area] = settings_obj = Settings(canonical_path, info)
+    #
+    #     return settings_obj
     def getSettings(self, area, defaults=None):
-
-        s = self.settings.get(area)
-
-        if s is None and hasattr(self, 'BASE_PATH'):
-            p = os.path.join(self.BASE_PATH, area)
-
-            if not os.path.exists(p):
+        # type: (basestring, dict) -> service.Settings
+        # NOTE: needed to change for tests
+        # TODO: Write to memory with mmap -> https://docs.python.org/2/library/mmap.html
+        settings_obj = self.settings.get(area)
+        if settings_obj is None:  # and hasattr(self, 'BASE_PATH'):
+            canonical_path = os.path.join(self.BASE_PATH, area) if hasattr(self, 'BASE_PATH') else ""
+            if not os.path.exists(canonical_path):  # path string or empty string.
                 info = {}
                 if defaults:
-                    for item in defaults:
-                        info[item] = defaults[item]
-
+                    info.update(defaults)
             else:
                 try:
-                    f = open(p, "rb")
-                    info = cPickle.load(f)
+                    with open(canonical_path, "rb") as f:
+                        info = cPickle.load(f)
                     for item in defaults:
                         if item not in info:
                             info[item] = defaults[item]
-
                 except:
                     info = {}
-                    if defaults:
-                        for item in defaults:
-                            info[item] = defaults[item]
+                    info.update(defaults)
 
-            self.settings[area] = s = Settings(p, info)
-
-        return s
+            self.settings[area] = settings_obj = Settings(canonical_path, info)
+        return settings_obj
 
     def saveAll(self):
         for settings in self.settings.itervalues():
@@ -84,12 +109,22 @@ class SettingsProvider(object):
 
 class Settings(object):
     def __init__(self, location, info):
+        # type: (basestring, dict) -> None
+        # path string or empty string.
         self.location = location
         self.info = info
 
+    # def save(self):
+    #     f = open(self.location, "wb")
+    #     cPickle.dump(self.info, f, cPickle.HIGHEST_PROTOCOL)
+
     def save(self):
-        f = open(self.location, "wb")
-        cPickle.dump(self.info, f, cPickle.HIGHEST_PROTOCOL)
+        # NOTE: needed to change for tests
+        if self.location is None or not self.location:
+            return
+        # NOTE: with + open -> file handle auto close
+        with open(self.location, "wb") as f:
+            cPickle.dump(self.info, f, cPickle.HIGHEST_PROTOCOL)
 
     def __getitem__(self, k):
         try:

tests/test_unread_desc.py

@@ -0,0 +1,88 @@
+"""
+    2017/04/05: unread description tests module.
+"""
+# noinspection PyPackageRequirements
+import pytest
+# Add root folder to python paths
+# This must be done on every test in order to pass in Travis
+import os
+import sys
+# nopep8
+import re
+# from utils.strfunctions import sequential_rep, replace_ltgt
+from utils.stopwatch import Stopwatch
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.realpath(os.path.join(script_dir, '..')))
+sys._called_from_test = True  # need db open for tests. (see eos/config.py#17
+# noinspection PyPep8
+from service.port import Port, IPortUser
+#
+# noinspection PyPackageRequirements
+# from _development.helpers import DBInMemory as DB
+
+"""
+NOTE:
+  description character length is restricted 4hundred by EVE client.
+  these things apply to multi byte environment too.
+
+
+    o read xml fit data (and encode to utf-8 if need.
+
+    o construct xml dom object, and extract "fitting" elements.
+
+    o apply _resolve_ship method to each "fitting" elements. (time measurement
+
+        o extract "hardware" elements from "fitting" element.
+
+        o apply _resolve_module method to each "hardware" elements. (time measurement
+
+xml files:
+    "jeffy_ja-en[99].xml"
+
+NOTE of @decorator:
+    o Function to receive arguments of function to be decorated
+    o A function that accepts the decorate target function itself as an argument
+    o A function that accepts arguments of the decorator itself
+
+for local coverage:
+    py.test --cov=./ --cov-report=html
+"""
+
+class PortUser(IPortUser):
+
+    def on_port_processing(self, action, data=None):
+        print(data)
+        return True
+
+
+stpw = Stopwatch('test measurementer')
+
+@pytest.fixture()
+def print_db_info():
+    # Output debug info
+    import eos
+    print
+    print "------------ data base connection info ------------"
+    print(eos.db.saveddata_engine)
+    print(eos.db.gamedata_engine)
+    print
+
+
+# noinspection PyUnusedLocal
+def test_import_xml(print_db_info):
+    usr = PortUser()
+# for path in XML_FILES:
+    xml_file = "jeffy_ja-en[99].xml"
+    fit_count = int(re.search(r"\[(\d+)\]", xml_file).group(1))
+    fits = None
+    with open(os.path.join(script_dir, xml_file), "r") as file_:
+        srcString = file_.read()
+        srcString = unicode(srcString, "utf-8")
+        #  (basestring, IPortUser, basestring) -> list[eos.saveddata.fit.Fit]
+        usr.on_port_process_start()
+        stpw.reset()
+        with stpw:
+            fits = Port.importXml(srcString, usr)
+
+        assert fits is not None and len(fits) is fit_count

utils/stopwatch.py

@@ -0,0 +1,122 @@
+# coding: utf-8
+
+import time
+import os
+
+
+class Stopwatch(object):
+    """
+ --- on python console ---
+import re
+from utils.stopwatch import Stopwatch
+
+# measurementor
+stpw = Stopwatch("test")
+# measurement re.sub
+def m_re_sub(t, set_count, executes, texts):
+    t.reset()
+    while set_count:
+        set_count -= 1
+        with t:
+            while executes:
+                executes -= 1
+                ret = re.sub("[a|s]+", "-", texts)
+    # stat string
+    return str(t)
+
+# statistics loop: 1000(exec re.sub: 100000)
+m_re_sub(stpw, 1000, 100000, "asdfadsasdaasdfadsasda")
+
+----------- records -----------
+ text: "asdfadsasda"
+    'elapsed record(ms): min=0.000602411446948, max=220.85578571'
+    'elapsed record(ms): min=0.000602411446948, max=217.331377504'
+
+ text: "asdfadsasdaasdfadsasda"
+    'elapsed record(ms): min=0.000602411446948, max=287.784902967'
+    'elapsed record(ms): min=0.000602411432737, max=283.653264016'
+
+    NOTE: about max
+      The value is large only at the first execution,
+      Will it be optimized, after that it will be significantly smaller
+    """
+
+    # time.clock() is μs? 1/1000ms
+    # https://docs.python.jp/2.7/library/time.html#time.clock
+    _tfunc = time.clock if os.name == "nt" else time.time
+
+    def __init__(self, name='', logger=None):
+        self.name = name
+        self.start = Stopwatch._tfunc()
+        self.__last = self.start
+        # __last field is means last checkpoint system clock value?
+        self.logger = logger
+        self.min = 0.0
+        self.max = 0.0
+        self.__first = True
+
+    @property
+    def stat(self):
+        # :return: (float, float)
+        return self.min, self.max
+
+    @property
+    def elapsed(self):
+        # :return: time as ms
+        return (Stopwatch._tfunc() - self.start) * 1000
+
+    @property
+    def last(self):
+        return self.__last * 1000
+
+    def __update_stat(self, v):
+        # :param v: float unit of ms
+        if self.__first:
+            self.__first = False
+            return
+        if self.min == 0.0 or self.min > v:
+            self.min = v
+        if self.max < v:
+            self.max = v
+
+    def checkpoint(self, name=''):
+        span = self.elapsed
+        self.__update_stat(span)
+        text = u'Stopwatch("{tname}") - {checkpoint} - {last:.6f}ms ({elapsed:.12f}ms elapsed)'.format(
+                tname=self.name,
+                checkpoint=unicode(name, "utf-8"),
+                last=self.last,
+                elapsed=span
+        ).strip()
+        self.__last = Stopwatch._tfunc()
+        if self.logger:
+            self.logger.debug(text)
+        else:
+            print(text)
+
+    @staticmethod
+    def CpuClock():
+        start = Stopwatch._tfunc()
+        time.sleep(1)
+        return Stopwatch._tfunc() - start
+
+    def reset(self):
+        # clear stat
+        self.min = 0.0
+        self.max = 0.0
+        self.__first = True
+
+    def __enter__(self):
+        self.start = Stopwatch._tfunc()
+        return self
+
+    def __exit__(self, type_, value, traceback):
+        # https://docs.python.org/2.7/reference/datamodel.html?highlight=__enter__#object.__exit__
+        # If the context was exited without an exception, all three arguments will be None
+        self.checkpoint('finished')
+        # ex: "type=None, value=None, traceback=None"
+        # print "type=%s, value=%s, traceback=%s" % (type, value, traceback)
+        return True
+
+    def __repr__(self):
+        return "elapsed record(ms): min=%s, max=%s" % self.stat

utils/strfunctions.py

@@ -0,0 +1,30 @@
+'''
+ string manipulation module
+'''
+import re
+
+
+def sequential_rep(text_, *args):
+    # type: (basestring, tuple) -> basestring
+    """
+    :param text_: string content
+    :param args: like <pattern>, <replacement>, <pattern>, <replacement>, ...
+    :return: if text_ length was zero or invalid parameters then no manipulation to text_
+    """
+    arg_len = len(args)
+    if arg_len % 2 == 0 and isinstance(text_, basestring) and len(text_) > 0:
+        i = 0
+        while i < arg_len:
+            text_ = re.sub(args[i], args[i + 1], text_)
+            i += 2
+
+    return text_
+
+
+def replace_ltgt(text_):
+    # type: (basestring) -> basestring
+    """if fit name contained "<" or ">" then reprace to named html entity by EVE client.
+    :param text_: string content of fit name from exported by EVE client.
+    :return: if text_ is not instance of basestring then no manipulation to text_.
+    """
+    return text_.replace("&lt;", "<").replace("&gt;", ">") if isinstance(text_, basestring) else text_