Split XML facilities into their own file as well

This commit is contained in:
DarkPhoenix
2018-08-30 16:15:23 +03:00
parent dd0fbfddb9
commit 4e5a70993e
2 changed files with 342 additions and 314 deletions

View File

@@ -19,135 +19,29 @@
import re
import os
import xml.dom
from logbook import Logger
import collections
import json
import threading
from bs4 import UnicodeDammit
import xml.dom
import xml.parsers.expat
from codecs import open
import xml.parsers.expat
from bs4 import UnicodeDammit
from logbook import Logger
from eos import db
from eos.saveddata.fit import ImplantLocation
from service.fit import Fit as svcFit
# noinspection PyPackageRequirements
import wx
from eos.saveddata.cargo import Cargo
from eos.saveddata.drone import Drone
from eos.saveddata.fighter import Fighter
from eos.saveddata.module import Module, State, Slot
from eos.saveddata.ship import Ship
from eos.saveddata.citadel import Citadel
from eos.saveddata.fit import Fit, ImplantLocation
from service.market import Market
from utils.strfunctions import sequential_rep, replace_ltgt
from service.port.dna import exportDna, importDna
from service.port.eft import EftPort, SLOT_ORDER as EFT_SLOT_ORDER
from service.port.esi import importESI, exportESI
from service.port.shared import IPortUser, UserCancelException, processing_notify
from service.port.xml import importXml, exportXml
pyfalog = Logger(__name__)
# 2017/04/05 NOTE: simple validation, for xml file
RE_XML_START = r'<\?xml\s+version="1.0"\s*\?>'
# -- 170327 Ignored description --
RE_LTGT = "&(lt|gt);"
L_MARK = "&lt;localized hint=&quot;"
# &lt;localized hint=&quot;([^"]+)&quot;&gt;([^\*]+)\*&lt;\/localized&gt;
LOCALIZED_PATTERN = re.compile(r'<localized hint="([^"]+)">([^\*]+)\*</localized>')
def _extract_match(t):
m = LOCALIZED_PATTERN.match(t)
# hint attribute, text content
return m.group(1), m.group(2)
def _resolve_ship(fitting, sMkt, b_localized):
# type: (xml.dom.minidom.Element, service.market.Market, bool) -> eos.saveddata.fit.Fit
""" NOTE: Since it is meaningless unless a correct ship object can be constructed,
process flow changed
"""
# ------ Confirm ship
# <localized hint="Maelstrom">Maelstrom</localized>
shipType = fitting.getElementsByTagName("shipType").item(0).getAttribute("value")
anything = None
if b_localized:
# expect an official name, emergency cache
shipType, anything = _extract_match(shipType)
limit = 2
ship = None
while True:
must_retry = False
try:
try:
ship = Ship(sMkt.getItem(shipType))
except ValueError:
ship = Citadel(sMkt.getItem(shipType))
except Exception as e:
pyfalog.warning("Caught exception on _resolve_ship")
pyfalog.error(e)
limit -= 1
if limit is 0:
break
shipType = anything
must_retry = True
if not must_retry:
break
if ship is None:
raise Exception("cannot resolve ship type.")
fitobj = Fit(ship=ship)
# ------ Confirm fit name
anything = fitting.getAttribute("name")
# 2017/03/29 NOTE:
# if fit name contained "<" or ">" then reprace to named html entity by EVE client
# if re.search(RE_LTGT, anything):
if "&lt;" in anything or "&gt;" in anything:
anything = replace_ltgt(anything)
fitobj.name = anything
return fitobj
def _resolve_module(hardware, sMkt, b_localized):
# type: (xml.dom.minidom.Element, service.market.Market, bool) -> eos.saveddata.module.Module
moduleName = hardware.getAttribute("type")
emergency = None
if b_localized:
# expect an official name, emergency cache
moduleName, emergency = _extract_match(moduleName)
item = None
limit = 2
while True:
must_retry = False
try:
item = sMkt.getItem(moduleName, eager="group.category")
except Exception as e:
pyfalog.warning("Caught exception on _resolve_module")
pyfalog.error(e)
limit -= 1
if limit is 0:
break
moduleName = emergency
must_retry = True
if not must_retry:
break
return item
class Port(object):
"""Service which houses all import/export format functions"""
@@ -288,7 +182,7 @@ class Port(object):
@staticmethod
def importFitFromBuffer(bufferStr, activeFit=None):
# type: (basestring, object) -> object
# type: (str, object) -> object
# TODO: catch the exception?
# activeFit is reserved?, bufferStr is unicode? (assume only clipboard string?
sFit = svcFit.getInstance()
@@ -307,7 +201,7 @@ class Port(object):
@classmethod
def importAuto(cls, string, path=None, activeFit=None, iportuser=None):
# type: (basestring, basestring, object, IPortUser, basestring) -> object
# type: (Port, str, str, object, IPortUser) -> object
# Get first line and strip space symbols of it to avoid possible detection errors
firstLine = re.split("[\n\r]+", string.strip(), maxsplit=1)[0]
firstLine = firstLine.strip()
@@ -335,206 +229,6 @@ class Port(object):
# Use DNA format for all other cases
return "DNA", (cls.importDna(string),)
@staticmethod
def importXml(text, iportuser=None):
# type: (str, IPortUser) -> list[eos.saveddata.fit.Fit]
sMkt = Market.getInstance()
doc = xml.dom.minidom.parseString(text)
# NOTE:
# When L_MARK is included at this point,
# Decided to be localized data
b_localized = L_MARK in text
fittings = doc.getElementsByTagName("fittings").item(0)
fittings = fittings.getElementsByTagName("fitting")
fit_list = []
failed = 0
for fitting in fittings:
try:
fitobj = _resolve_ship(fitting, sMkt, b_localized)
except:
failed += 1
continue
# -- 170327 Ignored description --
# read description from exported xml. (EVE client, EFT)
description = fitting.getElementsByTagName("description").item(0).getAttribute("value")
if description is None:
description = ""
elif len(description):
# convert <br> to "\n" and remove html tags.
if Port.is_tag_replace():
description = replace_ltgt(
sequential_rep(description, r"<(br|BR)>", "\n", r"<[^<>]+>", "")
)
fitobj.notes = description
hardwares = fitting.getElementsByTagName("hardware")
moduleList = []
for hardware in hardwares:
try:
item = _resolve_module(hardware, sMkt, b_localized)
if not item or not item.published:
continue
if item.category.name == "Drone":
d = Drone(item)
d.amount = int(hardware.getAttribute("qty"))
fitobj.drones.append(d)
elif item.category.name == "Fighter":
ft = Fighter(item)
ft.amount = int(hardware.getAttribute("qty")) if ft.amount <= ft.fighterSquadronMaxSize else ft.fighterSquadronMaxSize
fitobj.fighters.append(ft)
elif hardware.getAttribute("slot").lower() == "cargo":
# although the eve client only support charges in cargo, third-party programs
# may support items or "refits" in cargo. Support these by blindly adding all
# cargo, not just charges
c = Cargo(item)
c.amount = int(hardware.getAttribute("qty"))
fitobj.cargo.append(c)
else:
try:
m = Module(item)
# When item can't be added to any slot (unknown item or just charge), ignore it
except ValueError:
pyfalog.warning("item can't be added to any slot (unknown item or just charge), ignore it")
continue
# Add subsystems before modules to make sure T3 cruisers have subsystems installed
if item.category.name == "Subsystem":
if m.fits(fitobj):
m.owner = fitobj
fitobj.modules.append(m)
else:
if m.isValidState(State.ACTIVE):
m.state = State.ACTIVE
moduleList.append(m)
except KeyboardInterrupt:
pyfalog.warning("Keyboard Interrupt")
continue
# Recalc to get slot numbers correct for T3 cruisers
svcFit.getInstance().recalc(fitobj)
for module in moduleList:
if module.fits(fitobj):
module.owner = fitobj
fitobj.modules.append(module)
fit_list.append(fitobj)
if iportuser: # NOTE: Send current processing status
processing_notify(
iportuser, IPortUser.PROCESS_IMPORT | IPortUser.ID_UPDATE,
"Processing %s\n%s" % (fitobj.ship.name, fitobj.name)
)
return fit_list
@staticmethod
def exportXml(iportuser=None, *fits):
doc = xml.dom.minidom.Document()
fittings = doc.createElement("fittings")
# fit count
fit_count = len(fits)
fittings.setAttribute("count", "%s" % fit_count)
doc.appendChild(fittings)
sFit = svcFit.getInstance()
for i, fit in enumerate(fits):
try:
fitting = doc.createElement("fitting")
fitting.setAttribute("name", fit.name)
fittings.appendChild(fitting)
description = doc.createElement("description")
# -- 170327 Ignored description --
try:
notes = fit.notes # unicode
if notes:
notes = notes[:397] + '...' if len(notes) > 400 else notes
description.setAttribute(
"value", re.sub("(\r|\n|\r\n)+", "<br>", notes) if notes is not None else ""
)
except Exception as e:
pyfalog.warning("read description is failed, msg=%s\n" % e.args)
fitting.appendChild(description)
shipType = doc.createElement("shipType")
shipType.setAttribute("value", fit.ship.name)
fitting.appendChild(shipType)
charges = {}
slotNum = {}
for module in fit.modules:
if module.isEmpty:
continue
slot = module.slot
if slot == Slot.SUBSYSTEM:
# Order of subsystem matters based on this attr. See GH issue #130
slotId = module.getModifiedItemAttr("subSystemSlot") - 125
else:
if slot not in slotNum:
slotNum[slot] = 0
slotId = slotNum[slot]
slotNum[slot] += 1
hardware = doc.createElement("hardware")
hardware.setAttribute("type", module.item.name)
slotName = Slot.getName(slot).lower()
slotName = slotName if slotName != "high" else "hi"
hardware.setAttribute("slot", "%s slot %d" % (slotName, slotId))
fitting.appendChild(hardware)
if module.charge and sFit.serviceFittingOptions["exportCharges"]:
if module.charge.name not in charges:
charges[module.charge.name] = 0
# `or 1` because some charges (ie scripts) are without qty
charges[module.charge.name] += module.numCharges or 1
for drone in fit.drones:
hardware = doc.createElement("hardware")
hardware.setAttribute("qty", "%d" % drone.amount)
hardware.setAttribute("slot", "drone bay")
hardware.setAttribute("type", drone.item.name)
fitting.appendChild(hardware)
for fighter in fit.fighters:
hardware = doc.createElement("hardware")
hardware.setAttribute("qty", "%d" % fighter.amountActive)
hardware.setAttribute("slot", "fighter bay")
hardware.setAttribute("type", fighter.item.name)
fitting.appendChild(hardware)
for cargo in fit.cargo:
if cargo.item.name not in charges:
charges[cargo.item.name] = 0
charges[cargo.item.name] += cargo.amount
for name, qty in list(charges.items()):
hardware = doc.createElement("hardware")
hardware.setAttribute("qty", "%d" % qty)
hardware.setAttribute("slot", "cargo")
hardware.setAttribute("type", name)
fitting.appendChild(hardware)
except Exception as e:
# print("Failed on fitID: %d" % fit.ID)
pyfalog.error("Failed on fitID: %d, message: %s" % e.message)
continue
finally:
if iportuser:
processing_notify(
iportuser, IPortUser.PROCESS_EXPORT | IPortUser.ID_UPDATE,
(i, "convert to xml (%s/%s) %s" % (i + 1, fit_count, fit.ship.name))
)
# wx.CallAfter(callback, i, "(%s/%s) %s" % (i, fit_count, fit.ship.name))
return doc.toprettyxml()
@staticmethod
def exportMultiBuy(fit):
export = "%s\n" % fit.ship.item.name
@@ -611,3 +305,12 @@ class Port(object):
@staticmethod
def exportESI(fit):
exportESI(fit)
# XML-related methods
@staticmethod
def importXml(text, iportuser=None):
return importXml(text, iportuser)
@staticmethod
def exportXml(iportuser=None, *fits):
exportXml(iportuser, *fits)

325
service/port/xml.py Normal file
View File

@@ -0,0 +1,325 @@
# =============================================================================
# Copyright (C) 2014 Ryan Holmes
#
# This file is part of pyfa.
#
# pyfa is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# pyfa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with pyfa. If not, see <http://www.gnu.org/licenses/>.
# =============================================================================
import re
import xml.dom
import xml.parsers.expat
from logbook import Logger
from eos.saveddata.cargo import Cargo
from eos.saveddata.citadel import Citadel
from eos.saveddata.drone import Drone
from eos.saveddata.fighter import Fighter
from eos.saveddata.fit import Fit
from eos.saveddata.module import Module, State, Slot
from eos.saveddata.ship import Ship
from service.fit import Fit as svcFit
from service.market import Market
from utils.strfunctions import sequential_rep, replace_ltgt
from service.port.shared import IPortUser, processing_notify
pyfalog = Logger(__name__)
# -- 170327 Ignored description --
RE_LTGT = "&(lt|gt);"
L_MARK = "&lt;localized hint=&quot;"
# &lt;localized hint=&quot;([^"]+)&quot;&gt;([^\*]+)\*&lt;\/localized&gt;
LOCALIZED_PATTERN = re.compile(r'<localized hint="([^"]+)">([^\*]+)\*</localized>')
def _extract_match(t):
m = LOCALIZED_PATTERN.match(t)
# hint attribute, text content
return m.group(1), m.group(2)
def _resolve_ship(fitting, sMkt, b_localized):
# type: (xml.dom.minidom.Element, service.market.Market, bool) -> eos.saveddata.fit.Fit
""" NOTE: Since it is meaningless unless a correct ship object can be constructed,
process flow changed
"""
# ------ Confirm ship
# <localized hint="Maelstrom">Maelstrom</localized>
shipType = fitting.getElementsByTagName("shipType").item(0).getAttribute("value")
anything = None
if b_localized:
# expect an official name, emergency cache
shipType, anything = _extract_match(shipType)
limit = 2
ship = None
while True:
must_retry = False
try:
try:
ship = Ship(sMkt.getItem(shipType))
except ValueError:
ship = Citadel(sMkt.getItem(shipType))
except Exception as e:
pyfalog.warning("Caught exception on _resolve_ship")
pyfalog.error(e)
limit -= 1
if limit is 0:
break
shipType = anything
must_retry = True
if not must_retry:
break
if ship is None:
raise Exception("cannot resolve ship type.")
fitobj = Fit(ship=ship)
# ------ Confirm fit name
anything = fitting.getAttribute("name")
# 2017/03/29 NOTE:
# if fit name contained "<" or ">" then reprace to named html entity by EVE client
# if re.search(RE_LTGT, anything):
if "&lt;" in anything or "&gt;" in anything:
anything = replace_ltgt(anything)
fitobj.name = anything
return fitobj
def _resolve_module(hardware, sMkt, b_localized):
# type: (xml.dom.minidom.Element, service.market.Market, bool) -> eos.saveddata.module.Module
moduleName = hardware.getAttribute("type")
emergency = None
if b_localized:
# expect an official name, emergency cache
moduleName, emergency = _extract_match(moduleName)
item = None
limit = 2
while True:
must_retry = False
try:
item = sMkt.getItem(moduleName, eager="group.category")
except Exception as e:
pyfalog.warning("Caught exception on _resolve_module")
pyfalog.error(e)
limit -= 1
if limit is 0:
break
moduleName = emergency
must_retry = True
if not must_retry:
break
return item
def importXml(text, iportuser):
# type: (str, IPortUser) -> list[eos.saveddata.fit.Fit]
sMkt = Market.getInstance()
doc = xml.dom.minidom.parseString(text)
# NOTE:
# When L_MARK is included at this point,
# Decided to be localized data
b_localized = L_MARK in text
fittings = doc.getElementsByTagName("fittings").item(0)
fittings = fittings.getElementsByTagName("fitting")
fit_list = []
failed = 0
for fitting in fittings:
try:
fitobj = _resolve_ship(fitting, sMkt, b_localized)
except:
failed += 1
continue
# -- 170327 Ignored description --
# read description from exported xml. (EVE client, EFT)
description = fitting.getElementsByTagName("description").item(0).getAttribute("value")
if description is None:
description = ""
elif len(description):
# convert <br> to "\n" and remove html tags.
if Port.is_tag_replace():
description = replace_ltgt(
sequential_rep(description, r"<(br|BR)>", "\n", r"<[^<>]+>", "")
)
fitobj.notes = description
hardwares = fitting.getElementsByTagName("hardware")
moduleList = []
for hardware in hardwares:
try:
item = _resolve_module(hardware, sMkt, b_localized)
if not item or not item.published:
continue
if item.category.name == "Drone":
d = Drone(item)
d.amount = int(hardware.getAttribute("qty"))
fitobj.drones.append(d)
elif item.category.name == "Fighter":
ft = Fighter(item)
ft.amount = int(hardware.getAttribute("qty")) if ft.amount <= ft.fighterSquadronMaxSize else ft.fighterSquadronMaxSize
fitobj.fighters.append(ft)
elif hardware.getAttribute("slot").lower() == "cargo":
# although the eve client only support charges in cargo, third-party programs
# may support items or "refits" in cargo. Support these by blindly adding all
# cargo, not just charges
c = Cargo(item)
c.amount = int(hardware.getAttribute("qty"))
fitobj.cargo.append(c)
else:
try:
m = Module(item)
# When item can't be added to any slot (unknown item or just charge), ignore it
except ValueError:
pyfalog.warning("item can't be added to any slot (unknown item or just charge), ignore it")
continue
# Add subsystems before modules to make sure T3 cruisers have subsystems installed
if item.category.name == "Subsystem":
if m.fits(fitobj):
m.owner = fitobj
fitobj.modules.append(m)
else:
if m.isValidState(State.ACTIVE):
m.state = State.ACTIVE
moduleList.append(m)
except KeyboardInterrupt:
pyfalog.warning("Keyboard Interrupt")
continue
# Recalc to get slot numbers correct for T3 cruisers
svcFit.getInstance().recalc(fitobj)
for module in moduleList:
if module.fits(fitobj):
module.owner = fitobj
fitobj.modules.append(module)
fit_list.append(fitobj)
if iportuser: # NOTE: Send current processing status
processing_notify(
iportuser, IPortUser.PROCESS_IMPORT | IPortUser.ID_UPDATE,
"Processing %s\n%s" % (fitobj.ship.name, fitobj.name)
)
return fit_list
def exportXml(iportuser, *fits):
doc = xml.dom.minidom.Document()
fittings = doc.createElement("fittings")
# fit count
fit_count = len(fits)
fittings.setAttribute("count", "%s" % fit_count)
doc.appendChild(fittings)
sFit = svcFit.getInstance()
for i, fit in enumerate(fits):
try:
fitting = doc.createElement("fitting")
fitting.setAttribute("name", fit.name)
fittings.appendChild(fitting)
description = doc.createElement("description")
# -- 170327 Ignored description --
try:
notes = fit.notes # unicode
if notes:
notes = notes[:397] + '...' if len(notes) > 400 else notes
description.setAttribute(
"value", re.sub("(\r|\n|\r\n)+", "<br>", notes) if notes is not None else ""
)
except Exception as e:
pyfalog.warning("read description is failed, msg=%s\n" % e.args)
fitting.appendChild(description)
shipType = doc.createElement("shipType")
shipType.setAttribute("value", fit.ship.name)
fitting.appendChild(shipType)
charges = {}
slotNum = {}
for module in fit.modules:
if module.isEmpty:
continue
slot = module.slot
if slot == Slot.SUBSYSTEM:
# Order of subsystem matters based on this attr. See GH issue #130
slotId = module.getModifiedItemAttr("subSystemSlot") - 125
else:
if slot not in slotNum:
slotNum[slot] = 0
slotId = slotNum[slot]
slotNum[slot] += 1
hardware = doc.createElement("hardware")
hardware.setAttribute("type", module.item.name)
slotName = Slot.getName(slot).lower()
slotName = slotName if slotName != "high" else "hi"
hardware.setAttribute("slot", "%s slot %d" % (slotName, slotId))
fitting.appendChild(hardware)
if module.charge and sFit.serviceFittingOptions["exportCharges"]:
if module.charge.name not in charges:
charges[module.charge.name] = 0
# `or 1` because some charges (ie scripts) are without qty
charges[module.charge.name] += module.numCharges or 1
for drone in fit.drones:
hardware = doc.createElement("hardware")
hardware.setAttribute("qty", "%d" % drone.amount)
hardware.setAttribute("slot", "drone bay")
hardware.setAttribute("type", drone.item.name)
fitting.appendChild(hardware)
for fighter in fit.fighters:
hardware = doc.createElement("hardware")
hardware.setAttribute("qty", "%d" % fighter.amountActive)
hardware.setAttribute("slot", "fighter bay")
hardware.setAttribute("type", fighter.item.name)
fitting.appendChild(hardware)
for cargo in fit.cargo:
if cargo.item.name not in charges:
charges[cargo.item.name] = 0
charges[cargo.item.name] += cargo.amount
for name, qty in list(charges.items()):
hardware = doc.createElement("hardware")
hardware.setAttribute("qty", "%d" % qty)
hardware.setAttribute("slot", "cargo")
hardware.setAttribute("type", name)
fitting.appendChild(hardware)
except Exception as e:
pyfalog.error("Failed on fitID: %d, message: %s" % e.message)
continue
finally:
if iportuser:
processing_notify(
iportuser, IPortUser.PROCESS_EXPORT | IPortUser.ID_UPDATE,
(i, "convert to xml (%s/%s) %s" % (i + 1, fit_count, fit.ship.name))
)
return doc.toprettyxml()