From 943ee517f4380a8df44079a15aeda8e2d7a1a4be Mon Sep 17 00:00:00 2001 From: Filip Sufitchi Date: Mon, 2 Apr 2018 16:40:39 -0400 Subject: [PATCH 1/5] Add jargon service Jargon service translates search strings for the market using community-sourced abbreviations. It also creates the jargon.yaml file in the user's save directory, for future customization (either manual, or via the UI) and provides the code hooks to load and update it. The JargonLoader is capable of hot-loading new jargon.yaml configurations without restarting Pyfa. --- requirements.txt | 3 +- service/jargon/__init__.py | 21 ++++++++++ service/jargon/defaults.yaml | 3 ++ service/jargon/header.yaml | 14 +++++++ service/jargon/jargon.py | 47 +++++++++++++++++++++ service/jargon/loader.py | 81 ++++++++++++++++++++++++++++++++++++ service/jargon/resources.py | 23 ++++++++++ service/market.py | 19 +++++++-- 8 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 service/jargon/__init__.py create mode 100644 service/jargon/defaults.yaml create mode 100644 service/jargon/header.yaml create mode 100644 service/jargon/jargon.py create mode 100644 service/jargon/loader.py create mode 100644 service/jargon/resources.py diff --git a/requirements.txt b/requirements.txt index 2b38e8685..7a2e5f65f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ sqlalchemy >= 1.0.5 markdown2 packaging roman -beautifulsoup4 \ No newline at end of file +beautifulsoup4 +PyYAML diff --git a/service/jargon/__init__.py b/service/jargon/__init__.py new file mode 100644 index 000000000..447c917e3 --- /dev/null +++ b/service/jargon/__init__.py @@ -0,0 +1,21 @@ +# ============================================================================= +# Copyright (C) 2018 Filip Sufitchi +# +# This file is part of pyfa. +# +# pyfa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyfa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyfa. If not, see . +# ============================================================================= + +from .jargon import Jargon +from .loader import JargonLoader diff --git a/service/jargon/defaults.yaml b/service/jargon/defaults.yaml new file mode 100644 index 000000000..4fc978a1e --- /dev/null +++ b/service/jargon/defaults.yaml @@ -0,0 +1,3 @@ +2: II +haml: Heavy Assault Missile Launcher +mwd: Microwarpdrive diff --git a/service/jargon/header.yaml b/service/jargon/header.yaml new file mode 100644 index 000000000..031effff6 --- /dev/null +++ b/service/jargon/header.yaml @@ -0,0 +1,14 @@ +# This is the default Pyfa jargon file. +# +# It is essentially a giant set of find/replace statements in order to translate +# abbreviated Eve community terms into more useful full terms. It is intended +# for translation of strings such as "haml 2" "into "Heavy Assault Missile Launcher II".. +# +# These abbreviations are not case-sensitive. If abbreviations collide, the +# later one is used. +# +# Abbreviations with spaces are not supported. +# +# Syntax: +# +# abbreviation: full name diff --git a/service/jargon/jargon.py b/service/jargon/jargon.py new file mode 100644 index 000000000..0a5065faf --- /dev/null +++ b/service/jargon/jargon.py @@ -0,0 +1,47 @@ +# ============================================================================= +# Copyright (C) 2018 Filip Sufitchi +# +# This file is part of pyfa. +# +# pyfa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyfa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyfa. If not, see . +# ============================================================================= + +import config +import pkg_resources + +class Jargon(object): + def __init__(self, rawdata: dict): + self._rawdata = rawdata + + # copy the data to lowercase keys, ignore blank keys + self._data = {str(k).lower():v for k,v in rawdata.items() if k} + + def get(self, term: str) -> str: + return self._data.get(term.lower()) + + def get_rawdata() -> dict: + return self._rawdata + + def apply(self, query): + query_words = query.split() + parts = [] + + for word in query_words: + replacement = self.get(word) + if replacement: + parts.append(replacement) + else: + parts.append(word) + + return ' '.join(parts) diff --git a/service/jargon/loader.py b/service/jargon/loader.py new file mode 100644 index 000000000..11e95b7c4 --- /dev/null +++ b/service/jargon/loader.py @@ -0,0 +1,81 @@ +# ============================================================================= +# Copyright (C) 2018 Filip Sufitchi +# +# This file is part of pyfa. +# +# pyfa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyfa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyfa. If not, see . +# ============================================================================= + +import os +import config +import yaml + +from .jargon import Jargon +from .resources import DEFAULT_DATA, DEFAULT_HEADER + +JARGON_PATH = os.path.join(config.savePath, 'jargon.yaml') + +class JargonLoader(object): + def __init__(self, jargon_path: str): + self.jargon_path = jargon_path + self._jargon_mtime = 0 # type: int + self._jargon = None # type: Jargon + + def save_jargon(self, data: Jargon): + rawdata = data.get_rawdata() + with open(JARGON_PATH, 'w') as f: + yaml.dump(rawdata, stream=f, default_flow_style=False) + + def get_jargon(self) -> Jargon: + if self._is_stale(): + self._load_jargon() + return self._jargon + + def _is_stale(self): + return (not self._jargon or not self._jargon_mtime or + self.jargon_mtime != self._get_jargon_file_mtime()) + + def _load_jargon(self): + with open(JARGON_PATH) as f: + rawdata = yaml.load(f) + self.jargon_mtime = self._get_jargon_file_mtime() + self._jargon = Jargon(rawdata) + + def _get_jargon_file_mtime(self) -> int: + if not os.path.exists(self.jargon_path): + return 0 + return os.stat(self.jargon_path).st_mtime + + @staticmethod + def init_user_jargon(jargon_path): + values = yaml.load(DEFAULT_DATA) + if os.path.exists(jargon_path): + with open(jargon_path) as f: + custom_values = yaml.load(f) + if custom_values: + values.update(custom_values) + with open(jargon_path, 'w') as f: + f.write(DEFAULT_HEADER) + f.write('\n\n') + yaml.dump(values, stream=f, default_flow_style=False) + + _instance = None + @staticmethod + def instance(jargon_path=None): + if not JargonLoader._instance: + jargon_path = jargon_path or JARGON_PATH + JargonLoader._instance = JargonLoader(jargon_path) + return JargonLoader._instance + +JargonLoader.init_user_jargon(JARGON_PATH) diff --git a/service/jargon/resources.py b/service/jargon/resources.py new file mode 100644 index 000000000..f6c631b0e --- /dev/null +++ b/service/jargon/resources.py @@ -0,0 +1,23 @@ +# ============================================================================= +# Copyright (C) 2018 Filip Sufitchi +# +# This file is part of pyfa. +# +# pyfa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyfa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyfa. If not, see . +# ============================================================================= + +import pkg_resources + +DEFAULT_DATA = pkg_resources.resource_string(__name__, 'defaults.yaml').decode() +DEFAULT_HEADER = pkg_resources.resource_string(__name__, 'header.yaml').decode() diff --git a/service/market.py b/service/market.py index 06fd18465..b20538732 100644 --- a/service/market.py +++ b/service/market.py @@ -30,6 +30,7 @@ import config import eos.db from service import conversions from service.settings import SettingsProvider +from service.jargon import JargonLoader from eos.gamedata import Category as types_Category, Group as types_Group, Item as types_Item, MarketGroup as types_MarketGroup, \ MetaGroup as types_MetaGroup, MetaType as types_MetaType @@ -40,7 +41,6 @@ pyfalog = Logger(__name__) # Event which tells threads dependent on Market that it's initialized mktRdy = threading.Event() - class ShipBrowserWorkerThread(threading.Thread): def __init__(self): threading.Thread.__init__(self) @@ -83,7 +83,9 @@ class SearchWorkerThread(threading.Thread): def __init__(self): threading.Thread.__init__(self) self.name = "SearchWorker" - pyfalog.debug("Initialize SearchWorkerThread.") + self.jargonLoader = JargonLoader.instance() + self.jargonLoader.get_jargon() # load the jargon while in an out-of-thread context, to spot any problems + self.jargonLoader.get_jargon().apply('foobar baz') def run(self): self.cv = threading.Condition() @@ -110,13 +112,24 @@ class SearchWorkerThread(threading.Thread): else: filter_ = None + results = eos.db.searchItems(request, where=filter_, join=(types_Item.group, types_Group.category), eager=("icon", "group.category", "metaGroup", "metaGroup.parent")) + try: + jargon_request = self.jargonLoader.get_jargon().apply(request) + jargon_results = eos.db.searchItems(jargon_request, where=filter_, + join=(types_Item.group, types_Group.category), + eager=("icon", "group.category", "metaGroup", "metaGroup.parent")) + except Exception as e: + import sys + print(e, file=sys.stderr) + + items = set() # Return only published items, consult with Market service this time - for item in results: + for item in [*results, *jargon_results]: if sMkt.getPublicityByItem(item): items.add(item) wx.CallAfter(callback, items) From 9bbeec523e7ae4bca0f52d5abe8525b94aebb95d Mon Sep 17 00:00:00 2001 From: Filip Sufitchi Date: Mon, 2 Apr 2018 16:43:10 -0400 Subject: [PATCH 2/5] Remove some debug try/except stuff --- service/market.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/service/market.py b/service/market.py index b20538732..d38fb4704 100644 --- a/service/market.py +++ b/service/market.py @@ -84,8 +84,9 @@ class SearchWorkerThread(threading.Thread): threading.Thread.__init__(self) self.name = "SearchWorker" self.jargonLoader = JargonLoader.instance() - self.jargonLoader.get_jargon() # load the jargon while in an out-of-thread context, to spot any problems - self.jargonLoader.get_jargon().apply('foobar baz') + # load the jargon while in an out-of-thread context, to spot any problems while in the main thread + self.jargonLoader.get_jargon() + self.jargonLoader.get_jargon().apply('test string') def run(self): self.cv = threading.Condition() @@ -117,15 +118,10 @@ class SearchWorkerThread(threading.Thread): join=(types_Item.group, types_Group.category), eager=("icon", "group.category", "metaGroup", "metaGroup.parent")) - try: - jargon_request = self.jargonLoader.get_jargon().apply(request) - jargon_results = eos.db.searchItems(jargon_request, where=filter_, - join=(types_Item.group, types_Group.category), - eager=("icon", "group.category", "metaGroup", "metaGroup.parent")) - except Exception as e: - import sys - print(e, file=sys.stderr) - + jargon_request = self.jargonLoader.get_jargon().apply(request) + jargon_results = eos.db.searchItems(jargon_request, where=filter_, + join=(types_Item.group, types_Group.category), + eager=("icon", "group.category", "metaGroup", "metaGroup.parent")) items = set() # Return only published items, consult with Market service this time From efc07e1553f028363dbf5413895e99cc4572dcee Mon Sep 17 00:00:00 2001 From: Filip Sufitchi Date: Mon, 2 Apr 2018 17:23:30 -0400 Subject: [PATCH 3/5] Move minimum search length check to the service, to cover jargon --- config.py | 2 ++ gui/builtinMarketBrowser/itemView.py | 5 +---- service/market.py | 20 +++++++++++++------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/config.py b/config.py index 1ce7b1445..7ef0b9fa8 100644 --- a/config.py +++ b/config.py @@ -27,6 +27,8 @@ expansionName = "YC120.3" expansionVersion = "1.8" evemonMinVersion = "4081" +minItemSearchLength = 2 + pyfaPath = None savePath = None saveDB = None diff --git a/gui/builtinMarketBrowser/itemView.py b/gui/builtinMarketBrowser/itemView.py index eb0d6bb51..7a26cfed1 100644 --- a/gui/builtinMarketBrowser/itemView.py +++ b/gui/builtinMarketBrowser/itemView.py @@ -1,5 +1,6 @@ import wx +import config import gui.builtinMarketBrowser.pfSearchBox as SBox from gui.contextMenu import ContextMenu from gui.display import Display @@ -170,10 +171,6 @@ class ItemView(Display): if len(realsearch) == 0: self.selectionMade() return - # Show nothing if query is too short - elif len(realsearch) < 3: - self.clearSearch() - return self.marketBrowser.searchMode = True self.sMkt.searchItems(search, self.populateSearch) diff --git a/service/market.py b/service/market.py index d38fb4704..70794f79b 100644 --- a/service/market.py +++ b/service/market.py @@ -114,14 +114,20 @@ class SearchWorkerThread(threading.Thread): filter_ = None - results = eos.db.searchItems(request, where=filter_, - join=(types_Item.group, types_Group.category), - eager=("icon", "group.category", "metaGroup", "metaGroup.parent")) - jargon_request = self.jargonLoader.get_jargon().apply(request) - jargon_results = eos.db.searchItems(jargon_request, where=filter_, - join=(types_Item.group, types_Group.category), - eager=("icon", "group.category", "metaGroup", "metaGroup.parent")) + + + results = [] + if len(request) >= config.minItemSearchLength: + results = eos.db.searchItems(request, where=filter_, + join=(types_Item.group, types_Group.category), + eager=("icon", "group.category", "metaGroup", "metaGroup.parent")) + + jargon_results = [] + if len(jargon_request) >= config.minItemSearchLength: + jargon_results = eos.db.searchItems(jargon_request, where=filter_, + join=(types_Item.group, types_Group.category), + eager=("icon", "group.category", "metaGroup", "metaGroup.parent")) items = set() # Return only published items, consult with Market service this time From 30501feb9983e0cfff2993a9d473fc9125a7df40 Mon Sep 17 00:00:00 2001 From: Filip Sufitchi Date: Mon, 2 Apr 2018 17:25:45 -0400 Subject: [PATCH 4/5] Raise search limit back up to 3 --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 7ef0b9fa8..6b7159205 100644 --- a/config.py +++ b/config.py @@ -27,7 +27,7 @@ expansionName = "YC120.3" expansionVersion = "1.8" evemonMinVersion = "4081" -minItemSearchLength = 2 +minItemSearchLength = 3 pyfaPath = None savePath = None From 49bc9f50bc9363552c53b32bc8649e58c81c393c Mon Sep 17 00:00:00 2001 From: Filip Sufitchi Date: Mon, 2 Apr 2018 17:35:58 -0400 Subject: [PATCH 5/5] Add a comprehensive default jargon file --- service/jargon/defaults.yaml | 87 ++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/service/jargon/defaults.yaml b/service/jargon/defaults.yaml index 4fc978a1e..0217bc87c 100644 --- a/service/jargon/defaults.yaml +++ b/service/jargon/defaults.yaml @@ -1,3 +1,90 @@ +1: I 2: II +aar: Ancillary Armor Repairer +ab: Afterburner +ac: Autocannon +am: Antimatter +anp: Adaptive Nano Plating +acr: Ancillary Current Router +arty: Artillery +asb: Ancillary Shield Booster +bcs: Ballistic Control System +bcu: Ballistic Control System +boosh: Micro Jump Field Generator +ccc: Capacitor Control Circuit +cn: Caldari Navy +cnam: Caldari Navy Antimatter +cpr: Capacitor Power Relay +cpu: Co-Processor +coproc: Co-Processor +dc: Damage Control +dcu: Damage Control +disco: Smartbomb +eanm: Energized Adaptive Nano Membrane +enam: Energized Adaptive Nano Membrane +eccm: Sensor Booster +fn: Federation Navy +fnam: Federation Navy Antimatter +gd: Guidance Disruptor +ham: Heavy Assault Missile haml: Heavy Assault Missile Launcher +hm: Heavy Missile +hml: Heavy Missile Launcher +istab: Inertial Stabilizer +in: Imperial Navy +inmf: Imperial Navy Multifrequency +jam: ECM +lar: Large Armor Repairer +laar: Large Ancillary Armor Repairer +lasb: Large Ancillary Shield Booster +lm: Light Missile +lmjd: Large Micro Jump Drive +lml: Light Missile Launcher +lo: Liquid Ozone +lse: Large Shield Extender +maar: Medium Ancillary Armor Repairer +masb: Medium Ancillary Shield Booster +mf: Multifrequency +md: Guidance Disruptor +mjfg: Micro Jump Field Generator +mar: Medium Armor Repairer +mfs: Magnetic Field Stabilizer +mmjd: Medium Micro Jump Drive +mjd: Micro Jump Drive +mlu: Mining Laser Upgrade +msb: Medium Shield Booster +mse: Medium Shield Extender mwd: Microwarpdrive +odi: Overdrive Injector +point: Warp Disruptor +pdu: Power Diagnostic Unit +pp: Phased Plasma +rcu: Reactor Control Unit +rf: Republic Fleet +rhml: Rapid Heavy Missile Launcher +rl: Rocket Launcher +rlml: Rapid Light Missile Launcher +rr: Remote # Hacky, for shield, armor, and cap +rtc: Remote Tracking Computer +rtl: Rapid Torpedo Launcher +sar: Small Armor Repairer +saar: Small Ancillary Armor Repairer +sasb: Small Ancillary Shield Booster +sb: Sensor Booster # Or smartbomb? :/ +sebo: Sensor Booster +sd: Sensor Dampener +sg: Stasis Grappler +ssb: Small Shield Booster +sse: Small Shield Extender +spr: Shield Power Relay +sw: Stasis Webifier +tc: Tracking Computer +td: Tracking Disruptor +te: Tracking enhancer +tl: Remote Tracking Computer +tp: Target Painter +wcs: Warp Core Stabilizer +web: stasis +xl: X-Large +xlasb: X-Large Ancillary Shield Booster +xlsb: X-Large Shield Booster