diff --git a/requirements.txt b/requirements.txt index 2b38e8685..7a2e5f65f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ sqlalchemy >= 1.0.5 markdown2 packaging roman -beautifulsoup4 \ No newline at end of file +beautifulsoup4 +PyYAML diff --git a/service/jargon/__init__.py b/service/jargon/__init__.py new file mode 100644 index 000000000..447c917e3 --- /dev/null +++ b/service/jargon/__init__.py @@ -0,0 +1,21 @@ +# ============================================================================= +# Copyright (C) 2018 Filip Sufitchi +# +# This file is part of pyfa. +# +# pyfa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyfa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyfa. If not, see . +# ============================================================================= + +from .jargon import Jargon +from .loader import JargonLoader diff --git a/service/jargon/defaults.yaml b/service/jargon/defaults.yaml new file mode 100644 index 000000000..4fc978a1e --- /dev/null +++ b/service/jargon/defaults.yaml @@ -0,0 +1,3 @@ +2: II +haml: Heavy Assault Missile Launcher +mwd: Microwarpdrive diff --git a/service/jargon/header.yaml b/service/jargon/header.yaml new file mode 100644 index 000000000..031effff6 --- /dev/null +++ b/service/jargon/header.yaml @@ -0,0 +1,14 @@ +# This is the default Pyfa jargon file. +# +# It is essentially a giant set of find/replace statements in order to translate +# abbreviated Eve community terms into more useful full terms. It is intended +# for translation of strings such as "haml 2" "into "Heavy Assault Missile Launcher II".. +# +# These abbreviations are not case-sensitive. If abbreviations collide, the +# later one is used. +# +# Abbreviations with spaces are not supported. +# +# Syntax: +# +# abbreviation: full name diff --git a/service/jargon/jargon.py b/service/jargon/jargon.py new file mode 100644 index 000000000..0a5065faf --- /dev/null +++ b/service/jargon/jargon.py @@ -0,0 +1,47 @@ +# ============================================================================= +# Copyright (C) 2018 Filip Sufitchi +# +# This file is part of pyfa. +# +# pyfa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyfa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyfa. If not, see . +# ============================================================================= + +import config +import pkg_resources + +class Jargon(object): + def __init__(self, rawdata: dict): + self._rawdata = rawdata + + # copy the data to lowercase keys, ignore blank keys + self._data = {str(k).lower():v for k,v in rawdata.items() if k} + + def get(self, term: str) -> str: + return self._data.get(term.lower()) + + def get_rawdata() -> dict: + return self._rawdata + + def apply(self, query): + query_words = query.split() + parts = [] + + for word in query_words: + replacement = self.get(word) + if replacement: + parts.append(replacement) + else: + parts.append(word) + + return ' '.join(parts) diff --git a/service/jargon/loader.py b/service/jargon/loader.py new file mode 100644 index 000000000..11e95b7c4 --- /dev/null +++ b/service/jargon/loader.py @@ -0,0 +1,81 @@ +# ============================================================================= +# Copyright (C) 2018 Filip Sufitchi +# +# This file is part of pyfa. +# +# pyfa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyfa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyfa. If not, see . +# ============================================================================= + +import os +import config +import yaml + +from .jargon import Jargon +from .resources import DEFAULT_DATA, DEFAULT_HEADER + +JARGON_PATH = os.path.join(config.savePath, 'jargon.yaml') + +class JargonLoader(object): + def __init__(self, jargon_path: str): + self.jargon_path = jargon_path + self._jargon_mtime = 0 # type: int + self._jargon = None # type: Jargon + + def save_jargon(self, data: Jargon): + rawdata = data.get_rawdata() + with open(JARGON_PATH, 'w') as f: + yaml.dump(rawdata, stream=f, default_flow_style=False) + + def get_jargon(self) -> Jargon: + if self._is_stale(): + self._load_jargon() + return self._jargon + + def _is_stale(self): + return (not self._jargon or not self._jargon_mtime or + self.jargon_mtime != self._get_jargon_file_mtime()) + + def _load_jargon(self): + with open(JARGON_PATH) as f: + rawdata = yaml.load(f) + self.jargon_mtime = self._get_jargon_file_mtime() + self._jargon = Jargon(rawdata) + + def _get_jargon_file_mtime(self) -> int: + if not os.path.exists(self.jargon_path): + return 0 + return os.stat(self.jargon_path).st_mtime + + @staticmethod + def init_user_jargon(jargon_path): + values = yaml.load(DEFAULT_DATA) + if os.path.exists(jargon_path): + with open(jargon_path) as f: + custom_values = yaml.load(f) + if custom_values: + values.update(custom_values) + with open(jargon_path, 'w') as f: + f.write(DEFAULT_HEADER) + f.write('\n\n') + yaml.dump(values, stream=f, default_flow_style=False) + + _instance = None + @staticmethod + def instance(jargon_path=None): + if not JargonLoader._instance: + jargon_path = jargon_path or JARGON_PATH + JargonLoader._instance = JargonLoader(jargon_path) + return JargonLoader._instance + +JargonLoader.init_user_jargon(JARGON_PATH) diff --git a/service/jargon/resources.py b/service/jargon/resources.py new file mode 100644 index 000000000..f6c631b0e --- /dev/null +++ b/service/jargon/resources.py @@ -0,0 +1,23 @@ +# ============================================================================= +# Copyright (C) 2018 Filip Sufitchi +# +# This file is part of pyfa. +# +# pyfa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyfa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyfa. If not, see . +# ============================================================================= + +import pkg_resources + +DEFAULT_DATA = pkg_resources.resource_string(__name__, 'defaults.yaml').decode() +DEFAULT_HEADER = pkg_resources.resource_string(__name__, 'header.yaml').decode() diff --git a/service/market.py b/service/market.py index 06fd18465..b20538732 100644 --- a/service/market.py +++ b/service/market.py @@ -30,6 +30,7 @@ import config import eos.db from service import conversions from service.settings import SettingsProvider +from service.jargon import JargonLoader from eos.gamedata import Category as types_Category, Group as types_Group, Item as types_Item, MarketGroup as types_MarketGroup, \ MetaGroup as types_MetaGroup, MetaType as types_MetaType @@ -40,7 +41,6 @@ pyfalog = Logger(__name__) # Event which tells threads dependent on Market that it's initialized mktRdy = threading.Event() - class ShipBrowserWorkerThread(threading.Thread): def __init__(self): threading.Thread.__init__(self) @@ -83,7 +83,9 @@ class SearchWorkerThread(threading.Thread): def __init__(self): threading.Thread.__init__(self) self.name = "SearchWorker" - pyfalog.debug("Initialize SearchWorkerThread.") + self.jargonLoader = JargonLoader.instance() + self.jargonLoader.get_jargon() # load the jargon while in an out-of-thread context, to spot any problems + self.jargonLoader.get_jargon().apply('foobar baz') def run(self): self.cv = threading.Condition() @@ -110,13 +112,24 @@ class SearchWorkerThread(threading.Thread): else: filter_ = None + results = eos.db.searchItems(request, where=filter_, join=(types_Item.group, types_Group.category), eager=("icon", "group.category", "metaGroup", "metaGroup.parent")) + try: + jargon_request = self.jargonLoader.get_jargon().apply(request) + jargon_results = eos.db.searchItems(jargon_request, where=filter_, + join=(types_Item.group, types_Group.category), + eager=("icon", "group.category", "metaGroup", "metaGroup.parent")) + except Exception as e: + import sys + print(e, file=sys.stderr) + + items = set() # Return only published items, consult with Market service this time - for item in results: + for item in [*results, *jargon_results]: if sMkt.getPublicityByItem(item): items.add(item) wx.CallAfter(callback, items)