From 943ee517f4380a8df44079a15aeda8e2d7a1a4be Mon Sep 17 00:00:00 2001 From: Filip Sufitchi Date: Mon, 2 Apr 2018 16:40:39 -0400 Subject: [PATCH] Add jargon service Jargon service translates search strings for the market using community-sourced abbreviations. It also creates the jargon.yaml file in the user's save directory, for future customization (either manual, or via the UI) and provides the code hooks to load and update it. The JargonLoader is capable of hot-loading new jargon.yaml configurations without restarting Pyfa. --- requirements.txt | 3 +- service/jargon/__init__.py | 21 ++++++++++ service/jargon/defaults.yaml | 3 ++ service/jargon/header.yaml | 14 +++++++ service/jargon/jargon.py | 47 +++++++++++++++++++++ service/jargon/loader.py | 81 ++++++++++++++++++++++++++++++++++++ service/jargon/resources.py | 23 ++++++++++ service/market.py | 19 +++++++-- 8 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 service/jargon/__init__.py create mode 100644 service/jargon/defaults.yaml create mode 100644 service/jargon/header.yaml create mode 100644 service/jargon/jargon.py create mode 100644 service/jargon/loader.py create mode 100644 service/jargon/resources.py diff --git a/requirements.txt b/requirements.txt index 2b38e8685..7a2e5f65f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ sqlalchemy >= 1.0.5 markdown2 packaging roman -beautifulsoup4 \ No newline at end of file +beautifulsoup4 +PyYAML diff --git a/service/jargon/__init__.py b/service/jargon/__init__.py new file mode 100644 index 000000000..447c917e3 --- /dev/null +++ b/service/jargon/__init__.py @@ -0,0 +1,21 @@ +# ============================================================================= +# Copyright (C) 2018 Filip Sufitchi +# +# This file is part of pyfa. +# +# pyfa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyfa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyfa. If not, see . +# ============================================================================= + +from .jargon import Jargon +from .loader import JargonLoader diff --git a/service/jargon/defaults.yaml b/service/jargon/defaults.yaml new file mode 100644 index 000000000..4fc978a1e --- /dev/null +++ b/service/jargon/defaults.yaml @@ -0,0 +1,3 @@ +2: II +haml: Heavy Assault Missile Launcher +mwd: Microwarpdrive diff --git a/service/jargon/header.yaml b/service/jargon/header.yaml new file mode 100644 index 000000000..031effff6 --- /dev/null +++ b/service/jargon/header.yaml @@ -0,0 +1,14 @@ +# This is the default Pyfa jargon file. +# +# It is essentially a giant set of find/replace statements in order to translate +# abbreviated Eve community terms into more useful full terms. It is intended +# for translation of strings such as "haml 2" "into "Heavy Assault Missile Launcher II".. +# +# These abbreviations are not case-sensitive. If abbreviations collide, the +# later one is used. +# +# Abbreviations with spaces are not supported. +# +# Syntax: +# +# abbreviation: full name diff --git a/service/jargon/jargon.py b/service/jargon/jargon.py new file mode 100644 index 000000000..0a5065faf --- /dev/null +++ b/service/jargon/jargon.py @@ -0,0 +1,47 @@ +# ============================================================================= +# Copyright (C) 2018 Filip Sufitchi +# +# This file is part of pyfa. +# +# pyfa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyfa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyfa. If not, see . +# ============================================================================= + +import config +import pkg_resources + +class Jargon(object): + def __init__(self, rawdata: dict): + self._rawdata = rawdata + + # copy the data to lowercase keys, ignore blank keys + self._data = {str(k).lower():v for k,v in rawdata.items() if k} + + def get(self, term: str) -> str: + return self._data.get(term.lower()) + + def get_rawdata() -> dict: + return self._rawdata + + def apply(self, query): + query_words = query.split() + parts = [] + + for word in query_words: + replacement = self.get(word) + if replacement: + parts.append(replacement) + else: + parts.append(word) + + return ' '.join(parts) diff --git a/service/jargon/loader.py b/service/jargon/loader.py new file mode 100644 index 000000000..11e95b7c4 --- /dev/null +++ b/service/jargon/loader.py @@ -0,0 +1,81 @@ +# ============================================================================= +# Copyright (C) 2018 Filip Sufitchi +# +# This file is part of pyfa. +# +# pyfa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyfa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyfa. If not, see . +# ============================================================================= + +import os +import config +import yaml + +from .jargon import Jargon +from .resources import DEFAULT_DATA, DEFAULT_HEADER + +JARGON_PATH = os.path.join(config.savePath, 'jargon.yaml') + +class JargonLoader(object): + def __init__(self, jargon_path: str): + self.jargon_path = jargon_path + self._jargon_mtime = 0 # type: int + self._jargon = None # type: Jargon + + def save_jargon(self, data: Jargon): + rawdata = data.get_rawdata() + with open(JARGON_PATH, 'w') as f: + yaml.dump(rawdata, stream=f, default_flow_style=False) + + def get_jargon(self) -> Jargon: + if self._is_stale(): + self._load_jargon() + return self._jargon + + def _is_stale(self): + return (not self._jargon or not self._jargon_mtime or + self.jargon_mtime != self._get_jargon_file_mtime()) + + def _load_jargon(self): + with open(JARGON_PATH) as f: + rawdata = yaml.load(f) + self.jargon_mtime = self._get_jargon_file_mtime() + self._jargon = Jargon(rawdata) + + def _get_jargon_file_mtime(self) -> int: + if not os.path.exists(self.jargon_path): + return 0 + return os.stat(self.jargon_path).st_mtime + + @staticmethod + def init_user_jargon(jargon_path): + values = yaml.load(DEFAULT_DATA) + if os.path.exists(jargon_path): + with open(jargon_path) as f: + custom_values = yaml.load(f) + if custom_values: + values.update(custom_values) + with open(jargon_path, 'w') as f: + f.write(DEFAULT_HEADER) + f.write('\n\n') + yaml.dump(values, stream=f, default_flow_style=False) + + _instance = None + @staticmethod + def instance(jargon_path=None): + if not JargonLoader._instance: + jargon_path = jargon_path or JARGON_PATH + JargonLoader._instance = JargonLoader(jargon_path) + return JargonLoader._instance + +JargonLoader.init_user_jargon(JARGON_PATH) diff --git a/service/jargon/resources.py b/service/jargon/resources.py new file mode 100644 index 000000000..f6c631b0e --- /dev/null +++ b/service/jargon/resources.py @@ -0,0 +1,23 @@ +# ============================================================================= +# Copyright (C) 2018 Filip Sufitchi +# +# This file is part of pyfa. +# +# pyfa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyfa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyfa. If not, see . +# ============================================================================= + +import pkg_resources + +DEFAULT_DATA = pkg_resources.resource_string(__name__, 'defaults.yaml').decode() +DEFAULT_HEADER = pkg_resources.resource_string(__name__, 'header.yaml').decode() diff --git a/service/market.py b/service/market.py index 06fd18465..b20538732 100644 --- a/service/market.py +++ b/service/market.py @@ -30,6 +30,7 @@ import config import eos.db from service import conversions from service.settings import SettingsProvider +from service.jargon import JargonLoader from eos.gamedata import Category as types_Category, Group as types_Group, Item as types_Item, MarketGroup as types_MarketGroup, \ MetaGroup as types_MetaGroup, MetaType as types_MetaType @@ -40,7 +41,6 @@ pyfalog = Logger(__name__) # Event which tells threads dependent on Market that it's initialized mktRdy = threading.Event() - class ShipBrowserWorkerThread(threading.Thread): def __init__(self): threading.Thread.__init__(self) @@ -83,7 +83,9 @@ class SearchWorkerThread(threading.Thread): def __init__(self): threading.Thread.__init__(self) self.name = "SearchWorker" - pyfalog.debug("Initialize SearchWorkerThread.") + self.jargonLoader = JargonLoader.instance() + self.jargonLoader.get_jargon() # load the jargon while in an out-of-thread context, to spot any problems + self.jargonLoader.get_jargon().apply('foobar baz') def run(self): self.cv = threading.Condition() @@ -110,13 +112,24 @@ class SearchWorkerThread(threading.Thread): else: filter_ = None + results = eos.db.searchItems(request, where=filter_, join=(types_Item.group, types_Group.category), eager=("icon", "group.category", "metaGroup", "metaGroup.parent")) + try: + jargon_request = self.jargonLoader.get_jargon().apply(request) + jargon_results = eos.db.searchItems(jargon_request, where=filter_, + join=(types_Item.group, types_Group.category), + eager=("icon", "group.category", "metaGroup", "metaGroup.parent")) + except Exception as e: + import sys + print(e, file=sys.stderr) + + items = set() # Return only published items, consult with Market service this time - for item in results: + for item in [*results, *jargon_results]: if sMkt.getPublicityByItem(item): items.add(item) wx.CallAfter(callback, items)