From d451bda7ed2fdf1b3348d5a2ebdf627ed2d53088 Mon Sep 17 00:00:00 2001 From: DarkPhoenix Date: Mon, 3 Jun 2019 18:02:48 +0300 Subject: [PATCH] Add evepraisal as price source --- service/marketSources/__init__.py | 2 +- service/marketSources/evemarketdata.py | 16 +++--- service/marketSources/evemarketer.py | 22 ++++---- service/marketSources/evepraisal.py | 76 ++++++++++++++++++++++++++ service/network.py | 71 ++++++++++++++++-------- service/price.py | 2 +- service/update.py | 9 ++- 7 files changed, 151 insertions(+), 47 deletions(-) create mode 100644 service/marketSources/evepraisal.py diff --git a/service/marketSources/__init__.py b/service/marketSources/__init__.py index 863b26494..eb0162324 100644 --- a/service/marketSources/__init__.py +++ b/service/marketSources/__init__.py @@ -1 +1 @@ -__all__ = ['evemarketer', 'evemarketdata'] \ No newline at end of file +__all__ = ['evemarketer', 'evemarketdata', 'evepraisal'] diff --git a/service/marketSources/evemarketdata.py b/service/marketSources/evemarketdata.py index b592c9067..63f68f80e 100644 --- a/service/marketSources/evemarketdata.py +++ b/service/marketSources/evemarketdata.py @@ -31,7 +31,7 @@ pyfalog = Logger(__name__) class EveMarketData: - name = "eve-marketdata.com" + name = 'eve-marketdata.com' def __init__(self, priceMap, system, fetchTimeout): # Try selected system first @@ -42,24 +42,24 @@ class EveMarketData: @staticmethod def fetchPrices(priceMap, fetchTimeout, system=None): - params = {"type_ids": ','.join(str(typeID) for typeID in priceMap)} + params = {'type_ids': ','.join(str(typeID) for typeID in priceMap)} if system is not None: - params["system_id"] = system - baseurl = "https://eve-marketdata.com/api/item_prices.xml" + params['system_id'] = system + baseurl = 'https://eve-marketdata.com/api/item_prices.xml' network = Network.getInstance() - data = network.request(baseurl, network.PRICES, params=params, timeout=fetchTimeout) + data = network.get(url=baseurl, type=network.PRICES, params=params, timeout=fetchTimeout) xml = minidom.parseString(data.text) - types = xml.getElementsByTagName("eve").item(0).getElementsByTagName("price") + types = xml.getElementsByTagName('eve').item(0).getElementsByTagName('price') # Cycle through all types we've got from request for type_ in types: # Get data out of each typeID details tree - typeID = int(type_.getAttribute("id")) + typeID = int(type_.getAttribute('id')) try: price = float(type_.firstChild.data) except (TypeError, ValueError): - pyfalog.warning("Failed to get price for: {0}", type_) + pyfalog.warning('Failed to get price for: {0}', type_) continue # eve-marketdata returns 0 if price data doesn't even exist for the item diff --git a/service/marketSources/evemarketer.py b/service/marketSources/evemarketer.py index 6d85380ab..1e6d16260 100644 --- a/service/marketSources/evemarketer.py +++ b/service/marketSources/evemarketer.py @@ -31,7 +31,7 @@ pyfalog = Logger(__name__) class EveMarketer: - name = "evemarketer" + name = 'evemarketer' def __init__(self, priceMap, system, fetchTimeout): # Try selected system first @@ -42,24 +42,24 @@ class EveMarketer: @staticmethod def fetchPrices(priceMap, fetchTimeout, system=None): - params = {"typeid": {typeID for typeID in priceMap}} + params = {'typeid': {typeID for typeID in priceMap}} if system is not None: - params["usesystem"] = system - baseurl = "https://api.evemarketer.com/ec/marketstat" + params['usesystem'] = system + baseurl = 'https://api.evemarketer.com/ec/marketstat' network = Network.getInstance() - data = network.request(baseurl, network.PRICES, params=params, timeout=fetchTimeout) + data = network.get(url=baseurl, type=network.PRICES, params=params, timeout=fetchTimeout) xml = minidom.parseString(data.text) - types = xml.getElementsByTagName("marketstat").item(0).getElementsByTagName("type") + types = xml.getElementsByTagName('marketstat').item(0).getElementsByTagName('type') # Cycle through all types we've got from request for type_ in types: # Get data out of each typeID details tree - typeID = int(type_.getAttribute("id")) - sell = type_.getElementsByTagName("sell").item(0) - # If price data wasn't there, set price to zero + typeID = int(type_.getAttribute('id')) + sell = type_.getElementsByTagName('sell').item(0) + # If price data wasn't there, skip the item try: - percprice = float(sell.getElementsByTagName("percentile").item(0).firstChild.data) + percprice = float(sell.getElementsByTagName('percentile').item(0).firstChild.data) except (TypeError, ValueError): - pyfalog.warning("Failed to get price for: {0}", type_) + pyfalog.warning('Failed to get price for: {0}', type_) continue # Price is 0 if evemarketer has info on this item, but it is not available diff --git a/service/marketSources/evepraisal.py b/service/marketSources/evepraisal.py new file mode 100644 index 000000000..b09299368 --- /dev/null +++ b/service/marketSources/evepraisal.py @@ -0,0 +1,76 @@ +# ============================================================================= +# Copyright (C) 2010 Diego Duclos +# +# This file is part of pyfa. +# +# pyfa is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# pyfa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pyfa. If not, see . +# ============================================================================= + + +from logbook import Logger + +from eos.saveddata.price import PriceStatus +from service.network import Network +from service.price import Price + +pyfalog = Logger(__name__) + +systemAliases = { + None: 'universe', + 30000142: 'jita', + 30002187: 'amarr', + 30002659: 'dodixie', + 30002510: 'rens', + 30002053: 'hek'} + + +class EvePraisal: + + name = 'evepraisal' + + def __init__(self, priceMap, system, fetchTimeout): + # Try selected system first + self.fetchPrices(priceMap, max(2 * fetchTimeout / 3, 2), system) + # If price was not available - try globally + if priceMap: + self.fetchPrices(priceMap, max(fetchTimeout / 3, 2)) + + @staticmethod + def fetchPrices(priceMap, fetchTimeout, system=None): + jsonData = { + 'market_name': systemAliases[system], + 'items': [{'type_id': typeID for typeID in priceMap}]} + baseurl = 'https://evepraisal.com/appraisal/structured.json' + network = Network.getInstance() + resp = network.post(baseurl, network.PRICES, jsonData=jsonData, timeout=fetchTimeout) + data = resp.json() + try: + itemsData = data['appraisal']['items'] + except (KeyError, TypeError): + return False + # Cycle through all types we've got from request + for itemData in itemsData: + try: + typeID = int(itemData['typeID']) + percprice = itemData['prices']['sell']['percentile'] + except (KeyError, TypeError): + continue + # evepraisal returns 0 if price data doesn't even exist for the item + if percprice == 0: + continue + priceMap[typeID].update(PriceStatus.fetchSuccess, percprice) + del priceMap[typeID] + + +Price.register(EvePraisal) diff --git a/service/network.py b/service/network.py index 5ce413445..2c16980ea 100644 --- a/service/network.py +++ b/service/network.py @@ -70,36 +70,18 @@ class Network: return cls._instance - def request(self, url, type, *args, **kwargs): + def get(self, url, type, **kwargs): + self.__networkAccessCheck(type) - # URL is required to be https as of right now - # print "Starting request: %s\n\tType: %s\n\tPost Data: %s"%(url,type,data) - - # Make sure request is enabled - access = NetworkSettings.getInstance().getAccess() - - if not self.ENABLED & access or not type & access: - pyfalog.warning("Access not enabled - please enable in Preferences > Network") - raise Error("Access not enabled - please enable in Preferences > Network") - - # Set up some things for the request - versionString = "{0}".format(config.version) - headers = {"User-Agent": "pyfa {0} (python-requests {1})".format(versionString, requests.__version__)} - # user-agent: pyfa 2.0.0b4 git -YC120.2 1.2 (python-requests 2.18.4) - - # python-requests supports setting proxy for request as parameter to get() / post() - # in a form like: proxies = { 'http': 'http://10.10.1.10:3128', 'https': 'http://10.10.1.10:1080' } - # or with HTTP Basic auth support: proxies = {'http': 'http://user:pass@10.10.1.10:3128/'} - # then you do: requests.get('http://example.org', proxies=proxies) - - proxies = NetworkSettings.getInstance().getProxySettingsInRequestsFormat() + headers = self.__getHeaders() + proxies = self.__getProxies() try: resp = requests.get(url, headers=headers, proxies=proxies, **kwargs) resp.raise_for_status() return resp except requests.exceptions.HTTPError as error: - pyfalog.warning("HTTPError:") + pyfalog.warning('HTTPError:') pyfalog.warning(error) if error.response.status_code == 404: raise RequestError() @@ -112,3 +94,46 @@ class Network: raise TimeoutError() except Exception as error: raise Error(error) + + def post(self, url, type, jsonData, **kwargs): + self.__networkAccessCheck(type) + + headers = self.__getHeaders() + proxies = self.__getProxies() + + try: + resp = requests.post(url, json=jsonData, headers=headers, proxies=proxies, **kwargs) + resp.raise_for_status() + return resp + except requests.exceptions.HTTPError as error: + pyfalog.warning('HTTPError:') + pyfalog.warning(error) + if error.response.status_code == 404: + raise RequestError() + elif error.response.status_code == 403: + raise AuthenticationError() + elif error.response.status_code >= 500: + raise ServerError() + raise Error(error) + except requests.exceptions.Timeout: + raise TimeoutError() + except Exception as error: + raise Error(error) + + def __networkAccessCheck(self, type): + # Make sure request is enabled + access = NetworkSettings.getInstance().getAccess() + if not self.ENABLED & access or not type & access: + pyfalog.warning('Access not enabled - please enable in Preferences > Network') + raise Error('Access not enabled - please enable in Preferences > Network') + + def __getHeaders(self): + versionString = '{0}'.format(config.version) + return {'User-Agent': 'pyfa {0} (python-requests {1})'.format(versionString, requests.__version__)} + + def __getProxies(self): + # python-requests supports setting proxy for request as parameter to get() / post() + # in a form like: proxies = { 'http': 'http://10.10.1.10:3128', 'https': 'http://10.10.1.10:1080' } + # or with HTTP Basic auth support: proxies = {'http': 'http://user:pass@10.10.1.10:3128/'} + # then you do: requests.get('http://example.org', proxies=proxies) + return NetworkSettings.getInstance().getProxySettingsInRequestsFormat() diff --git a/service/price.py b/service/price.py index 23b6afbcf..2b1d07ca5 100644 --- a/service/price.py +++ b/service/price.py @@ -247,4 +247,4 @@ class PriceWorkerThread(threading.Thread): # Import market sources only to initialize price source modules, they register on their own -from service.marketSources import evemarketer, evemarketdata # noqa: E402 +from service.marketSources import evemarketer, evemarketdata, evepraisal # noqa: E402 diff --git a/service/update.py b/service/update.py index a2f4cf730..5adff0044 100644 --- a/service/update.py +++ b/service/update.py @@ -47,10 +47,13 @@ class CheckUpdateThread(threading.Thread): try: try: - response = network.request('https://www.pyfa.io/update_check?pyfa_version={}&client_hash={}'.format( - config.version, config.getClientSecret()), network.UPDATE) + response = network.get( + url='https://www.pyfa.io/update_check?pyfa_version={}&client_hash={}'.format(config.version, config.getClientSecret()), + type=network.UPDATE) except Exception as e: - response = network.request('https://api.github.com/repos/pyfa-org/Pyfa/releases', network.UPDATE) + response = network.get( + url='https://api.github.com/repos/pyfa-org/Pyfa/releases', + type=network.UPDATE) jsonResponse = response.json() jsonResponse.sort(