dave/pyfa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

Allocate passed timeout across multiple fetches to avoid exceeding it, with focus on primary data source

Browse Source
This commit is contained in:
DarkPhoenix
2019-02-21 16:05:35 +03:00
parent f9eef5ee07
commit c66f80d6e5
3 changed files with 28 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
service/marketSources/evemarketdata.py
View File

@@ -35,9 +35,9 @@ class EveMarketData:
def __init__(self, priceMap, system, fetchTimeout):
# Try selected system first
self.fetchPrices(priceMap, fetchTimeout, system)
self.fetchPrices(priceMap, max(2 * fetchTimeout / 3, 2), system)
# If price was not available - try globally
self.fetchPrices(priceMap, fetchTimeout)
self.fetchPrices(priceMap, max(fetchTimeout / 3, 2))
def fetchPrices(self, priceMap, fetchTimeout, system=None):
params = {"type_ids": ','.join(str(typeID) for typeID in priceMap)}

4
service/marketSources/evemarketer.py
View File

@@ -35,9 +35,9 @@ class EveMarketer:
def __init__(self, priceMap, system, fetchTimeout):
# Try selected system first
self.fetchPrices(priceMap, fetchTimeout, system)
self.fetchPrices(priceMap, max(2 * fetchTimeout / 3, 2), system)
# If price was not available - try globally
self.fetchPrices(priceMap, fetchTimeout)
self.fetchPrices(priceMap, max(fetchTimeout / 3, 2))
def fetchPrices(self, priceMap, fetchTimeout, system=None):
params = {"typeid": {typeID for typeID in priceMap}}

39
service/price.py
View File

@@ -101,30 +101,39 @@ class Price:
return
# attempt to find user's selected price source, otherwise get first one
sourcesToTry = list(cls.sources.keys())
curr = sFit.serviceFittingOptions["priceSource"] if sFit.serviceFittingOptions["priceSource"] in sourcesToTry else sourcesToTry[0]
sourceAll = list(cls.sources.keys())
sourcePrimary = sFit.serviceFittingOptions["priceSource"] if sFit.serviceFittingOptions["priceSource"] in sourceAll else sourceAll[0]
# Format: {source name: timeout weight}
sources = {sourcePrimary: len(sourceAll)}
for source in sourceAll:
if source == sourcePrimary:
continue
sources[source] = min(sources.values()) - 1
timeoutWeightMult = fetchTimeout / sum(sources.values())
# Record timeouts as it will affect our final decision
timeouts = {}
timedOutSources = {}
while priceMap and sourcesToTry:
timeouts[curr] = False
sourcesToTry.remove(curr)
for source, timeoutWeight in sources.items():
pyfalog.info('Trying {}'.format(source))
timedOutSources[source] = False
sourceFetchTimeout = timeoutWeight * timeoutWeightMult
try:
sourceCls = cls.sources.get(curr)
sourceCls(priceMap, cls.systemsList[sFit.serviceFittingOptions["priceSystem"]], fetchTimeout)
sourceCls = cls.sources.get(source)
sourceCls(priceMap, cls.systemsList[sFit.serviceFittingOptions["priceSystem"]], sourceFetchTimeout)
except TimeoutError:
pyfalog.warning("Price fetch timeout for source {}".format(curr))
timeouts[curr] = True
pyfalog.warning("Price fetch timeout for source {}".format(source))
timedOutSources[source] = True
except Exception as e:
pyfalog.warn('Failed to fetch prices from price source {}: {}'.format(curr, e))
if sourcesToTry:
curr = sourcesToTry[0]
pyfalog.warn('Trying {}'.format(curr))
pyfalog.warn('Failed to fetch prices from price source {}: {}'.format(source, e))
# Sources remove price map items as they fetch info, if none remain then we're done
if not priceMap:
break
# If we get to this point, then we've failed to get price with all our sources
# If all sources failed due to timeouts, set one status
if all(to is True for to in timeouts.values()):
if all(to is True for to in timedOutSources.values()):
for typeID in priceMap.keys():
priceMap[typeID].update(PriceStatus.fetchTimeout)
# If some sources failed due to any other reason, then it's definitely not network