From 04666781761122f52b5dcb204dc20e76b9a5d2b5 Mon Sep 17 00:00:00 2001 From: Felix Wehnert Date: Wed, 9 May 2018 10:20:11 +0200 Subject: [PATCH 01/11] Add Drone Damage Amplifier to defaults --- service/jargon/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/service/jargon/defaults.yaml b/service/jargon/defaults.yaml index 0217bc87c..50d2ab13d 100644 --- a/service/jargon/defaults.yaml +++ b/service/jargon/defaults.yaml @@ -19,6 +19,7 @@ cpu: Co-Processor coproc: Co-Processor dc: Damage Control dcu: Damage Control +dda: Drone Damage Amplifier disco: Smartbomb eanm: Energized Adaptive Nano Membrane enam: Energized Adaptive Nano Membrane From 4d666907c9a41540f11c5184ab1807e9cd8a07b4 Mon Sep 17 00:00:00 2001 From: blitzmann Date: Fri, 11 May 2018 23:24:30 -0400 Subject: [PATCH 02/11] Start breaking out esipy, first up: getting for login --- service/esi.py | 17 +++++++++++++++-- service/esi_security_proxy.py | 13 ------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/service/esi.py b/service/esi.py index 76f870123..943f2cbbc 100644 --- a/service/esi.py +++ b/service/esi.py @@ -31,15 +31,24 @@ cache_path = os.path.join(config.savePath, config.ESI_CACHE) from esipy.events import AFTER_TOKEN_REFRESH +from urllib.parse import urlencode + if not os.path.exists(cache_path): os.mkdir(cache_path) file_cache = FileCache(cache_path) +sso_url = "https://www.pyfa.io" # "https://login.eveonline.com" for actual login +esi_url = "https://esi.tech.ccp.is" + +oauth_authorize = '%s/oauth/authorize' % sso_url +oauth_token = '%s/oauth/token' % sso_url + class EsiException(Exception): pass + class Servers(Enum): TQ = 0 SISI = 1 @@ -201,6 +210,8 @@ class Esi(object): char.accessTokenExpires = datetime.datetime.fromtimestamp(time.time() + tokenResponse['expires_in']) if 'refresh_token' in tokenResponse: char.refreshToken = config.cipher.encrypt(tokenResponse['refresh_token'].encode()) + + # remove, no longer need? if char.esi_client is not None: char.esi_client.security.update_token(tokenResponse) @@ -219,7 +230,6 @@ class Esi(object): def getLoginURI(self, redirect=None): self.state = str(uuid.uuid4()) - esisecurity = EsiSecurityProxy(sso_url=config.ESI_AUTH_PROXY) args = { 'state': self.state, @@ -231,7 +241,10 @@ class Esi(object): if redirect is not None: args['redirect'] = redirect - return esisecurity.get_auth_uri(**args) + return '%s?%s' % ( + oauth_authorize, + urlencode(args) + ) def startServer(self): # todo: break this out into two functions: starting the server, and getting the URI pyfalog.debug("Starting server") diff --git a/service/esi_security_proxy.py b/service/esi_security_proxy.py index 7c29f80b0..277eb3aa1 100644 --- a/service/esi_security_proxy.py +++ b/service/esi_security_proxy.py @@ -116,19 +116,6 @@ class EsiSecurityProxy(object): return request_params - def get_auth_uri(self, *args, **kwargs): - """ Constructs the full auth uri and returns it. - - :param state: The state to pass through the auth process - :param redirect: The URI that the proxy server will redirect to - :return: the authorizationUrl with the correct parameters. - """ - - return '%s?%s' % ( - self.oauth_authorize, - urlencode(kwargs) - ) - def get_refresh_token_params(self): """ Return the param object for the post() call to get the access_token from the refresh_token From 0365f71c00be9da15f9cf5a098339d9b4a795cac Mon Sep 17 00:00:00 2001 From: blitzmann Date: Sat, 12 May 2018 00:46:26 -0400 Subject: [PATCH 03/11] Move over ESI functionality to be completely separate from esipy --- eos/saveddata/ssocharacter.py | 4 + gui/esiFittings.py | 3 +- requirements.txt | 1 - service/esi.py | 245 +++++++++++++++++++++------------- service/esi_security_proxy.py | 222 ------------------------------ 5 files changed, 156 insertions(+), 319 deletions(-) delete mode 100644 service/esi_security_proxy.py diff --git a/eos/saveddata/ssocharacter.py b/eos/saveddata/ssocharacter.py index 9ffed8d46..352e3724a 100644 --- a/eos/saveddata/ssocharacter.py +++ b/eos/saveddata/ssocharacter.py @@ -50,6 +50,10 @@ class SsoCharacter(object): ).total_seconds() } + def is_token_expired(self): + if self.accessTokenExpires is None: + return True + return datetime.datetime.now() >= self.accessTokenExpires def __repr__(self): return "SsoCharacter(ID={}, name={}, client={}) at {}".format( diff --git a/gui/esiFittings.py b/gui/esiFittings.py index 92fd89b83..2f53a5819 100644 --- a/gui/esiFittings.py +++ b/gui/esiFittings.py @@ -16,8 +16,7 @@ import gui.globalEvents as GE from logbook import Logger import calendar -from service.esi import Esi -from esipy.exceptions import APIException +from service.esi import Esi, APIException from service.port import ESIExportException pyfalog = Logger(__name__) diff --git a/requirements.txt b/requirements.txt index c55887d95..a938d0217 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ matplotlib >= 2.0.0 python-dateutil requests >= 2.0.0 sqlalchemy >= 1.0.5 -esipy == 0.3.4 cryptography diskcache markdown2 diff --git a/service/esi.py b/service/esi.py index 943f2cbbc..adbf7e774 100644 --- a/service/esi.py +++ b/service/esi.py @@ -19,35 +19,53 @@ import gui.globalEvents as GE from service.server import StoppableHTTPServer, AuthHandler from service.settings import EsiSettings -from .esi_security_proxy import EsiSecurityProxy -from esipy import EsiClient, EsiApp -from esipy.cache import FileCache - import wx +from requests import Session pyfalog = Logger(__name__) -cache_path = os.path.join(config.savePath, config.ESI_CACHE) - -from esipy.events import AFTER_TOKEN_REFRESH - from urllib.parse import urlencode -if not os.path.exists(cache_path): - os.mkdir(cache_path) +# todo: reimplement Caching for calls +# from esipy.cache import FileCache +# file_cache = FileCache(cache_path) +# cache_path = os.path.join(config.savePath, config.ESI_CACHE) +# +# if not os.path.exists(cache_path): +# os.mkdir(cache_path) +# -file_cache = FileCache(cache_path) - - -sso_url = "https://www.pyfa.io" # "https://login.eveonline.com" for actual login +sso_url = "https://www.pyfa.io" # "https://login.eveonline.com" for actual login esi_url = "https://esi.tech.ccp.is" oauth_authorize = '%s/oauth/authorize' % sso_url oauth_token = '%s/oauth/token' % sso_url -class EsiException(Exception): - pass +class APIException(Exception): + """ Exception for SSO related errors """ + + def __init__(self, url, code, json_response): + self.url = url + self.status_code = code + self.response = json_response + super(APIException, self).__init__(str(self)) + + def __str__(self): + if 'error' in self.response: + return 'HTTP Error %s: %s' % (self.status_code, + self.response['error']) + elif 'message' in self.response: + return 'HTTP Error %s: %s' % (self.status_code, + self.response['message']) + return 'HTTP Error %s' % (self.status_code) + + +class ESIEndpoints(Enum): + CHAR = "/v4/characters/{character_id}/" + CHAR_SKILLS = "/v4/characters/{character_id}/skills/" # prepend https://esi.evetech.net/ + CHAR_FITTINGS = "/v1/characters/{character_id}/fittings/" + CHAR_DEL_FIT = "/v1/characters/{character_id}/fittings/{fitting_id}/" class Servers(Enum): TQ = 0 @@ -60,31 +78,8 @@ class LoginMethod(Enum): class Esi(object): - esiapp = None - esi_v1 = None - esi_v4 = None - - _initializing = None - _instance = None - @classmethod - def initEsiApp(cls): - if cls._initializing is None: - cls._initializing = True - cls.esiapp = EsiApp(cache=file_cache, cache_time=None, cache_prefix='pyfa{0}-esipy-'.format(config.version)) - cls.esi_v1 = cls.esiapp.get_v1_swagger - cls.esi_v4 = cls.esiapp.get_v4_swagger - cls._initializing = False - - @classmethod - def genEsiClient(cls, security=None): - return EsiClient( - security=EsiSecurityProxy(sso_url=config.ESI_AUTH_PROXY) if security is None else security, - cache=file_cache, - headers={'User-Agent': 'pyfa esipy'} - ) - @classmethod def getInstance(cls): if cls._instance is None: @@ -93,18 +88,8 @@ class Esi(object): return cls._instance def __init__(self): - try: - Esi.initEsiApp() - except Exception as e: - # todo: this is a stop-gap for #1546. figure out a better way of handling esi service failing. - pyfalog.error(e) - wx.MessageBox("The ESI module failed to initialize. This can sometimes happen on first load on a slower connection. Please try again.") - return - self.settings = EsiSettings.getInstance() - AFTER_TOKEN_REFRESH.add_receiver(self.tokenUpdate) - # these will be set when needed self.httpd = None self.state = None @@ -112,17 +97,31 @@ class Esi(object): self.implicitCharacter = None - # The database cache does not seem to be working for some reason. Use - # this as a temporary measure - self.charCache = {} - # need these here to post events import gui.mainFrame # put this here to avoid loop self.mainFrame = gui.mainFrame.MainFrame.getInstance() - def tokenUpdate(self, **kwargs): - print(kwargs) - pass + if sso_url is None or sso_url == "": + raise AttributeError("sso_url cannot be None or empty " + "without app parameter") + + self.oauth_authorize = '%s/oauth/authorize' % sso_url + self.oauth_token = '%s/oauth/token' % sso_url + + # use ESI url for verify, since it's better for caching + if esi_url is None or esi_url == "": + raise AttributeError("esi_url cannot be None or empty") + self.oauth_verify = '%s/verify/' % esi_url + + + # session request stuff + self._session = Session() + self._session.headers.update({ + 'Accept': 'application/json', + 'User-Agent': ( + 'pyfa v{}'.format(config.version) + ) + }) def delSsoCharacter(self, id): char = eos.db.getSsoCharacter(id, config.getClientSecret()) @@ -151,46 +150,36 @@ class Esi(object): eos.db.commit() return char + def getSkills(self, id): char = self.getSsoCharacter(id) - op = Esi.esi_v4.op['get_characters_character_id_skills'](character_id=char.characterID) - resp = self.check_response(char.esi_client.request(op)) - return resp.data + resp = self.get(char, ESIEndpoints.CHAR_SKILLS, character_id=char.characterID) + # resp = self.check_response(char.esi_client.request(op)) + return resp.json() def getSecStatus(self, id): char = self.getSsoCharacter(id) - op = Esi.esi_v4.op['get_characters_character_id'](character_id=char.characterID) - resp = self.check_response(char.esi_client.request(op)) - return resp.data + resp = self.get(char, ESIEndpoints.CHAR, character_id=char.characterID) + return resp.json() def getFittings(self, id): char = self.getSsoCharacter(id) - op = Esi.esi_v1.op['get_characters_character_id_fittings'](character_id=char.characterID) - resp = self.check_response(char.esi_client.request(op)) - return resp.data + resp = self.get(char, ESIEndpoints.CHAR_FITTINGS, character_id=char.characterID) + return resp.json() def postFitting(self, id, json_str): # @todo: new fitting ID can be recovered from resp.data, char = self.getSsoCharacter(id) - op = Esi.esi_v1.op['post_characters_character_id_fittings']( - character_id=char.characterID, - fitting=json.loads(json_str) - ) - resp = self.check_response(char.esi_client.request(op)) - return resp.data + resp = self.post(char, ESIEndpoints.CHAR_FITTINGS, json_str, character_id=char.characterID) + return resp.json() def delFitting(self, id, fittingID): char = self.getSsoCharacter(id) - op = Esi.esi_v1.op['delete_characters_character_id_fittings_fitting_id']( - character_id=char.characterID, - fitting_id=fittingID - ) - resp = self.check_response(char.esi_client.request(op)) - return resp.data + self.delete(char, ESIEndpoints.CHAR_DEL_FIT, character_id=char.characterID, fitting_id=fittingID) def check_response(self, resp): - if resp.status >= 400: - raise EsiException(resp.status) + # if resp.status >= 400: + # raise EsiException(resp.status) return resp @staticmethod @@ -211,10 +200,6 @@ class Esi(object): if 'refresh_token' in tokenResponse: char.refreshToken = config.cipher.encrypt(tokenResponse['refresh_token'].encode()) - # remove, no longer need? - if char.esi_client is not None: - char.esi_client.security.update_token(tokenResponse) - def login(self): serverAddr = None if self.settings.get('loginMode') == LoginMethod.SERVER: @@ -263,31 +248,72 @@ class Esi(object): return 'http://localhost:{}'.format(port) + def get_oauth_header(self, token): + """ Return the Bearer Authorization header required in oauth calls + + :return: a dict with the authorization header + """ + return {'Authorization': 'Bearer %s' % token} + + def get_refresh_token_params(self, refreshToken): + """ Return the param object for the post() call to get the access_token + from the refresh_token + + :param code: the refresh token + :return: a dict with the url, params and header + """ + if refreshToken is None: + raise AttributeError('No refresh token is defined.') + + return { + 'data': { + 'grant_type': 'refresh_token', + 'refresh_token': refreshToken, + }, + 'url': self.oauth_token, + } + + def refresh(self, ssoChar): + request_data = self.get_refresh_token_params(config.cipher.decrypt(ssoChar.refreshToken).decode()) + res = self._session.post(**request_data) + if res.status_code != 200: + raise APIException( + request_data['url'], + res.status_code, + res.json() + ) + json_res = res.json() + self.update_token(ssoChar, json_res) + return json_res + def handleLogin(self, ssoInfo): auth_response = json.loads(base64.b64decode(ssoInfo)) - # We need to preload the ESI Security object beforehand with the auth response so that we can use verify to - # get character information - # init the security object - esisecurity = EsiSecurityProxy(sso_url=config.ESI_AUTH_PROXY) - - esisecurity.update_token(auth_response) - - # we get the character information - cdata = esisecurity.verify() + res = self._session.get( + self.oauth_verify, + headers=self.get_oauth_header(auth_response['access_token']) + ) + if res.status_code != 200: + raise APIException( + self.oauth_verify, + res.status_code, + res.json() + ) + cdata = res.json() print(cdata) currentCharacter = self.getSsoCharacter(cdata['CharacterName']) if currentCharacter is None: currentCharacter = SsoCharacter(cdata['CharacterID'], cdata['CharacterName'], config.getClientSecret()) - currentCharacter.esi_client = Esi.genEsiClient(esisecurity) - Esi.update_token(currentCharacter, auth_response) # this also sets the esi security token + Esi.update_token(currentCharacter, auth_response) eos.db.save(currentCharacter) wx.PostEvent(self.mainFrame, GE.SsoLogin(character=currentCharacter)) + # get (endpoint, char, data?) + def handleServerLogin(self, message): if not message: raise Exception("Could not parse out querystring parameters.") @@ -299,3 +325,34 @@ class Esi(object): pyfalog.debug("Handling SSO login with: {0}", message) self.handleLogin(message['SSOInfo'][0]) + + def __before_request(self, ssoChar): + if ssoChar.is_token_expired(): + json_response = self.refresh(ssoChar) + # AFTER_TOKEN_REFRESH.send(**json_response) + + if ssoChar.accessToken is not None: + self._session.headers.update(self.get_oauth_header(ssoChar.accessToken)) + + def get(self, ssoChar, endpoint, *args, **kwargs): + self.__before_request(ssoChar) + endpoint = endpoint.format(**kwargs) + return self._session.get("{}{}".format(esi_url, endpoint)) + + # check for warnings, also status > 400 + + + def post(self, ssoChar, endpoint, json, *args, **kwargs): + self.__before_request(ssoChar) + endpoint = endpoint.format(**kwargs) + return self._session.post("{}{}".format(esi_url, endpoint), data=json) + + # check for warnings, also status > 400 + + def delete(self, ssoChar, endpoint, *args, **kwargs): + self.__before_request(ssoChar) + endpoint = endpoint.format(**kwargs) + return self._session.delete("{}{}".format(esi_url, endpoint)) + + # check for warnings, also status > 400 + diff --git a/service/esi_security_proxy.py b/service/esi_security_proxy.py deleted file mode 100644 index 277eb3aa1..000000000 --- a/service/esi_security_proxy.py +++ /dev/null @@ -1,222 +0,0 @@ -# -*- encoding: utf-8 -*- -""" EsiPy Security Proxy - An ESI Security class that directs authentication towards a third-party service. -Client key/secret not needed. -""" - -from __future__ import absolute_import - -import base64 -import logging -import time - -from requests import Session -from requests.utils import quote -from six.moves.urllib.parse import urlparse -from urllib.parse import urlencode - -from esipy.events import AFTER_TOKEN_REFRESH -from esipy.exceptions import APIException -LOGGER = logging.getLogger(__name__) - - -class EsiSecurityProxy(object): - """ Contains all the OAuth2 knowledge for ESI use. - Based on pyswagger Security object, to be used with pyswagger BaseClient - implementation. - """ - - def __init__( - self, - **kwargs): - """ Init the ESI Security Object - - :param sso_url: the default sso URL used when no "app" is provided - :param esi_url: the default esi URL used for verify endpoint - :param app: (optionnal) the pyswagger app object - :param security_name: (optionnal) the name of the object holding the - informations in the securityDefinitions, used to check authed endpoint - """ - - app = kwargs.pop('app', None) - sso_url = kwargs.pop('sso_url', "https://login.eveonline.com") - esi_url = kwargs.pop('esi_url', "https://esi.tech.ccp.is") - - self.security_name = kwargs.pop('security_name', 'evesso') - - # we provide app object, so we don't use sso_url - if app is not None: - # check if the security_name exists in the securityDefinition - security = app.root.securityDefinitions.get( - self.security_name, - None - ) - if security is None: - raise NameError( - "%s is not defined in the securityDefinitions" % - self.security_name - ) - - self.oauth_authorize = security.authorizationUrl - - # some URL we still need to "manually" define... sadly - # we parse the authUrl so we don't care if it's TQ or SISI. - # https://github.com/ccpgames/esi-issues/issues/92 - parsed_uri = urlparse(security.authorizationUrl) - self.oauth_token = '%s://%s/oauth/token' % ( - parsed_uri.scheme, - parsed_uri.netloc - ) - - # no app object is provided, so we use direct URLs - else: - if sso_url is None or sso_url == "": - raise AttributeError("sso_url cannot be None or empty " - "without app parameter") - - self.oauth_authorize = '%s/oauth/authorize' % sso_url - self.oauth_token = '%s/oauth/token' % sso_url - - # use ESI url for verify, since it's better for caching - if esi_url is None or esi_url == "": - raise AttributeError("esi_url cannot be None or empty") - self.oauth_verify = '%s/verify/' % esi_url - - # session request stuff - self._session = Session() - self._session.headers.update({ - 'Accept': 'application/json', - 'User-Agent': ( - 'EsiPy/Security/ - ' - 'https://github.com/Kyria/EsiPy' - ) - }) - - # token data - self.refresh_token = None - self.access_token = None - self.token_expiry = None - - def __get_oauth_header(self): - """ Return the Bearer Authorization header required in oauth calls - - :return: a dict with the authorization header - """ - return {'Authorization': 'Bearer %s' % self.access_token} - - def __make_token_request_parameters(self, params): - """ Return the token uri from the securityDefinition - - :param params: the data given to the request - :return: the oauth/token uri - """ - request_params = { - 'data': params, - 'url': self.oauth_token, - } - - return request_params - - def get_refresh_token_params(self): - """ Return the param object for the post() call to get the access_token - from the refresh_token - - :param code: the refresh token - :return: a dict with the url, params and header - """ - if self.refresh_token is None: - raise AttributeError('No refresh token is defined.') - - return self.__make_token_request_parameters( - { - 'grant_type': 'refresh_token', - 'refresh_token': self.refresh_token, - } - ) - - def update_token(self, response_json): - """ Update access_token, refresh_token and token_expiry from the - response body. - The response must be converted to a json object before being passed as - a parameter - - :param response_json: the response body to use. - """ - self.access_token = response_json['access_token'] - self.token_expiry = int(time.time()) + response_json['expires_in'] - - if 'refresh_token' in response_json: - self.refresh_token = response_json['refresh_token'] - - def is_token_expired(self, offset=0): - """ Return true if the token is expired. - - The offset can be used to change the expiry time: - - positive value decrease the time (sooner) - - negative value increase the time (later) - If the expiry is not set, always return True. This case allow the users - to define a security object, only knowing the refresh_token and get - a new access_token / expiry_time without errors. - - :param offset: the expiry offset (in seconds) [default: 0] - :return: boolean true if expired, else false. - """ - if self.token_expiry is None: - return True - return int(time.time()) >= (self.token_expiry - offset) - - def refresh(self): - """ Update the auth data (tokens) using the refresh token in auth. - """ - request_data = self.get_refresh_token_params() - res = self._session.post(**request_data) - if res.status_code != 200: - raise APIException( - request_data['url'], - res.status_code, - res.json() - ) - json_res = res.json() - self.update_token(json_res) - return json_res - - def verify(self): - """ Make a get call to the oauth/verify endpoint to get the user data - - :return: the json with the data. - """ - res = self._session.get( - self.oauth_verify, - headers=self.__get_oauth_header() - ) - if res.status_code != 200: - raise APIException( - self.oauth_verify, - res.status_code, - res.json() - ) - return res.json() - - def __call__(self, request): - """ Check if the request need security header and apply them. - Required for pyswagger.core.BaseClient.request(). - - :param request: the pyswagger request object to check - :return: the updated request. - """ - if not request._security: - return request - - if self.is_token_expired(): - json_response = self.refresh() - AFTER_TOKEN_REFRESH.send(**json_response) - - for security in request._security: - if self.security_name not in security: - LOGGER.warning( - "Missing Securities: [%s]" % ", ".join(security.keys()) - ) - continue - if self.access_token is not None: - request._p['header'].update(self.__get_oauth_header()) - - return request From 5cc6b6c69cad60ac0675e95136ca0a33e3e764fc Mon Sep 17 00:00:00 2001 From: blitzmann Date: Sat, 12 May 2018 13:53:56 -0400 Subject: [PATCH 04/11] Move over all esi stuff to it's own class, which the esi service extends from. Also fix an issue in the EVE fittings browser where deleting a fit didn't actually remove it from the list of fits (due to that list being populated by the return data of ESI, which can be cached). More clean up. --- eos/saveddata/ssocharacter.py | 14 +-- gui/esiFittings.py | 20 ++-- service/esi.py | 211 +++------------------------------- service/esiAccess.py | 211 ++++++++++++++++++++++++++++++++++ 4 files changed, 239 insertions(+), 217 deletions(-) create mode 100644 service/esiAccess.py diff --git a/eos/saveddata/ssocharacter.py b/eos/saveddata/ssocharacter.py index 352e3724a..d1234c2c4 100644 --- a/eos/saveddata/ssocharacter.py +++ b/eos/saveddata/ssocharacter.py @@ -32,23 +32,11 @@ class SsoCharacter(object): self.accessToken = accessToken self.refreshToken = refreshToken self.accessTokenExpires = None - self.esi_client = None @reconstructor def init(self): - self.esi_client = None - - def get_sso_data(self): - """ Little "helper" function to get formated data for esipy security - """ - return { - 'access_token': self.accessToken, - 'refresh_token': self.refreshToken, - 'expires_in': ( - self.accessTokenExpires - datetime.datetime.utcnow() - ).total_seconds() - } + pass def is_token_expired(self): if self.accessTokenExpires is None: diff --git a/gui/esiFittings.py b/gui/esiFittings.py index 2f53a5819..0ac8e90db 100644 --- a/gui/esiFittings.py +++ b/gui/esiFittings.py @@ -1,5 +1,3 @@ -import time -import webbrowser import json # noinspection PyPackageRequirements import wx @@ -15,8 +13,8 @@ from gui.display import Display import gui.globalEvents as GE from logbook import Logger -import calendar -from service.esi import Esi, APIException +from service.esi import Esi +from service.esiAccess import APIException from service.port import ESIExportException pyfalog = Logger(__name__) @@ -110,11 +108,11 @@ class EveFittings(wx.Frame): waitDialog = wx.BusyInfo("Fetching fits, please wait...", parent=self) try: - fittings = sEsi.getFittings(self.getActiveCharacter()) + self.fittings = sEsi.getFittings(self.getActiveCharacter()) # self.cacheTime = fittings.get('cached_until') # self.updateCacheStatus(None) # self.cacheTimer.Start(1000) - self.fitTree.populateSkillTree(fittings) + self.fitTree.populateSkillTree(self.fittings) del waitDialog except requests.exceptions.ConnectionError: msg = "Connection error, please check your internet connection" @@ -149,6 +147,9 @@ class EveFittings(wx.Frame): if dlg.ShowModal() == wx.ID_YES: try: sEsi.delFitting(self.getActiveCharacter(), data['fitting_id']) + # repopulate the fitting list + self.fitTree.populateSkillTree(self.fittings) + self.fitView.update([]) except requests.exceptions.ConnectionError: msg = "Connection error, please check your internet connection" pyfalog.error(msg) @@ -156,8 +157,9 @@ class EveFittings(wx.Frame): class ESIExceptionHandler(object): + # todo: make this a generate excetpion handler for all calls def __init__(self, parentWindow, ex): - if ex.response['error'] == "invalid_token": + if ex.response['error'].startswith('Token is not valid'): dlg = wx.MessageDialog(parentWindow, "There was an error validating characters' SSO token. Please try " "logging into the character again to reset the token.", "Invalid Token", @@ -361,9 +363,13 @@ class FittingsTreeView(wx.Panel): tree = self.fittingsTreeCtrl tree.DeleteChildren(root) + sEsi = Esi.getInstance() + dict = {} fits = data for fit in fits: + if (fit['fitting_id'] in sEsi.fittings_deleted): + continue ship = getItem(fit['ship_type_id']) if ship.name not in dict: dict[ship.name] = [] diff --git a/service/esi.py b/service/esi.py index adbf7e774..78c80e04a 100644 --- a/service/esi.py +++ b/service/esi.py @@ -2,82 +2,33 @@ import wx from logbook import Logger import threading -import uuid import time -import config import base64 import json -import os import config import webbrowser import eos.db -import datetime from eos.enum import Enum from eos.saveddata.ssocharacter import SsoCharacter +from service.esiAccess import APIException import gui.globalEvents as GE from service.server import StoppableHTTPServer, AuthHandler from service.settings import EsiSettings +from service.esiAccess import EsiAccess import wx from requests import Session pyfalog = Logger(__name__) -from urllib.parse import urlencode - -# todo: reimplement Caching for calls -# from esipy.cache import FileCache -# file_cache = FileCache(cache_path) -# cache_path = os.path.join(config.savePath, config.ESI_CACHE) -# -# if not os.path.exists(cache_path): -# os.mkdir(cache_path) -# - -sso_url = "https://www.pyfa.io" # "https://login.eveonline.com" for actual login -esi_url = "https://esi.tech.ccp.is" - -oauth_authorize = '%s/oauth/authorize' % sso_url -oauth_token = '%s/oauth/token' % sso_url - - -class APIException(Exception): - """ Exception for SSO related errors """ - - def __init__(self, url, code, json_response): - self.url = url - self.status_code = code - self.response = json_response - super(APIException, self).__init__(str(self)) - - def __str__(self): - if 'error' in self.response: - return 'HTTP Error %s: %s' % (self.status_code, - self.response['error']) - elif 'message' in self.response: - return 'HTTP Error %s: %s' % (self.status_code, - self.response['message']) - return 'HTTP Error %s' % (self.status_code) - - -class ESIEndpoints(Enum): - CHAR = "/v4/characters/{character_id}/" - CHAR_SKILLS = "/v4/characters/{character_id}/skills/" # prepend https://esi.evetech.net/ - CHAR_FITTINGS = "/v1/characters/{character_id}/fittings/" - CHAR_DEL_FIT = "/v1/characters/{character_id}/fittings/{fitting_id}/" - -class Servers(Enum): - TQ = 0 - SISI = 1 - class LoginMethod(Enum): SERVER = 0 MANUAL = 1 -class Esi(object): +class Esi(EsiAccess): _instance = None @classmethod @@ -88,6 +39,7 @@ class Esi(object): return cls._instance def __init__(self): + super().__init__() self.settings = EsiSettings.getInstance() # these will be set when needed @@ -97,32 +49,14 @@ class Esi(object): self.implicitCharacter = None + # until I can get around to making proper caching and modifications to said cache, storee deleted fittings here + # so that we can easily hide them in the fitting browser + self.fittings_deleted = set() + # need these here to post events import gui.mainFrame # put this here to avoid loop self.mainFrame = gui.mainFrame.MainFrame.getInstance() - if sso_url is None or sso_url == "": - raise AttributeError("sso_url cannot be None or empty " - "without app parameter") - - self.oauth_authorize = '%s/oauth/authorize' % sso_url - self.oauth_token = '%s/oauth/token' % sso_url - - # use ESI url for verify, since it's better for caching - if esi_url is None or esi_url == "": - raise AttributeError("esi_url cannot be None or empty") - self.oauth_verify = '%s/verify/' % esi_url - - - # session request stuff - self._session = Session() - self._session.headers.update({ - 'Accept': 'application/json', - 'User-Agent': ( - 'pyfa v{}'.format(config.version) - ) - }) - def delSsoCharacter(self, id): char = eos.db.getSsoCharacter(id, config.getClientSecret()) @@ -139,66 +73,35 @@ class Esi(object): return chars def getSsoCharacter(self, id): - """ - Get character, and modify to include the eve connection - """ char = eos.db.getSsoCharacter(id, config.getClientSecret()) - if char is not None and char.esi_client is None: - char.esi_client = Esi.genEsiClient() - Esi.update_token(char, Esi.get_sso_data(char)) # don't use update_token on security directly, se still need to apply the values here - eos.db.commit() return char - def getSkills(self, id): char = self.getSsoCharacter(id) - resp = self.get(char, ESIEndpoints.CHAR_SKILLS, character_id=char.characterID) - # resp = self.check_response(char.esi_client.request(op)) + resp = super().getSkills(char) return resp.json() def getSecStatus(self, id): char = self.getSsoCharacter(id) - resp = self.get(char, ESIEndpoints.CHAR, character_id=char.characterID) + resp = super().getSecStatus(char) return resp.json() def getFittings(self, id): char = self.getSsoCharacter(id) - resp = self.get(char, ESIEndpoints.CHAR_FITTINGS, character_id=char.characterID) + resp = super().getFittings(char) return resp.json() def postFitting(self, id, json_str): # @todo: new fitting ID can be recovered from resp.data, char = self.getSsoCharacter(id) - resp = self.post(char, ESIEndpoints.CHAR_FITTINGS, json_str, character_id=char.characterID) + resp = super().postFitting(char, json_str) return resp.json() def delFitting(self, id, fittingID): char = self.getSsoCharacter(id) - self.delete(char, ESIEndpoints.CHAR_DEL_FIT, character_id=char.characterID, fitting_id=fittingID) - - def check_response(self, resp): - # if resp.status >= 400: - # raise EsiException(resp.status) - return resp - - @staticmethod - def get_sso_data(char): - """ Little "helper" function to get formated data for esipy security - """ - return { - 'access_token': char.accessToken, - 'refresh_token': config.cipher.decrypt(char.refreshToken).decode(), - 'expires_in': (char.accessTokenExpires - datetime.datetime.utcnow()).total_seconds() - } - - @staticmethod - def update_token(char, tokenResponse): - """ helper function to update token data from SSO response """ - char.accessToken = tokenResponse['access_token'] - char.accessTokenExpires = datetime.datetime.fromtimestamp(time.time() + tokenResponse['expires_in']) - if 'refresh_token' in tokenResponse: - char.refreshToken = config.cipher.encrypt(tokenResponse['refresh_token'].encode()) + super().delFitting(char, fittingID) + self.fittings_deleted.add(fittingID) def login(self): serverAddr = None @@ -213,24 +116,6 @@ class Esi(object): self.httpd.stop() self.httpd = None - def getLoginURI(self, redirect=None): - self.state = str(uuid.uuid4()) - - args = { - 'state': self.state, - 'pyfa_version': config.version, - 'login_method': self.settings.get('loginMode'), - 'client_hash': config.getClientSecret() - } - - if redirect is not None: - args['redirect'] = redirect - - return '%s?%s' % ( - oauth_authorize, - urlencode(args) - ) - def startServer(self): # todo: break this out into two functions: starting the server, and getting the URI pyfalog.debug("Starting server") @@ -248,44 +133,6 @@ class Esi(object): return 'http://localhost:{}'.format(port) - def get_oauth_header(self, token): - """ Return the Bearer Authorization header required in oauth calls - - :return: a dict with the authorization header - """ - return {'Authorization': 'Bearer %s' % token} - - def get_refresh_token_params(self, refreshToken): - """ Return the param object for the post() call to get the access_token - from the refresh_token - - :param code: the refresh token - :return: a dict with the url, params and header - """ - if refreshToken is None: - raise AttributeError('No refresh token is defined.') - - return { - 'data': { - 'grant_type': 'refresh_token', - 'refresh_token': refreshToken, - }, - 'url': self.oauth_token, - } - - def refresh(self, ssoChar): - request_data = self.get_refresh_token_params(config.cipher.decrypt(ssoChar.refreshToken).decode()) - res = self._session.post(**request_data) - if res.status_code != 200: - raise APIException( - request_data['url'], - res.status_code, - res.json() - ) - json_res = res.json() - self.update_token(ssoChar, json_res) - return json_res - def handleLogin(self, ssoInfo): auth_response = json.loads(base64.b64decode(ssoInfo)) @@ -326,33 +173,3 @@ class Esi(object): self.handleLogin(message['SSOInfo'][0]) - def __before_request(self, ssoChar): - if ssoChar.is_token_expired(): - json_response = self.refresh(ssoChar) - # AFTER_TOKEN_REFRESH.send(**json_response) - - if ssoChar.accessToken is not None: - self._session.headers.update(self.get_oauth_header(ssoChar.accessToken)) - - def get(self, ssoChar, endpoint, *args, **kwargs): - self.__before_request(ssoChar) - endpoint = endpoint.format(**kwargs) - return self._session.get("{}{}".format(esi_url, endpoint)) - - # check for warnings, also status > 400 - - - def post(self, ssoChar, endpoint, json, *args, **kwargs): - self.__before_request(ssoChar) - endpoint = endpoint.format(**kwargs) - return self._session.post("{}{}".format(esi_url, endpoint), data=json) - - # check for warnings, also status > 400 - - def delete(self, ssoChar, endpoint, *args, **kwargs): - self.__before_request(ssoChar) - endpoint = endpoint.format(**kwargs) - return self._session.delete("{}{}".format(esi_url, endpoint)) - - # check for warnings, also status > 400 - diff --git a/service/esiAccess.py b/service/esiAccess.py new file mode 100644 index 000000000..864ca397d --- /dev/null +++ b/service/esiAccess.py @@ -0,0 +1,211 @@ +# noinspection PyPackageRequirements +from logbook import Logger +import uuid +import time +import config + +import datetime +from eos.enum import Enum +from eos.saveddata.ssocharacter import SsoCharacter +from service.settings import EsiSettings + +from requests import Session +from urllib.parse import urlencode + +pyfalog = Logger(__name__) + +# todo: reimplement Caching for calls +# from esipy.cache import FileCache +# file_cache = FileCache(cache_path) +# cache_path = os.path.join(config.savePath, config.ESI_CACHE) +# +# if not os.path.exists(cache_path): +# os.mkdir(cache_path) +# + +# todo: move these over to getters that automatically determine which endpoint we use. +sso_url = "https://www.pyfa.io" # "https://login.eveonline.com" for actual login +esi_url = "https://esi.tech.ccp.is" + +oauth_authorize = '%s/oauth/authorize' % sso_url +oauth_token = '%s/oauth/token' % sso_url + + +class APIException(Exception): + """ Exception for SSO related errors """ + + def __init__(self, url, code, json_response): + self.url = url + self.status_code = code + self.response = json_response + super(APIException, self).__init__(str(self)) + + def __str__(self): + if 'error' in self.response: + return 'HTTP Error %s: %s' % (self.status_code, + self.response['error']) + elif 'message' in self.response: + return 'HTTP Error %s: %s' % (self.status_code, + self.response['message']) + return 'HTTP Error %s' % (self.status_code) + + +class ESIEndpoints(Enum): + CHAR = "/v4/characters/{character_id}/" + CHAR_SKILLS = "/v4/characters/{character_id}/skills/" + CHAR_FITTINGS = "/v1/characters/{character_id}/fittings/" + CHAR_DEL_FIT = "/v1/characters/{character_id}/fittings/{fitting_id}/" + + +# class Servers(Enum): +# TQ = 0 +# SISI = 1 + +class EsiAccess(object): + def __init__(self): + if sso_url is None or sso_url == "": + raise AttributeError("sso_url cannot be None or empty " + "without app parameter") + + self.settings = EsiSettings.getInstance() + + self.oauth_authorize = '%s/oauth/authorize' % sso_url + self.oauth_token = '%s/oauth/token' % sso_url + + # use ESI url for verify, since it's better for caching + if esi_url is None or esi_url == "": + raise AttributeError("esi_url cannot be None or empty") + self.oauth_verify = '%s/verify/' % esi_url + + + # session request stuff + self._session = Session() + self._session.headers.update({ + 'Accept': 'application/json', + 'User-Agent': ( + 'pyfa v{}'.format(config.version) + ) + }) + + def getSkills(self, char): + return self.get(char, ESIEndpoints.CHAR_SKILLS, character_id=char.characterID) + + def getSecStatus(self, char): + return self.get(char, ESIEndpoints.CHAR, character_id=char.characterID) + + def getFittings(self, char): + return self.get(char, ESIEndpoints.CHAR_FITTINGS, character_id=char.characterID) + + def postFitting(self, char, json_str): + # @todo: new fitting ID can be recovered from resp.data, + return self.post(char, ESIEndpoints.CHAR_FITTINGS, json_str, character_id=char.characterID) + + def delFitting(self, char, fittingID): + return self.delete(char, ESIEndpoints.CHAR_DEL_FIT, character_id=char.characterID, fitting_id=fittingID) + + @staticmethod + def update_token(char, tokenResponse): + """ helper function to update token data from SSO response """ + char.accessToken = tokenResponse['access_token'] + char.accessTokenExpires = datetime.datetime.fromtimestamp(time.time() + tokenResponse['expires_in']) + if 'refresh_token' in tokenResponse: + char.refreshToken = config.cipher.encrypt(tokenResponse['refresh_token'].encode()) + + def getLoginURI(self, redirect=None): + self.state = str(uuid.uuid4()) + + args = { + 'state': self.state, + 'pyfa_version': config.version, + 'login_method': self.settings.get('loginMode'), + 'client_hash': config.getClientSecret() + } + + if redirect is not None: + args['redirect'] = redirect + + return '%s?%s' % ( + oauth_authorize, + urlencode(args) + ) + + def get_oauth_header(self, token): + """ Return the Bearer Authorization header required in oauth calls + + :return: a dict with the authorization header + """ + return {'Authorization': 'Bearer %s' % token} + + def get_refresh_token_params(self, refreshToken): + """ Return the param object for the post() call to get the access_token + from the refresh_token + + :param code: the refresh token + :return: a dict with the url, params and header + """ + if refreshToken is None: + raise AttributeError('No refresh token is defined.') + + return { + 'data': { + 'grant_type': 'refresh_token', + 'refresh_token': refreshToken, + }, + 'url': self.oauth_token, + } + + def refresh(self, ssoChar): + request_data = self.get_refresh_token_params(config.cipher.decrypt(ssoChar.refreshToken).decode()) + res = self._session.post(**request_data) + if res.status_code != 200: + raise APIException( + request_data['url'], + res.status_code, + res.json() + ) + json_res = res.json() + self.update_token(ssoChar, json_res) + return json_res + + def _before_request(self, ssoChar): + if ssoChar.is_token_expired(): + pyfalog.info("Refreshing token for {}".format(ssoChar.characterName)) + self.refresh(ssoChar) + + if ssoChar.accessToken is not None: + self._session.headers.update(self.get_oauth_header(ssoChar.accessToken)) + + def _after_request(self, resp): + if ("warning" in resp.headers): + pyfalog.warn("{} - {}".format(resp.headers["warning"], resp.url)) + + if resp.status_code >= 400: + raise APIException( + resp.url, + resp.status_code, + resp.json() + ) + + return resp + + def get(self, ssoChar, endpoint, *args, **kwargs): + self._before_request(ssoChar) + endpoint = endpoint.format(**kwargs) + return self._after_request(self._session.get("{}{}".format(esi_url, endpoint))) + + # check for warnings, also status > 400 + + def post(self, ssoChar, endpoint, json, *args, **kwargs): + self._before_request(ssoChar) + endpoint = endpoint.format(**kwargs) + return self._after_request(self._session.post("{}{}".format(esi_url, endpoint), data=json)) + + # check for warnings, also status > 400 + + def delete(self, ssoChar, endpoint, *args, **kwargs): + self._before_request(ssoChar) + endpoint = endpoint.format(**kwargs) + return self._after_request(self._session.delete("{}{}".format(esi_url, endpoint))) + + # check for warnings, also status > 400 + From aa2ffaf1ea1b720aaecf29a609b02ca580cdfb51 Mon Sep 17 00:00:00 2001 From: blitzmann Date: Sat, 12 May 2018 22:49:18 -0400 Subject: [PATCH 05/11] Update installer script and create new dist script for windows. todo: find out hwo to invoke PyInstaller from python --- dist_assets/win/dist.py | 46 ++++ {scripts => dist_assets/win}/pyfa-setup.iss | 13 +- dist_assets/win/pyfa.spec | 3 +- scripts/dist.py | 257 -------------------- 4 files changed, 57 insertions(+), 262 deletions(-) create mode 100644 dist_assets/win/dist.py rename {scripts => dist_assets/win}/pyfa-setup.iss (91%) delete mode 100755 scripts/dist.py diff --git a/dist_assets/win/dist.py b/dist_assets/win/dist.py new file mode 100644 index 000000000..a34ed3140 --- /dev/null +++ b/dist_assets/win/dist.py @@ -0,0 +1,46 @@ +# helper script to zip up pyinstaller distribution and create installer file + +import os.path +from subprocess import call +import zipfile + + +def zipdir(path, zip): + for root, dirs, files in os.walk(path): + for file in files: + zip.write(os.path.join(root, file)) + +config = {} + +exec(compile(open("config.py").read(), "config.py", 'exec'), config) + +iscc = "C:\Program Files (x86)\Inno Setup 5\ISCC.exe" # inno script location via wine + +print("Creating archive") + +source = os.path.join(os.getcwd(), "dist", "pyfa") + +fileName = "pyfa-{}-{}-{}-win".format( + config['version'], + config['expansionName'].lower(), + config['expansionVersion'] +) + +archive = zipfile.ZipFile(os.path.join(os.getcwd(), "dist", fileName + ".zip"), 'w', compression=zipfile.ZIP_DEFLATED) +zipdir(source, archive) +archive.close() + +print("Compiling EXE") + +expansion = "%s %s" % (config['expansionName'], config['expansionVersion']), + +call([ + iscc, + os.path.join(os.getcwd(), "dist_assets", "win", "pyfa-setup.iss"), + "/dMyAppVersion=%s" % (config['version']), + "/dMyAppExpansion=%s" % (expansion), + "/dMyAppDir=%s" % source, + "/dMyOutputDir=%s" % os.path.join(os.getcwd(), "dist"), + "/dMyOutputFile=%s" % fileName]) # stdout=devnull, stderr=devnull + +print("Done") diff --git a/scripts/pyfa-setup.iss b/dist_assets/win/pyfa-setup.iss similarity index 91% rename from scripts/pyfa-setup.iss rename to dist_assets/win/pyfa-setup.iss index 9c25f11f2..3d3e3df2d 100644 --- a/scripts/pyfa-setup.iss +++ b/dist_assets/win/pyfa-setup.iss @@ -19,7 +19,8 @@ #define MyAppExeName "pyfa.exe" ; What version starts with the new structure (1.x.0). This is used to determine if we run directory structure cleanup -#define VersionFlag 16 +#define MajorVersionFlag 2 +#define MinorVersionFlag 0 #ifndef MyOutputFile #define MyOutputFile LowerCase(StringChange(MyAppName+'-'+MyAppVersion+'-'+MyAppExpansion+'-win-wx3', " ", "-")) @@ -138,15 +139,19 @@ var V: Integer; iResultCode: Integer; sUnInstallString: string; - iOldVersion: Cardinal; + iOldVersionMajor: Cardinal; + iOldVersionMinor: Cardinal; begin Result := True; // in case when no previous version is found if RegValueExists(HKEY_LOCAL_MACHINE,'Software\Microsoft\Windows\CurrentVersion\Uninstall\{3DA39096-C08D-49CD-90E0-1D177F32C8AA}_is1', 'UninstallString') then //Your App GUID/ID begin RegQueryDWordValue(HKEY_LOCAL_MACHINE, 'Software\Microsoft\Windows\CurrentVersion\Uninstall\{3DA39096-C08D-49CD-90E0-1D177F32C8AA}_is1', - 'MinorVersion', iOldVersion); - if iOldVersion < {#VersionFlag} then // If old version with old structure is installed. + 'MajorVersion', iOldVersionMajor); + RegQueryDWordValue(HKEY_LOCAL_MACHINE, + 'Software\Microsoft\Windows\CurrentVersion\Uninstall\{3DA39096-C08D-49CD-90E0-1D177F32C8AA}_is1', + 'MinorVersion', iOldVersionMinor); + if (iOldVersionMajor < {#MajorVersionFlag}) or ((iOldVersionMajor = {#MajorVersionFlag}) and (iOldVersionMinor < {#MinorVersionFlag})) then // If old version with old structure is installed. begin V := MsgBox(ExpandConstant('An old version of pyfa was detected. Due to recent changes in the application structure, you must uninstall the previous version first. This will not affect your user data (saved fittings, characters, etc.). Do you want to uninstall now?'), mbInformation, MB_YESNO); //Custom Message if App installed if V = IDYES then diff --git a/dist_assets/win/pyfa.spec b/dist_assets/win/pyfa.spec index 70eb4673d..445f94824 100644 --- a/dist_assets/win/pyfa.spec +++ b/dist_assets/win/pyfa.spec @@ -79,4 +79,5 @@ coll = COLLECT( upx=True, name='pyfa', icon='dist_assets/win/pyfa.ico', - ) \ No newline at end of file + ) + diff --git a/scripts/dist.py b/scripts/dist.py deleted file mode 100755 index 07b4ec473..000000000 --- a/scripts/dist.py +++ /dev/null @@ -1,257 +0,0 @@ -#!/usr/bin/env python -""" -Script for generating distributables based on platform skeletons. - -User supplies path for pyfa code base, root skeleton directory, and where the -builds go. The builds are automatically named depending on the pyfa config -values of `version` and `tag`. If it's a Stable release, the naming -convention is: - - pyfa-pyfaversion-expansion-expversion-platform - -If it is not Stable (tag=git), we determine if the pyfa code base includes -the git repo to use as an ID. If not, uses randomly generated 6-character ID. -The unstable naming convention: - - pyfa-YYYMMDD-id-platform - -dist.py can also build the Windows installer provided that it has a path to -Inno Setup (and, for generating on non-Windows platforms, that WINE is -installed). To build the EXE file, `win` must be included in the platforms to -be built. -""" - -#@todo: ensure build directory can be written to -# todo: default build and dist directories - -from optparse import OptionParser -import os.path -import shutil -import sys -import tarfile -import datetime -import random -import string -import zipfile -import errno -from subprocess import call - -class FileStub(): - def write(self, *args): - pass - - def flush(self, *args): - pass - -i = 0 -def loginfo(path, names): - # Print out a "progress" and return directories / files to ignore - global i - i += 1 - if i % 10 == 0: - sys.stdout.write(".") - sys.stdout.flush() - return () - -def copyanything(src, dst): - try: - shutil.copytree(src, dst, ignore=loginfo) - except: # python >2.5 - try: - shutil.copy(src, dst) - except: - raise - -def id_generator(size=6, chars=string.ascii_uppercase + string.digits): - return ''.join(random.choice(chars) for x in range(size)) - -def zipdir(path, zip): - for root, dirs, files in os.walk(path): - for file in files: - zip.write(os.path.join(root, file)) - -skels = ['win', 'src', 'mac', 'mac-deprecated'] -iscc = "C:\Program Files (x86)\Inno Setup 5\ISCC.exe" # inno script location via wine - -if __name__ == "__main__": - oldstd = sys.stdout - parser = OptionParser() - parser.add_option("-s", "--skeleton", dest="skeleton", help="Location of Pyfa-skel directory") - parser.add_option("-b", "--base", dest="base", help="Location of cleaned read-only base directory") - parser.add_option("-d", "--destination", dest="destination", help="Where to copy our distributable") - parser.add_option("-p", "--platforms", dest="platforms", help="Comma-separated list of platforms to build", default=','.join(skels)) - parser.add_option("-q", "--quiet", dest="silent", action="store_true") - parser.add_option("-w", "--winexe", dest="winexe", action="store_true", help="Build the Windows installer file (needs Inno Setup). Must include 'win' in platform options") - parser.add_option("-z", "--zip", dest="zip", action="store_true", help="zip archive instead of tar") - - options, args = parser.parse_args() - - if options.skeleton is None or options.base is None or options.destination is None: - print("Need --skeleton argument as well as --base and --destination argument") - parser.print_help() - sys.exit() - - if options.silent: - sys.stdout = FileStub() - - options.platforms = options.platforms.split(",") - - for skel in skels: - if skel not in options.platforms: - continue - - print("\n======== %s ========"%skel) - - info = {} - config = {} - setup = {} - skeleton = os.path.expanduser(os.path.join(options.skeleton, skel)) - - exec(compile(open(os.path.join(options.base, "config.py")).read(), os.path.join(options.base, "config.py"), 'exec'), config) - exec(compile(open(os.path.join(skeleton, "info.py")).read(), os.path.join(skeleton, "info.py"), 'exec'), info) - exec(compile(open(os.path.join(options.base, "setup.py")).read(), os.path.join(options.base, "setup.py"), 'exec'), setup) - - destination = os.path.expanduser(options.destination) - if not os.path.isdir(destination) or not os.access(destination, os.W_OK | os.X_OK): - print("Destination directory does not exist or is not writable: {}".format(destination)) - sys.exit() - - dirName = info["arcname"] - - nowdt = datetime.datetime.now() - now = "%04d%02d%02d" % (nowdt.year, nowdt.month, nowdt.day) - - git = False - if config['tag'].lower() == "git": - try: # if there is a git repo associated with base, use master commit - with open(os.path.join(options.base, ".git", "refs", "heads", "master"), 'r') as f: - id = f.readline()[0:6] - git = True - except: # else, use custom ID - id = id_generator() - fileName = "pyfa-{}-{}-{}".format(now, id, info["os"]) - else: - fileName = "pyfa-{}-{}-{}-{}".format( - config['version'], - config['expansionName'].lower(), - config['expansionVersion'], - info["os"] - ) - - archiveName = "{}.{}".format(fileName, "zip" if options.zip else "tar.bz2") - tmpDir = os.path.join(os.getcwd(), dirName) # tmp directory where files are copied - tmpFile = os.path.join(os.getcwd(), archiveName) - - try: - print("Copying skeleton to ", tmpDir) - shutil.copytree(skeleton, tmpDir, ignore=loginfo) - print() - source = os.path.expanduser(options.base) - root = os.path.join(tmpDir, info["base"]) - - # it is easier to work from the source directory - oldcwd = os.getcwd() - os.chdir(source) - - if info["library"]: - print("Injecting files into", info["library"]) - libraryFile = os.path.join(root, info["library"]) - - with zipfile.ZipFile(libraryFile, 'a') as library: - for dir in setup['packages']: - zipdir(dir, library) - library.write('pyfa.py', 'pyfa__main__.py') - library.write('config.py') - else: # platforms where we don't have a packaged library - print("Copying modules into", root) - for dir in setup['packages']: - copyanything(dir, os.path.join(root, dir)) - - # add some additional files to root dir for these platforms - # (hopefully can figure out a way later for OS X to use the one in - # it's library) - if skel == 'mac': - setup['include_files'] += ['pyfa.py'] - if skel in ('src', 'mac-deprecated'): - setup['include_files'] += ['pyfa.py', 'config.py'] - - print() - print("Copying included files:", end=' ') - - for file in setup['include_files']: - if isinstance(file, str): - print(file, end=' ') - copyanything(file, os.path.join(root, file)) - - print() - print("Creating images zipfile:", end=' ') - os.chdir('imgs') - imagesFile = os.path.join(root, "imgs.zip") - - with zipfile.ZipFile(imagesFile, 'w') as images: - for dir in setup['icon_dirs']: - print(dir, end=' ') - zipdir(dir, images) - os.chdir(oldcwd) - - print() - print("Creating archive") - if options.zip: - archive = zipfile.ZipFile(tmpFile, 'w', compression=zipfile.ZIP_DEFLATED) - zipdir(dirName, archive) - archive.close() - else: - archive = tarfile.open(tmpFile, "w:bz2") - archive.add(tmpDir, arcname=info["arcname"]) - archive.close() - - print("Moving archive to ", destination) - shutil.move(tmpFile, destination) - - if "win" in skel and options.winexe: - print("Compiling EXE") - - if config['tag'].lower() == "git": - if git: # if git repo info available, use git commit - expansion = "git-%s"%(id) - else: # if there is no git repo, use timestamp - expansion = now - else: # if code is Stable, use expansion name - expansion = "%s %s"%(config['expansionName'], config['expansionVersion']), - - calllist = ["wine"] if 'win' not in sys.platform else [] - - call(calllist + [ - iscc, - "pyfa-setup.iss", - "/dMyAppVersion=%s"%(config['version']), - "/dMyAppExpansion=%s"%(expansion), - "/dMyAppDir=pyfa", - "/dMyOutputDir=%s"%destination, - "/dMyOutputFile=%s"%fileName]) #stdout=devnull, stderr=devnull - - print("EXE completed") - - except Exception as e: - print("Encountered an error: \n\t", e) - raise - finally: - print("Deleting tmp files\n") - try: - shutil.rmtree("dist") # Inno dir - except: - pass - try: - shutil.rmtree(tmpDir) - except: - pass - try: - os.unlink(tmpFile) - except: - pass - - sys.stdout = oldstd - if os.path.isdir(destination): - print(os.path.join(destination, os.path.split(tmpFile)[1])) - else: - print(destination) From 86576581cd7f580c18737bebf45128dbf64fca9d Mon Sep 17 00:00:00 2001 From: blitzmann Date: Sun, 13 May 2018 01:17:18 -0400 Subject: [PATCH 06/11] Break out the phobos dump from the data compilation (dump has to work in py2, whereas compilation requires py3 due to hooking directly in with EOS classes). protip: PyCharm configurations are awesome --- scripts/compile_data.py | 63 +++++++++++++++++++++++ scripts/dump_data.py | 81 +++++++++++++++++++++++++++++ scripts/jsonToSql.py | 9 ++-- scripts/prep_data.py | 109 ---------------------------------------- 4 files changed, 150 insertions(+), 112 deletions(-) create mode 100644 scripts/compile_data.py create mode 100644 scripts/dump_data.py delete mode 100644 scripts/prep_data.py diff --git a/scripts/compile_data.py b/scripts/compile_data.py new file mode 100644 index 000000000..9de7c5562 --- /dev/null +++ b/scripts/compile_data.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +""" +This script bootstraps Phobos from a supplied path and feeds it +information regarding EVE data paths and where to dump data. It then imports +some other scripts and uses them to convert the json data into a SQLite +database and then compare the new database to the existing one, producing a +diff which can then be used to assist in the updating. +""" + +import sys +import os + +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument("-d", "--dump", dest="dump_path", help="Location of Phobos JSON dump directory", required=True) + +args = parser.parse_args() +dump_path = os.path.expanduser(args.dump_path) +script_path = os.path.dirname(__file__) + +def header(text, subtext=None): + print() + print("* "*30) + print(text.center(60)) + if subtext: + print(subtext.center(60)) + print("* "*30) + print() + +### SQL Convert +import jsonToSql + +db_file = os.path.join(dump_path, "eve.db") +header("Converting Data to SQL", db_file) + +if os.path.isfile(db_file): + os.remove(db_file) + +jsonToSql.main("sqlite:///" + db_file, dump_path) + +### Diff generation +import itemDiff + +diff_file = os.path.join(dump_path, "diff.txt") +old_db = os.path.join(script_path, "..", "eve.db") + +header("Generating DIFF", diff_file) +old_stdout = sys.stdout +sys.stdout = open(diff_file, 'w') +itemDiff.main(old=old_db, new=db_file) +sys.stdout = old_stdout + +header("Commiting changes for ", diff_file) + +from subprocess import call + +os.chdir(dump_path) + +call(["git.exe", "add", "."]) +call(["git.exe", "commit", "-m", "Commit"]) + +print("\nAll done.") \ No newline at end of file diff --git a/scripts/dump_data.py b/scripts/dump_data.py new file mode 100644 index 000000000..2393fb8c4 --- /dev/null +++ b/scripts/dump_data.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +""" +This script bootstraps Phobos from a supplied path and feeds it +information regarding EVE data paths and where to dump data. It then imports +some other scripts and uses them to convert the json data into a SQLite +database and then compare the new database to the existing one, producing a +diff which can then be used to assist in the updating. +""" + +import sys +import os + +# Phobos location +phb_path = os.path.expanduser("path/to/phobos") + +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument("-e", "--eve", dest="eve_path", help="Location of EVE directory", required=True) +parser.add_argument("-c", "--cache", dest="cache_path", help="Location of EVE cache directory. If not specified, an attempt will be make to automatically determine path.") +parser.add_argument("-r", "--res", dest="res_path", help="Location of EVE shared resource cache. If not specified, an attempt will be make to automatically determine path.") +parser.add_argument("-d", "--dump", dest="dump_path", help="Location of Phobos JSON dump directory", required=True) +parser.add_argument("-p", "--phobos", dest="phb_path", help="Location of Phobos, defaults to path noted in script", default=phb_path) +parser.add_argument("-s", "--singularity", action="store_true", help="Singularity build") + +args = parser.parse_args() +eve_path = os.path.expanduser(args.eve_path) +cache_path = os.path.expanduser(args.cache_path) if args.cache_path else None +res_path = os.path.expanduser(args.res_path) if args.res_path else None +dump_path = os.path.expanduser(args.dump_path) +script_path = os.path.dirname(__file__) + +### Append Phobos to path +sys.path.append(os.path.expanduser(args.phb_path)) + +def header(text, subtext=None): + print() + print("* "*30) + print(text.center(60)) + if subtext: + print(subtext.center(60)) + print("* "*30) + print() + +header("Dumping Phobos Data", dump_path) + +import reverence +from flow import FlowManager +from miner import * +from translator import Translator +from writer import * + +rvr = reverence.blue.EVE(eve_path, cachepath=args.cache_path, sharedcachepath=res_path, server="singularity" if args.singularity else "tranquility") +print("EVE Directory: {}".format(rvr.paths.root)) +print("Cache Directory: {}".format(rvr.paths.cache)) +print("Shared Resource Directory: {}".format(rvr.paths.sharedcache)) + +pickle_miner = ResourcePickleMiner(rvr) +trans = Translator(pickle_miner) +bulkdata_miner = BulkdataMiner(rvr, trans) +staticcache_miner = ResourceStaticCacheMiner(rvr, trans) +miners = ( + MetadataMiner(eve_path), + bulkdata_miner, + staticcache_miner, + TraitMiner(staticcache_miner, bulkdata_miner, trans), + SqliteMiner(rvr.paths.root, trans), + CachedCallsMiner(rvr, trans), + pickle_miner +) + +writers = ( + JsonWriter(dump_path, indent=2), +) + +list = "dgmexpressions,dgmattribs,dgmeffects,dgmtypeattribs,dgmtypeeffects,"\ + "dgmunits,invcategories,invgroups,invmetagroups,invmetatypes,"\ + "invtypes,mapbulk_marketGroups,phbmetadata,phbtraits,fsdTypeOverrides,"\ + "evegroups,evetypes,evecategories,mapbulk_marketGroups,clonegrades" + +FlowManager(miners, writers).run(list, "en-us") diff --git a/scripts/jsonToSql.py b/scripts/jsonToSql.py index 6629c62b2..717713e64 100755 --- a/scripts/jsonToSql.py +++ b/scripts/jsonToSql.py @@ -20,16 +20,19 @@ import os import sys +import functools import re # Add eos root path to sys.path so we can import ourselves -path = os.path.dirname(str(__file__, sys.getfilesystemencoding())) +path = os.path.dirname(__file__) sys.path.append(os.path.realpath(os.path.join(path, ".."))) import json import argparse def main(db, json_path): + if os.path.isfile(db): + os.remove(db) jsonPath = os.path.expanduser(json_path) @@ -130,7 +133,7 @@ def main(db, json_path): check[ID] = {} check[ID][int(skill["typeID"])] = int(skill["level"]) - if not reduce(lambda a, b: a if a == b else False, [v for _, v in check.iteritems()]): + if not functools.reduce(lambda a, b: a if a == b else False, [v for _, v in check.items()]): raise Exception("Alpha Clones not all equal") newData = [x for x in newData if x['alphaCloneID'] == 1] @@ -188,7 +191,7 @@ def main(db, json_path): # Dump all data to memory so we can easely cross check ignored rows for jsonName, cls in tables.items(): - with open(os.path.join(jsonPath, "{}.json".format(jsonName))) as f: + with open(os.path.join(jsonPath, "{}.json".format(jsonName)), encoding="utf-8") as f: tableData = json.load(f) if jsonName in rowsInValues: tableData = list(tableData.values()) diff --git a/scripts/prep_data.py b/scripts/prep_data.py deleted file mode 100644 index d8907aa58..000000000 --- a/scripts/prep_data.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python -""" -This script bootstraps Phobos from a supplied path and feeds it -information regarding EVE data paths and where to dump data. It then imports -some other scripts and uses them to convert the json data into a SQLite -database and then compare the new database to the existing one, producing a -diff which can then be used to assist in the updating. -""" - -import sys -import os - -# Phobos location -phb_path = os.path.expanduser("path/to/phobos") - -import argparse - -parser = argparse.ArgumentParser() -parser.add_argument("-e", "--eve", dest="eve_path", help="Location of EVE directory", required=True) -parser.add_argument("-c", "--cache", dest="cache_path", help="Location of EVE cache directory. If not specified, an attempt will be make to automatically determine path.") -parser.add_argument("-r", "--res", dest="res_path", help="Location of EVE shared resource cache. If not specified, an attempt will be make to automatically determine path.") -parser.add_argument("-d", "--dump", dest="dump_path", help="Location of Phobos JSON dump directory", required=True) -parser.add_argument("-p", "--phobos", dest="phb_path", help="Location of Phobos, defaults to path noted in script", default=phb_path) -parser.add_argument("-s", "--singularity", action="store_true", help="Singularity build") -parser.add_argument("-j", "--nojson", dest="nojson", action="store_true", help="Skip Phobos JSON data dump.") - -args = parser.parse_args() -eve_path = os.path.expanduser(str(args.eve_path, sys.getfilesystemencoding())) -cache_path = os.path.expanduser(str(args.cache_path, sys.getfilesystemencoding())) if args.cache_path else None -res_path = os.path.expanduser(str(args.res_path, sys.getfilesystemencoding())) if args.res_path else None -dump_path = os.path.expanduser(str(args.dump_path, sys.getfilesystemencoding())) -script_path = os.path.dirname(str(__file__, sys.getfilesystemencoding())) - -### Append Phobos to path -sys.path.append(os.path.expanduser(str(args.phb_path, sys.getfilesystemencoding()))) - -def header(text, subtext=None): - print() - print("* "*30) - print(text.center(60)) - if subtext: - print(subtext.center(60)) - print("* "*30) - print() - -### Data dump -if not args.nojson: - header("Dumping Phobos Data", dump_path) - - import reverence - from flow import FlowManager - from miner import * - from translator import Translator - from writer import * - - rvr = reverence.blue.EVE(eve_path, cachepath=args.cache_path, sharedcachepath=res_path, server="singularity" if args.singularity else "tranquility") - print("EVE Directory: {}".format(rvr.paths.root)) - print("Cache Directory: {}".format(rvr.paths.cache)) - print("Shared Resource Directory: {}".format(rvr.paths.sharedcache)) - print() - - pickle_miner = ResourcePickleMiner(rvr) - trans = Translator(pickle_miner) - bulkdata_miner = BulkdataMiner(rvr, trans) - staticcache_miner = ResourceStaticCacheMiner(rvr, trans) - miners = ( - MetadataMiner(eve_path), - bulkdata_miner, - staticcache_miner, - TraitMiner(staticcache_miner, bulkdata_miner, trans), - SqliteMiner(rvr.paths.root, trans), - CachedCallsMiner(rvr, trans), - pickle_miner - ) - - writers = ( - JsonWriter(dump_path, indent=2), - ) - - list = "dgmexpressions,dgmattribs,dgmeffects,dgmtypeattribs,dgmtypeeffects,"\ - "dgmunits,invcategories,invgroups,invmetagroups,invmetatypes,"\ - "invtypes,mapbulk_marketGroups,phbmetadata,phbtraits,fsdTypeOverrides,"\ - "evegroups,evetypes,evecategories,mapbulk_marketGroups,clonegrades" - - FlowManager(miners, writers).run(list, "en-us") - -### SQL Convert -import jsonToSql - -db_file = os.path.join(dump_path, "eve.db") -header("Converting Data to SQL", db_file) - -if os.path.isfile(db_file): - os.remove(db_file) - -jsonToSql.main("sqlite:///"+db_file, dump_path) - -### Diff generation -import itemDiff -diff_file = os.path.join(dump_path, "diff.txt") -old_db = os.path.join(script_path, "..", "eve.db") - -header("Generating DIFF", diff_file) -old_stdout = sys.stdout -sys.stdout = open(diff_file, 'w') -itemDiff.main(old=old_db, new=db_file) -sys.stdout = old_stdout - -print("\nAll done.") From e29ab817af530ae2be2ca88ad6cfded37396a843 Mon Sep 17 00:00:00 2001 From: blitzmann Date: Sun, 13 May 2018 13:27:49 -0400 Subject: [PATCH 07/11] Add the client details back to the preferences page --- .../pyfaEsiPreferences.py | 86 ++++++++++++++++--- service/settings.py | 10 ++- 2 files changed, 85 insertions(+), 11 deletions(-) diff --git a/gui/builtinPreferenceViews/pyfaEsiPreferences.py b/gui/builtinPreferenceViews/pyfaEsiPreferences.py index 029003a83..ef81b7480 100644 --- a/gui/builtinPreferenceViews/pyfaEsiPreferences.py +++ b/gui/builtinPreferenceViews/pyfaEsiPreferences.py @@ -45,20 +45,68 @@ class PFEsiPref(PreferenceView): ['Local Server', 'Manual'], 1, wx.RA_SPECIFY_COLS) self.rbMode.SetItemToolTip(0, "This options starts a local webserver that the web application will call back to with information about the character login.") self.rbMode.SetItemToolTip(1, "This option prompts users to copy and paste information from the web application to allow for character login. Use this if having issues with the local server.") - # self.rbServer = wx.RadioBox(panel, -1, "Server", wx.DefaultPosition, wx.DefaultSize, - # ['Tranquility', 'Singularity'], 1, wx.RA_SPECIFY_COLS) + + self.rbSsoMode = wx.RadioBox(panel, -1, "SSO Mode", wx.DefaultPosition, wx.DefaultSize, + ['pyfa.io', 'Custom application'], 1, wx.RA_SPECIFY_COLS) + self.rbSsoMode.SetItemToolTip(0, "This options routes SSO Logins through pyfa.io, allowing you to easily login without any configuration. When in doubt, use this option.") + self.rbSsoMode.SetItemToolTip(1, "This option goes through EVE SSO directly, but requires more configuration. Use this is pyfa.io is blocked for some reason, or if you do not wish to route data throguh pyfa.io.") self.rbMode.SetSelection(self.settings.get('loginMode')) - # self.rbServer.SetSelection(self.settings.get('server')) + self.rbSsoMode.SetSelection(self.settings.get('ssoMode')) + rbSizer.Add(self.rbSsoMode, 1, wx.ALL, 5) rbSizer.Add(self.rbMode, 1, wx.TOP | wx.RIGHT, 5) - # rbSizer.Add(self.rbServer, 1, wx.ALL, 5) self.rbMode.Bind(wx.EVT_RADIOBOX, self.OnModeChange) - # self.rbServer.Bind(wx.EVT_RADIOBOX, self.OnServerChange) + self.rbSsoMode.Bind(wx.EVT_RADIOBOX, self.OnSSOChange) mainSizer.Add(rbSizer, 1, wx.ALL | wx.EXPAND, 0) + detailsTitle = wx.StaticText(panel, wx.ID_ANY, "Custom Application", wx.DefaultPosition, wx.DefaultSize, 0) + detailsTitle.Wrap(-1) + detailsTitle.SetFont(wx.Font(12, 70, 90, 90, False, wx.EmptyString)) + + mainSizer.Add(detailsTitle, 0, wx.ALL, 5) + mainSizer.Add(wx.StaticLine(panel, wx.ID_ANY, wx.DefaultPosition, wx.DefaultSize, wx.LI_HORIZONTAL), 0, + wx.EXPAND, 5) + + # self.stInfo = wx.StaticText(panel, wx.ID_ANY, + # u"Using custom applications details will let pyfa to access the SSO under your application, rather than the pyfa application that is automatically set up. This requires you to set up your own ESI client application and accept CCPs License Agreement. Additionally, when setting up your client, make sure the callback url is set to 'http://localhost:6461'. Please see the pyfa wiki for more information regarding this", + # wx.DefaultPosition, wx.DefaultSize, 0) + # self.stInfo.Wrap(dlgWidth) + # mainSizer.Add(self.stInfo, 0, wx.EXPAND | wx.TOP | wx.BOTTOM, 5) + + fgAddrSizer = wx.FlexGridSizer(2, 2, 0, 0) + fgAddrSizer.AddGrowableCol(1) + fgAddrSizer.SetFlexibleDirection(wx.BOTH) + fgAddrSizer.SetNonFlexibleGrowMode(wx.FLEX_GROWMODE_SPECIFIED) + + self.stSetID = wx.StaticText(panel, wx.ID_ANY, u"Client ID:", wx.DefaultPosition, wx.DefaultSize, 0) + self.stSetID.Wrap(-1) + fgAddrSizer.Add(self.stSetID, 0, wx.ALL | wx.ALIGN_CENTER_VERTICAL, 5) + + self.inputClientID = wx.TextCtrl(panel, wx.ID_ANY, self.settings.get('clientID'), wx.DefaultPosition, + wx.DefaultSize, 0) + + fgAddrSizer.Add(self.inputClientID, 0, wx.ALL | wx.ALIGN_CENTER_VERTICAL | wx.EXPAND, 5) + + self.stSetSecret = wx.StaticText(panel, wx.ID_ANY, u"Client Secret:", wx.DefaultPosition, wx.DefaultSize, 0) + self.stSetSecret.Wrap(-1) + + fgAddrSizer.Add(self.stSetSecret, 0, wx.ALL | wx.ALIGN_CENTER_VERTICAL, 5) + + self.inputClientSecret = wx.TextCtrl(panel, wx.ID_ANY, self.settings.get('clientSecret'), wx.DefaultPosition, + wx.DefaultSize, 0) + + fgAddrSizer.Add(self.inputClientSecret, 0, wx.ALL | wx.ALIGN_CENTER_VERTICAL | wx.EXPAND, 5) + + self.inputClientID.Bind(wx.EVT_TEXT, self.OnClientDetailChange) + self.inputClientSecret.Bind(wx.EVT_TEXT, self.OnClientDetailChange) + + mainSizer.Add(fgAddrSizer, 0, wx.EXPAND, 5) + + + timeoutSizer = wx.BoxSizer(wx.HORIZONTAL) # self.stTimout = wx.StaticText(panel, wx.ID_ANY, "Timeout (seconds):", wx.DefaultPosition, wx.DefaultSize, 0) @@ -112,6 +160,7 @@ class PFEsiPref(PreferenceView): # self.ToggleProxySettings(self.settings.get('loginMode')) + self.ToggleSSOMode(self.settings.get('ssoMode')) panel.SetSizer(mainSizer) panel.Layout() @@ -121,14 +170,31 @@ class PFEsiPref(PreferenceView): def OnModeChange(self, event): self.settings.set('loginMode', event.GetInt()) - def OnServerChange(self, event): - self.settings.set('server', event.GetInt()) + def OnSSOChange(self, event): + self.settings.set('ssoMode', event.GetInt()) + self.ToggleSSOMode(event.GetInt()) - def OnBtnApply(self, event): + def ToggleSSOMode(self, mode): + if mode: + self.stSetID.Enable() + self.inputClientID.Enable() + self.stSetSecret.Enable() + self.inputClientSecret.Enable() + self.rbMode.Disable() + else: + self.stSetID.Disable() + self.inputClientID.Disable() + self.stSetSecret.Disable() + self.inputClientSecret.Disable() + self.rbMode.Enable() + + def OnClientDetailChange(self, evt): self.settings.set('clientID', self.inputClientID.GetValue().strip()) self.settings.set('clientSecret', self.inputClientSecret.GetValue().strip()) - sEsi = Esi.getInstance() - sEsi.delAllCharacters() + + # sEsi = Esi.getInstance() + # sEsi.delAllCharacters() + # def getImage(self): return BitmapLoader.getBitmap("eve", "gui") diff --git a/service/settings.py b/service/settings.py index ae5ce0e11..6852dddcb 100644 --- a/service/settings.py +++ b/service/settings.py @@ -363,10 +363,18 @@ class EsiSettings(object): return cls._instance def __init__(self): + # SSO Mode: + # 0 - pyfa.io + # 1 - custom application # LoginMode: # 0 - Server Start Up # 1 - User copy and paste data from website to pyfa - defaults = {"loginMode": 0, "clientID": "", "clientSecret": "", "timeout": 60} + defaults = { + "ssoMode": 0, + "loginMode": 0, + "clientID": "", + "clientSecret": "", + "timeout": 60} self.settings = SettingsProvider.getInstance().getSettings( "pyfaServiceEsiSettings", From b6a1c4b30818f2f040159b22cb08faf25d96baa6 Mon Sep 17 00:00:00 2001 From: blitzmann Date: Sun, 13 May 2018 22:33:58 -0400 Subject: [PATCH 08/11] Add support for using own client details (messy code, but it works!) --- config.py | 1 - gui/esiFittings.py | 3 +- gui/mainFrame.py | 5 +- service/esi.py | 29 +++++--- service/esiAccess.py | 173 ++++++++++++++++++++++++++++++------------- 5 files changed, 147 insertions(+), 64 deletions(-) diff --git a/config.py b/config.py index 5da7854bc..0ff71c37b 100644 --- a/config.py +++ b/config.py @@ -42,7 +42,6 @@ logging_setup = None cipher = None clientHash = None -ESI_AUTH_PROXY = "https://www.pyfa.io" # "http://localhost:5015" ESI_CACHE = 'esi_cache' LOGLEVEL_MAP = { diff --git a/gui/esiFittings.py b/gui/esiFittings.py index 0ac8e90db..98944fde1 100644 --- a/gui/esiFittings.py +++ b/gui/esiFittings.py @@ -123,6 +123,7 @@ class EveFittings(wx.Frame): ESIExceptionHandler(self, ex) except Exception as ex: del waitDialog + raise ex def importFitting(self, event): selection = self.fitView.fitSelection @@ -159,7 +160,7 @@ class EveFittings(wx.Frame): class ESIExceptionHandler(object): # todo: make this a generate excetpion handler for all calls def __init__(self, parentWindow, ex): - if ex.response['error'].startswith('Token is not valid'): + if ex.response['error'].startswith('Token is not valid') or ex.response['error'] == 'invalid_token': # todo: this seems messy, figure out a better response dlg = wx.MessageDialog(parentWindow, "There was an error validating characters' SSO token. Please try " "logging into the character again to reset the token.", "Invalid Token", diff --git a/gui/mainFrame.py b/gui/mainFrame.py index c9e5fa09c..4a8a6d1a1 100644 --- a/gui/mainFrame.py +++ b/gui/mainFrame.py @@ -69,6 +69,7 @@ from service.settings import SettingsProvider from service.fit import Fit from service.character import Character from service.update import Update +from service.esiAccess import SsoMode # import this to access override setting from eos.modifiedAttributeDict import ModifiedAttributeDict @@ -241,12 +242,12 @@ class MainFrame(wx.Frame): self.Bind(GE.EVT_SSO_LOGGING_IN, self.ShowSsoLogin) def ShowSsoLogin(self, event): - if getattr(event, "login_mode", LoginMethod.SERVER) == LoginMethod.MANUAL: + if getattr(event, "login_mode", LoginMethod.SERVER) == LoginMethod.MANUAL and getattr(event, "sso_mode", SsoMode.AUTO) == SsoMode.AUTO: dlg = SsoLogin(self) if dlg.ShowModal() == wx.ID_OK: sEsi = Esi.getInstance() # todo: verify that this is a correct SSO Info block - sEsi.handleLogin(dlg.ssoInfoCtrl.Value.strip()) + sEsi.handleLogin({'SSOInfo': [dlg.ssoInfoCtrl.Value.strip()]}) def ShowUpdateBox(self, release, version): dlg = UpdateDialog(self, release, version) diff --git a/service/esi.py b/service/esi.py index 78c80e04a..aafb60044 100644 --- a/service/esi.py +++ b/service/esi.py @@ -11,7 +11,7 @@ import webbrowser import eos.db from eos.enum import Enum from eos.saveddata.ssocharacter import SsoCharacter -from service.esiAccess import APIException +from service.esiAccess import APIException, SsoMode import gui.globalEvents as GE from service.server import StoppableHTTPServer, AuthHandler from service.settings import EsiSettings @@ -39,9 +39,10 @@ class Esi(EsiAccess): return cls._instance def __init__(self): - super().__init__() self.settings = EsiSettings.getInstance() + super().__init__() + # these will be set when needed self.httpd = None self.state = None @@ -105,18 +106,19 @@ class Esi(EsiAccess): def login(self): serverAddr = None - if self.settings.get('loginMode') == LoginMethod.SERVER: - serverAddr = self.startServer() + # always start the local server if user is using client details. Otherwise, start only if they choose to do so. + if self.settings.get('ssoMode') == SsoMode.CUSTOM or self.settings.get('loginMode') == LoginMethod.SERVER: + serverAddr = self.startServer(6461 if self.settings.get('ssoMode') == SsoMode.CUSTOM else 0) # random port, or if it's custom application, use a defined port uri = self.getLoginURI(serverAddr) webbrowser.open(uri) - wx.PostEvent(self.mainFrame, GE.SsoLoggingIn(login_mode=self.settings.get('loginMode'))) + wx.PostEvent(self.mainFrame, GE.SsoLoggingIn(sso_mode=self.settings.get('ssoMode'), login_mode=self.settings.get('loginMode'))) def stopServer(self): pyfalog.debug("Stopping Server") self.httpd.stop() self.httpd = None - def startServer(self): # todo: break this out into two functions: starting the server, and getting the URI + def startServer(self, port): # todo: break this out into two functions: starting the server, and getting the URI pyfalog.debug("Starting server") # we need this to ensure that the previous get_request finishes, and then the socket will close @@ -124,7 +126,7 @@ class Esi(EsiAccess): self.stopServer() time.sleep(1) - self.httpd = StoppableHTTPServer(('localhost', 0), AuthHandler) + self.httpd = StoppableHTTPServer(('localhost', port), AuthHandler) port = self.httpd.socket.getsockname()[1] self.serverThread = threading.Thread(target=self.httpd.serve, args=(self.handleServerLogin,)) self.serverThread.name = "SsoCallbackServer" @@ -133,8 +135,15 @@ class Esi(EsiAccess): return 'http://localhost:{}'.format(port) - def handleLogin(self, ssoInfo): - auth_response = json.loads(base64.b64decode(ssoInfo)) + def handleLogin(self, message): + + # we already have authenticated stuff for the auto mode + if (self.settings.get('ssoMode') == SsoMode.AUTO): + ssoInfo = message['SSOInfo'][0] + auth_response = json.loads(base64.b64decode(ssoInfo)) + else: + # otherwise, we need to fetch the information + auth_response = self.auth(message['code'][0]) res = self._session.get( self.oauth_verify, @@ -171,5 +180,5 @@ class Esi(EsiAccess): pyfalog.debug("Handling SSO login with: {0}", message) - self.handleLogin(message['SSOInfo'][0]) + self.handleLogin(message) diff --git a/service/esiAccess.py b/service/esiAccess.py index 864ca397d..7636249a4 100644 --- a/service/esiAccess.py +++ b/service/esiAccess.py @@ -1,16 +1,27 @@ +''' + +A lot of the inspiration (and straight up code copying!) for this class comes from EsiPy +Much of the credit goes to the maintainer of that package, Kyria . The reasoning for no +longer using EsiPy was due to it's reliance on pyswagger, which has caused a bit of a headache in how it operates on a +low level. + +Eventually I'll rewrite this to be a bit cleaner and a bit more generic, but for now, it works! + +''' + # noinspection PyPackageRequirements from logbook import Logger import uuid import time import config +import base64 import datetime from eos.enum import Enum -from eos.saveddata.ssocharacter import SsoCharacter from service.settings import EsiSettings from requests import Session -from urllib.parse import urlencode +from urllib.parse import urlencode, quote pyfalog = Logger(__name__) @@ -23,12 +34,17 @@ pyfalog = Logger(__name__) # os.mkdir(cache_path) # -# todo: move these over to getters that automatically determine which endpoint we use. -sso_url = "https://www.pyfa.io" # "https://login.eveonline.com" for actual login -esi_url = "https://esi.tech.ccp.is" -oauth_authorize = '%s/oauth/authorize' % sso_url -oauth_token = '%s/oauth/token' % sso_url +scopes = [ + 'esi-skills.read_skills.v1', + 'esi-fittings.read_fittings.v1', + 'esi-fittings.write_fittings.v1' +] + + +class SsoMode(Enum): + AUTO = 0 + CUSTOM = 1 class APIException(Exception): @@ -57,27 +73,10 @@ class ESIEndpoints(Enum): CHAR_DEL_FIT = "/v1/characters/{character_id}/fittings/{fitting_id}/" -# class Servers(Enum): -# TQ = 0 -# SISI = 1 - class EsiAccess(object): def __init__(self): - if sso_url is None or sso_url == "": - raise AttributeError("sso_url cannot be None or empty " - "without app parameter") - self.settings = EsiSettings.getInstance() - self.oauth_authorize = '%s/oauth/authorize' % sso_url - self.oauth_token = '%s/oauth/token' % sso_url - - # use ESI url for verify, since it's better for caching - if esi_url is None or esi_url == "": - raise AttributeError("esi_url cannot be None or empty") - self.oauth_verify = '%s/verify/' % esi_url - - # session request stuff self._session = Session() self._session.headers.update({ @@ -87,6 +86,28 @@ class EsiAccess(object): ) }) + @property + def sso_url(self): + if (self.settings.get("ssoMode") == SsoMode.CUSTOM): + return "https://login.eveonline.com" + return "https://www.pyfa.io" + + @property + def esi_url(self): + return "https://esi.tech.ccp.is" + + @property + def oauth_verify(self): + return '%s/verify/' % self.esi_url + + @property + def oauth_authorize(self): + return '%s/oauth/authorize' % self.sso_url + + @property + def oauth_token(self): + return '%s/oauth/token' % self.sso_url + def getSkills(self, char): return self.get(char, ESIEndpoints.CHAR_SKILLS, character_id=char.characterID) @@ -114,20 +135,30 @@ class EsiAccess(object): def getLoginURI(self, redirect=None): self.state = str(uuid.uuid4()) - args = { - 'state': self.state, - 'pyfa_version': config.version, - 'login_method': self.settings.get('loginMode'), - 'client_hash': config.getClientSecret() - } + if (self.settings.get("ssoMode") == SsoMode.AUTO): + args = { + 'state': self.state, + 'pyfa_version': config.version, + 'login_method': self.settings.get('loginMode'), + 'client_hash': config.getClientSecret() + } - if redirect is not None: - args['redirect'] = redirect + if redirect is not None: + args['redirect'] = redirect - return '%s?%s' % ( - oauth_authorize, - urlencode(args) - ) + return '%s?%s' % ( + self.oauth_authorize, + urlencode(args) + ) + else: + return '%s?response_type=%s&redirect_uri=%s&client_id=%s%s%s' % ( + self.oauth_authorize, + 'code', + quote('http://localhost:6461', safe=''), + self.settings.get('clientID'), + '&scope=%s' % '+'.join(scopes) if scopes else '', + '&state=%s' % self.state + ) def get_oauth_header(self, token): """ Return the Bearer Authorization header required in oauth calls @@ -146,14 +177,62 @@ class EsiAccess(object): if refreshToken is None: raise AttributeError('No refresh token is defined.') - return { - 'data': { - 'grant_type': 'refresh_token', - 'refresh_token': refreshToken, - }, + data = { + 'grant_type': 'refresh_token', + 'refresh_token': refreshToken, + } + + if self.settings.get('ssoMode') == SsoMode.AUTO: + # data is all we really need, the rest is handled automatically by pyfa.io + return { + 'data': data, + 'url': self.oauth_token, + } + + # otherwise, we need to make the token with the client keys + return self.__make_token_request_parameters(data) + + def __get_token_auth_header(self): + """ Return the Basic Authorization header required to get the tokens + + :return: a dict with the headers + """ + # encode/decode for py2/py3 compatibility + auth_b64 = "%s:%s" % (self.settings.get('clientID'), self.settings.get('clientSecret')) + auth_b64 = base64.b64encode(auth_b64.encode('latin-1')) + auth_b64 = auth_b64.decode('latin-1') + + return {'Authorization': 'Basic %s' % auth_b64} + + def __make_token_request_parameters(self, params): + request_params = { + 'headers': self.__get_token_auth_header(), + 'data': params, 'url': self.oauth_token, } + return request_params + + def get_access_token_request_params(self, code): + return self.__make_token_request_parameters( + { + 'grant_type': 'authorization_code', + 'code': code, + } + ) + + def auth(self, code): + request_data = self.get_access_token_request_params(code) + res = self._session.post(**request_data) + if res.status_code != 200: + raise Exception( + request_data['url'], + res.status_code, + res.json() + ) + json_res = res.json() + return json_res + def refresh(self, ssoChar): request_data = self.get_refresh_token_params(config.cipher.decrypt(ssoChar.refreshToken).decode()) res = self._session.post(**request_data) @@ -191,21 +270,15 @@ class EsiAccess(object): def get(self, ssoChar, endpoint, *args, **kwargs): self._before_request(ssoChar) endpoint = endpoint.format(**kwargs) - return self._after_request(self._session.get("{}{}".format(esi_url, endpoint))) - - # check for warnings, also status > 400 + return self._after_request(self._session.get("{}{}".format(self.esi_url, endpoint))) def post(self, ssoChar, endpoint, json, *args, **kwargs): self._before_request(ssoChar) endpoint = endpoint.format(**kwargs) - return self._after_request(self._session.post("{}{}".format(esi_url, endpoint), data=json)) - - # check for warnings, also status > 400 + return self._after_request(self._session.post("{}{}".format(self.esi_url, endpoint), data=json)) def delete(self, ssoChar, endpoint, *args, **kwargs): self._before_request(ssoChar) endpoint = endpoint.format(**kwargs) - return self._after_request(self._session.delete("{}{}".format(esi_url, endpoint))) - - # check for warnings, also status > 400 + return self._after_request(self._session.delete("{}{}".format(self.esi_url, endpoint))) From 51e610830f7a777c46fc197853d38d80e799e8d7 Mon Sep 17 00:00:00 2001 From: blitzmann Date: Mon, 14 May 2018 18:27:53 -0400 Subject: [PATCH 09/11] Fix (not really) an issue with a dead character editor still receiving events --- gui/characterEditor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gui/characterEditor.py b/gui/characterEditor.py index c7a2fef8c..f7520e42c 100644 --- a/gui/characterEditor.py +++ b/gui/characterEditor.py @@ -788,6 +788,8 @@ class APIView(wx.Panel): return self.charChoice.GetClientData(selection) if selection is not -1 else None def ssoListChanged(self, event): + if not self: # todo: fix event not unbinding properly + return sEsi = Esi.getInstance() ssoChars = sEsi.getSsoCharacters() From 3e1244a27a5f373f25f640e04cc95dbef140e682 Mon Sep 17 00:00:00 2001 From: blitzmann Date: Mon, 14 May 2018 19:26:23 -0400 Subject: [PATCH 10/11] Fix for not being able to drag fit to tab area (#1569) --- gui/builtinViews/fittingView.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/builtinViews/fittingView.py b/gui/builtinViews/fittingView.py index 7e4eccb03..9e4e65904 100644 --- a/gui/builtinViews/fittingView.py +++ b/gui/builtinViews/fittingView.py @@ -88,7 +88,7 @@ class FitSpawner(gui.multiSwitch.TabSpawner): def handleDrag(self, type, fitID): if type == "fit": - for page in self.multiSwitch.pages: + for page in self.multiSwitch._pages: if isinstance(page, FittingView) and page.activeFitID == fitID: index = self.multiSwitch.GetPageIndex(page) self.multiSwitch.SetSelection(index) From 8023b2ea29aeeaaf47fc68984fbdb0eeedc73180 Mon Sep 17 00:00:00 2001 From: blitzmann Date: Mon, 14 May 2018 21:59:25 -0400 Subject: [PATCH 11/11] Bump version --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 0ff71c37b..24adc62c7 100644 --- a/config.py +++ b/config.py @@ -24,7 +24,7 @@ saveInRoot = False # Version data -version = "2.0.0" +version = "2.0.1" tag = "Stable" expansionName = "YC120.3" expansionVersion = "1.8"