Added pycrest with a few fixes.
This commit is contained in:
blitzmann
13
pycrest/__init__.py
Normal file
13
pycrest/__init__.py
Normal file
@@ -0,0 +1,13 @@
|
||||
import logging
|
||||
|
||||
|
||||
class NullHandler(logging.Handler):
|
||||
def emit(self, record):
|
||||
pass
|
||||
|
||||
logger = logging.getLogger('pycrest')
|
||||
logger.addHandler(NullHandler())
|
||||
|
||||
version = "0.0.1"
|
||||
|
||||
from .eve import EVE
|
||||
24
pycrest/compat.py
Normal file
24
pycrest/compat.py
Normal file
@@ -0,0 +1,24 @@
|
||||
import sys
|
||||
|
||||
PY3 = sys.version_info[0] == 3
|
||||
|
||||
if PY3: # pragma: no cover
|
||||
string_types = str,
|
||||
text_type = str
|
||||
binary_type = bytes
|
||||
else: # pragma: no cover
|
||||
string_types = basestring,
|
||||
text_type = unicode
|
||||
binary_type = str
|
||||
|
||||
|
||||
def text_(s, encoding='latin-1', errors='strict'): # pragma: no cover
|
||||
if isinstance(s, binary_type):
|
||||
return s.decode(encoding, errors)
|
||||
return s
|
||||
|
||||
|
||||
def bytes_(s, encoding='latin-1', errors='strict'): # pragma: no cover
|
||||
if isinstance(s, text_type):
|
||||
return s.encode(encoding, errors)
|
||||
return s
|
||||
2
pycrest/errors.py
Normal file
2
pycrest/errors.py
Normal file
@@ -0,0 +1,2 @@
|
||||
class APIException(Exception):
|
||||
pass
|
||||
331
pycrest/eve.py
Normal file
331
pycrest/eve.py
Normal file
@@ -0,0 +1,331 @@
|
||||
import os
|
||||
import base64
|
||||
import requests
|
||||
import time
|
||||
import zlib
|
||||
from pycrest import version
|
||||
from pycrest.compat import bytes_, text_
|
||||
from pycrest.errors import APIException
|
||||
from pycrest.weak_ciphers import WeakCiphersAdapter
|
||||
|
||||
try:
|
||||
from urllib.parse import urlparse, urlunparse, parse_qsl
|
||||
except ImportError: # pragma: no cover
|
||||
from urlparse import urlparse, urlunparse, parse_qsl
|
||||
|
||||
try:
|
||||
import pickle
|
||||
except ImportError: # pragma: no cover
|
||||
import cPickle as pickle
|
||||
|
||||
try:
|
||||
from urllib.parse import quote
|
||||
except ImportError: # pragma: no cover
|
||||
from urllib import quote
|
||||
import logging
|
||||
import re
|
||||
|
||||
logger = logging.getLogger("pycrest.eve")
|
||||
cache_re = re.compile(r'max-age=([0-9]+)')
|
||||
|
||||
|
||||
class APICache(object):
|
||||
def put(self, key, value):
|
||||
raise NotImplementedError
|
||||
|
||||
def get(self, key):
|
||||
raise NotImplementedError
|
||||
|
||||
def invalidate(self, key):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class FileCache(APICache):
|
||||
def __init__(self, path):
|
||||
self._cache = {}
|
||||
self.path = path
|
||||
if not os.path.isdir(self.path):
|
||||
os.mkdir(self.path, 0o700)
|
||||
|
||||
def _getpath(self, key):
|
||||
return os.path.join(self.path, str(hash(key)) + '.cache')
|
||||
|
||||
def put(self, key, value):
|
||||
with open(self._getpath(key), 'wb') as f:
|
||||
f.write(zlib.compress(pickle.dumps(value, -1)))
|
||||
self._cache[key] = value
|
||||
|
||||
def get(self, key):
|
||||
if key in self._cache:
|
||||
return self._cache[key]
|
||||
|
||||
try:
|
||||
with open(self._getpath(key), 'rb') as f:
|
||||
return pickle.loads(zlib.decompress(f.read()))
|
||||
except IOError as ex:
|
||||
if ex.errno == 2: # file does not exist (yet)
|
||||
return None
|
||||
else:
|
||||
raise
|
||||
|
||||
def invalidate(self, key):
|
||||
self._cache.pop(key, None)
|
||||
|
||||
try:
|
||||
os.unlink(self._getpath(key))
|
||||
except OSError as ex:
|
||||
if ex.errno == 2: # does not exist
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
class DictCache(APICache):
|
||||
def __init__(self):
|
||||
self._dict = {}
|
||||
|
||||
def get(self, key):
|
||||
return self._dict.get(key, None)
|
||||
|
||||
def put(self, key, value):
|
||||
self._dict[key] = value
|
||||
|
||||
def invalidate(self, key):
|
||||
self._dict.pop(key, None)
|
||||
|
||||
|
||||
class APIConnection(object):
|
||||
def __init__(self, additional_headers=None, user_agent=None, cache_dir=None, cache=None):
|
||||
# Set up a Requests Session
|
||||
session = requests.Session()
|
||||
if additional_headers is None:
|
||||
additional_headers = {}
|
||||
if user_agent is None:
|
||||
user_agent = "PyCrest/{0}".format(version)
|
||||
session.headers.update({
|
||||
"User-Agent": user_agent,
|
||||
"Accept": "application/json",
|
||||
})
|
||||
session.headers.update(additional_headers)
|
||||
session.mount('https://public-crest.eveonline.com',
|
||||
WeakCiphersAdapter())
|
||||
self._session = session
|
||||
if cache:
|
||||
if isinstance(cache, APICache):
|
||||
self.cache = cache # Inherit from parents
|
||||
elif isinstance(cache, type):
|
||||
self.cache = cache() # Instantiate a new cache
|
||||
elif cache_dir:
|
||||
self.cache_dir = cache_dir
|
||||
self.cache = FileCache(self.cache_dir)
|
||||
else:
|
||||
self.cache = DictCache()
|
||||
|
||||
def get(self, resource, params=None):
|
||||
print resource, params
|
||||
logger.debug('Getting resource %s', resource)
|
||||
if params is None:
|
||||
params = {}
|
||||
|
||||
# remove params from resource URI (needed for paginated stuff)
|
||||
parsed_uri = urlparse(resource)
|
||||
qs = parsed_uri.query
|
||||
resource = urlunparse(parsed_uri._replace(query=''))
|
||||
prms = {}
|
||||
for tup in parse_qsl(qs):
|
||||
prms[tup[0]] = tup[1]
|
||||
|
||||
# params supplied to self.get() override parsed params
|
||||
for key in params:
|
||||
prms[key] = params[key]
|
||||
|
||||
# check cache
|
||||
key = (resource, frozenset(self._session.headers.items()), frozenset(prms.items()))
|
||||
cached = self.cache.get(key)
|
||||
if cached and cached['expires'] > time.time():
|
||||
logger.debug('Cache hit for resource %s (params=%s)', resource, prms)
|
||||
return cached['payload']
|
||||
elif cached:
|
||||
logger.debug('Cache stale for resource %s (params=%s)', resource, prms)
|
||||
self.cache.invalidate(key)
|
||||
else:
|
||||
logger.debug('Cache miss for resource %s (params=%s', resource, prms)
|
||||
|
||||
logger.debug('Getting resource %s (params=%s)', resource, prms)
|
||||
res = self._session.get(resource, params=prms)
|
||||
if res.status_code != 200:
|
||||
raise APIException("Got unexpected status code from server: %i" % res.status_code)
|
||||
|
||||
ret = res.json()
|
||||
|
||||
# cache result
|
||||
key = (resource, frozenset(self._session.headers.items()), frozenset(prms.items()))
|
||||
expires = self._get_expires(res)
|
||||
if expires > 0:
|
||||
self.cache.put(key, {'expires': time.time() + expires, 'payload': ret})
|
||||
|
||||
return ret
|
||||
|
||||
def _get_expires(self, response):
|
||||
if 'Cache-Control' not in response.headers:
|
||||
return 0
|
||||
if any([s in response.headers['Cache-Control'] for s in ['no-cache', 'no-store']]):
|
||||
return 0
|
||||
match = cache_re.search(response.headers['Cache-Control'])
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
return 0
|
||||
|
||||
|
||||
class EVE(APIConnection):
|
||||
def __init__(self, **kwargs):
|
||||
self.api_key = kwargs.pop('api_key', None)
|
||||
self.client_id = kwargs.pop('client_id', None)
|
||||
self.redirect_uri = kwargs.pop('redirect_uri', None)
|
||||
if kwargs.pop('testing', False):
|
||||
self._public_endpoint = "http://public-crest-sisi.testeveonline.com/"
|
||||
self._authed_endpoint = "https://api-sisi.testeveonline.com/"
|
||||
self._image_server = "https://image.testeveonline.com/"
|
||||
self._oauth_endpoint = "https://sisilogin.testeveonline.com/oauth"
|
||||
else:
|
||||
self._public_endpoint = "https://public-crest.eveonline.com/"
|
||||
self._authed_endpoint = "https://crest-tq.eveonline.com/"
|
||||
self._image_server = "https://image.eveonline.com/"
|
||||
self._oauth_endpoint = "https://login.eveonline.com/oauth"
|
||||
self._endpoint = self._public_endpoint
|
||||
self._cache = {}
|
||||
self._data = None
|
||||
APIConnection.__init__(self, **kwargs)
|
||||
|
||||
def __call__(self):
|
||||
if not self._data:
|
||||
self._data = APIObject(self.get(self._endpoint), self)
|
||||
return self._data
|
||||
|
||||
def __getattr__(self, item):
|
||||
return self._data.__getattr__(item)
|
||||
|
||||
def auth_uri(self, scopes=None, state=None):
|
||||
s = [] if not scopes else scopes
|
||||
return "%s/authorize?response_type=code&redirect_uri=%s&client_id=%s%s%s" % (
|
||||
self._oauth_endpoint,
|
||||
quote(self.redirect_uri, safe=''),
|
||||
self.client_id,
|
||||
"&scope=%s" % ' '.join(s) if scopes else '',
|
||||
"&state=%s" % state if state else ''
|
||||
)
|
||||
|
||||
def _authorize(self, params):
|
||||
auth = text_(base64.b64encode(bytes_("%s:%s" % (self.client_id, self.api_key))))
|
||||
headers = {"Authorization": "Basic %s" % auth}
|
||||
res = self._session.post("%s/token" % self._oauth_endpoint, params=params, headers=headers)
|
||||
if res.status_code != 200:
|
||||
raise APIException("Got unexpected status code from API: %i" % res.status_code)
|
||||
return res.json()
|
||||
|
||||
def authorize(self, code):
|
||||
res = self._authorize(params={"grant_type": "authorization_code", "code": code})
|
||||
return AuthedConnection(res,
|
||||
self._authed_endpoint,
|
||||
self._oauth_endpoint,
|
||||
self.client_id,
|
||||
self.api_key,
|
||||
cache=self.cache)
|
||||
|
||||
def refr_authorize(self, refresh_token):
|
||||
res = self._authorize(params={"grant_type": "refresh_token", "refresh_token": refresh_token})
|
||||
return AuthedConnection({'access_token': res['access_token'],
|
||||
'refresh_token': refresh_token,
|
||||
'expires_in': res['expires_in']},
|
||||
self._authed_endpoint,
|
||||
self._oauth_endpoint,
|
||||
self.client_id,
|
||||
self.api_key,
|
||||
cache=self.cache)
|
||||
|
||||
def temptoken_authorize(self, access_token, expires_in, refresh_token):
|
||||
return AuthedConnection({'access_token': access_token,
|
||||
'refresh_token': refresh_token,
|
||||
'expires_in': expires_in},
|
||||
self._authed_endpoint,
|
||||
self._oauth_endpoint,
|
||||
self.client_id,
|
||||
self.api_key,
|
||||
cache=self.cache)
|
||||
|
||||
|
||||
class AuthedConnection(EVE):
|
||||
def __init__(self, res, endpoint, oauth_endpoint, client_id=None, api_key=None, **kwargs):
|
||||
EVE.__init__(self, **kwargs)
|
||||
self.client_id = client_id
|
||||
self.api_key = api_key
|
||||
self.token = res['access_token']
|
||||
self.refresh_token = res['refresh_token']
|
||||
self.expires = int(time.time()) + res['expires_in']
|
||||
self._oauth_endpoint = oauth_endpoint
|
||||
self._endpoint = endpoint
|
||||
self._session.headers.update({"Authorization": "Bearer %s" % self.token})
|
||||
|
||||
def __call__(self):
|
||||
if not self._data:
|
||||
self._data = APIObject(self.get(self._endpoint), self)
|
||||
return self._data
|
||||
|
||||
def whoami(self):
|
||||
if 'whoami' not in self._cache:
|
||||
self._cache['whoami'] = self.get("%s/verify" % self._oauth_endpoint)
|
||||
return self._cache['whoami']
|
||||
|
||||
def refresh(self):
|
||||
res = self._authorize(params={"grant_type": "refresh_token", "refresh_token": self.refresh_token})
|
||||
self.token = res['access_token']
|
||||
self.expires = int(time.time()) + res['expires_in']
|
||||
self._session.headers.update({"Authorization": "Bearer %s" % self.token})
|
||||
return self # for backwards compatibility
|
||||
|
||||
def get(self, resource, params=None):
|
||||
if int(time.time()) >= self.expires:
|
||||
self.refresh()
|
||||
return super(self.__class__, self).get(resource, params)
|
||||
|
||||
|
||||
class APIObject(object):
|
||||
def __init__(self, parent, connection):
|
||||
self._dict = {}
|
||||
self.connection = connection
|
||||
for k, v in parent.items():
|
||||
if type(v) is dict:
|
||||
self._dict[k] = APIObject(v, connection)
|
||||
elif type(v) is list:
|
||||
self._dict[k] = self._wrap_list(v)
|
||||
else:
|
||||
self._dict[k] = v
|
||||
|
||||
def _wrap_list(self, list_):
|
||||
new = []
|
||||
for item in list_:
|
||||
if type(item) is dict:
|
||||
new.append(APIObject(item, self.connection))
|
||||
elif type(item) is list:
|
||||
new.append(self._wrap_list(item))
|
||||
else:
|
||||
new.append(item)
|
||||
return new
|
||||
|
||||
def __getattr__(self, item):
|
||||
if item in self._dict:
|
||||
return self._dict[item]
|
||||
raise AttributeError(item)
|
||||
|
||||
def __call__(self, **kwargs):
|
||||
# Caching is now handled by APIConnection
|
||||
if 'href' in self._dict:
|
||||
return APIObject(self.connection.get(self._dict['href'], params=kwargs), self.connection)
|
||||
else:
|
||||
return self
|
||||
|
||||
def __str__(self): # pragma: no cover
|
||||
return self._dict.__str__()
|
||||
|
||||
def __repr__(self): # pragma: no cover
|
||||
return self._dict.__repr__()
|
||||
140
pycrest/weak_ciphers.py
Normal file
140
pycrest/weak_ciphers.py
Normal file
@@ -0,0 +1,140 @@
|
||||
import datetime
|
||||
import ssl
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
from requests.adapters import HTTPAdapter
|
||||
|
||||
try:
|
||||
from requests.packages import urllib3
|
||||
from requests.packages.urllib3.util import ssl_
|
||||
|
||||
from requests.packages.urllib3.exceptions import (
|
||||
SystemTimeWarning,
|
||||
SecurityWarning,
|
||||
)
|
||||
from requests.packages.urllib3.packages.ssl_match_hostname import \
|
||||
match_hostname
|
||||
except:
|
||||
import urllib3
|
||||
from urllib3.util import ssl_
|
||||
|
||||
from urllib3.exceptions import (
|
||||
SystemTimeWarning,
|
||||
SecurityWarning,
|
||||
)
|
||||
from urllib3.packages.ssl_match_hostname import \
|
||||
match_hostname
|
||||
|
||||
|
||||
|
||||
|
||||
class WeakCiphersHTTPSConnection(
|
||||
urllib3.connection.VerifiedHTTPSConnection): # pragma: no cover
|
||||
|
||||
# Python versions >=2.7.9 and >=3.4.1 do not (by default) allow ciphers
|
||||
# with MD5. Unfortunately, the CREST public server _only_ supports
|
||||
# TLS_RSA_WITH_RC4_128_MD5 (as of 5 Jan 2015). The cipher list below is
|
||||
# nearly identical except for allowing that cipher as a last resort (and
|
||||
# excluding export versions of ciphers).
|
||||
DEFAULT_CIPHERS = (
|
||||
'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:'
|
||||
'ECDH+HIGH:DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:'
|
||||
'RSA+3DES:ECDH+RC4:DH+RC4:RSA+RC4:!aNULL:!eNULL:!EXP:-MD5:RSA+RC4+MD5'
|
||||
)
|
||||
|
||||
def __init__(self, host, port, ciphers=None, **kwargs):
|
||||
self.ciphers = ciphers if ciphers is not None else self.DEFAULT_CIPHERS
|
||||
super(WeakCiphersHTTPSConnection, self).__init__(host, port, **kwargs)
|
||||
|
||||
def connect(self):
|
||||
# Yup, copied in VerifiedHTTPSConnection.connect just to change the
|
||||
# default cipher list.
|
||||
|
||||
# Add certificate verification
|
||||
conn = self._new_conn()
|
||||
|
||||
resolved_cert_reqs = ssl_.resolve_cert_reqs(self.cert_reqs)
|
||||
resolved_ssl_version = ssl_.resolve_ssl_version(self.ssl_version)
|
||||
|
||||
hostname = self.host
|
||||
if getattr(self, '_tunnel_host', None):
|
||||
# _tunnel_host was added in Python 2.6.3
|
||||
# (See: http://hg.python.org/cpython/rev/0f57b30a152f)
|
||||
|
||||
self.sock = conn
|
||||
# Calls self._set_hostport(), so self.host is
|
||||
# self._tunnel_host below.
|
||||
self._tunnel()
|
||||
# Mark this connection as not reusable
|
||||
self.auto_open = 0
|
||||
|
||||
# Override the host with the one we're requesting data from.
|
||||
hostname = self._tunnel_host
|
||||
|
||||
is_time_off = datetime.date.today() < urllib3.connection.RECENT_DATE
|
||||
if is_time_off:
|
||||
warnings.warn((
|
||||
'System time is way off (before {0}). This will probably '
|
||||
'lead to SSL verification errors').format(
|
||||
urllib3.connection.RECENT_DATE),
|
||||
SystemTimeWarning
|
||||
)
|
||||
|
||||
# Wrap socket using verification with the root certs in
|
||||
# trusted_root_certs
|
||||
self.sock = ssl_.ssl_wrap_socket(conn, self.key_file, self.cert_file,
|
||||
cert_reqs=resolved_cert_reqs,
|
||||
ca_certs=self.ca_certs,
|
||||
server_hostname=hostname,
|
||||
ssl_version=resolved_ssl_version,
|
||||
ciphers=self.ciphers)
|
||||
|
||||
if self.assert_fingerprint:
|
||||
ssl_.assert_fingerprint(self.sock.getpeercert(binary_form=True),
|
||||
self.assert_fingerprint)
|
||||
elif resolved_cert_reqs != ssl.CERT_NONE \
|
||||
and self.assert_hostname is not False:
|
||||
cert = self.sock.getpeercert()
|
||||
if not cert.get('subjectAltName', ()):
|
||||
warnings.warn((
|
||||
'Certificate has no `subjectAltName`, falling back to check for a `commonName` for now. '
|
||||
'This feature is being removed by major browsers and deprecated by RFC 2818. '
|
||||
'(See https://github.com/shazow/urllib3/issues/497 for details.)'),
|
||||
SecurityWarning
|
||||
)
|
||||
match_hostname(cert, self.assert_hostname or hostname)
|
||||
|
||||
self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED
|
||||
or self.assert_fingerprint is not None)
|
||||
|
||||
|
||||
class WeakCiphersHTTPSConnectionPool(
|
||||
urllib3.connectionpool.HTTPSConnectionPool):
|
||||
|
||||
ConnectionCls = WeakCiphersHTTPSConnection
|
||||
|
||||
|
||||
class WeakCiphersPoolManager(urllib3.poolmanager.PoolManager):
|
||||
|
||||
def _new_pool(self, scheme, host, port):
|
||||
if scheme == 'https':
|
||||
return WeakCiphersHTTPSConnectionPool(host, port,
|
||||
**(self.connection_pool_kw))
|
||||
return super(WeakCiphersPoolManager, self)._new_pool(scheme, host,
|
||||
port)
|
||||
|
||||
|
||||
class WeakCiphersAdapter(HTTPAdapter):
|
||||
""""Transport adapter" that allows us to use TLS_RSA_WITH_RC4_128_MD5."""
|
||||
|
||||
def init_poolmanager(self, connections, maxsize, block=False,
|
||||
**pool_kwargs):
|
||||
# Rewrite of the requests.adapters.HTTPAdapter.init_poolmanager method
|
||||
# to use WeakCiphersPoolManager instead of urllib3's PoolManager
|
||||
self._pool_connections = connections
|
||||
self._pool_maxsize = maxsize
|
||||
self._pool_block = block
|
||||
|
||||
self.poolmanager = WeakCiphersPoolManager(num_pools=connections,
|
||||
maxsize=maxsize, block=block, strict=True, **pool_kwargs)
|
||||
Reference in New Issue
Block a user