From bc7b7154be293b5b2cc7efd0cf251dc2852bf392 Mon Sep 17 00:00:00 2001 From: Exodus4D Date: Thu, 13 Apr 2017 00:36:12 +0200 Subject: [PATCH] - switched system jump/kill-data update cronjob from XML API to ESI - use internal DEBUG level for MySQL ERRORMODE - increased MySQL version 5.6 => 5.7 - fixed bug in System auto search form element (select2) - code cleanup --- app/environment.ini | 14 ++- app/main/controller/api/system.php | 24 ----- app/main/controller/ccp/sso.php | 4 +- app/main/controller/controller.php | 4 +- app/main/controller/setup.php | 1 - app/main/cron/ccpsystemsupdate.php | 112 +++++--------------- app/main/data/mapper/crestservicestatus.php | 19 ---- app/main/data/mapper/crestusercounts.php | 18 ---- app/main/db/database.php | 19 +++- app/main/lib/web.php | 2 +- app/requirements.ini | 2 +- js/app/ui/form_element.js | 5 +- public/js/v1.2.2/app/ui/form_element.js | 5 +- 13 files changed, 60 insertions(+), 169 deletions(-) delete mode 100644 app/main/data/mapper/crestservicestatus.php delete mode 100644 app/main/data/mapper/crestusercounts.php diff --git a/app/environment.ini b/app/environment.ini index 6ae719da..4f300efa 100644 --- a/app/environment.ini +++ b/app/environment.ini @@ -26,17 +26,16 @@ DB_CCP_NAME = eve_citadel_min DB_CCP_USER = root DB_CCP_PASS = -; CCP SSO settings (OAuth2) - visit: https://developers.eveonline.com/applications +; CCP SSO (OAuth2) - visit: https://developers.eveonline.com/applications CCP_SSO_URL = https://sisilogin.testeveonline.com CCP_SSO_CLIENT_ID = CCP_SSO_SECRET_KEY = + +; CCP ESI API CCP_ESI_URL = https://esi.tech.ccp.is CCP_ESI_DATASOURCE = singularity CCP_ESI_SCOPES = esi-location.read_location.v1,esi-location.read_ship_type.v1,esi-ui.write_waypoint.v1,esi-ui.open_window.v1 -; CCP XML APIv2 -CCP_XML = https://api.testeveonline.com - ; SMTP settings (optional) SMTP_HOST = localhost SMTP_PORT = 25 @@ -72,17 +71,16 @@ DB_CCP_NAME = DB_CCP_USER = DB_CCP_PASS = -; CCP SSO settings +; CCP SSO CCP_SSO_URL = https://login.eveonline.com CCP_SSO_CLIENT_ID = CCP_SSO_SECRET_KEY = + +; CCP ESI API CCP_ESI_URL = https://esi.tech.ccp.is CCP_ESI_DATASOURCE = tranquility CCP_ESI_SCOPES = esi-location.read_location.v1,esi-location.read_ship_type.v1,esi-ui.write_waypoint.v1,esi-ui.open_window.v1 -; CCP XML APIv2 -CCP_XML = https://api.eveonline.com - ; SMTP settings (optional) SMTP_HOST = localhost SMTP_PORT = 25 diff --git a/app/main/controller/api/system.php b/app/main/controller/api/system.php index f97c9500..c384e064 100644 --- a/app/main/controller/api/system.php +++ b/app/main/controller/api/system.php @@ -130,13 +130,9 @@ class System extends Controller\AccessController { * @return array */ public function getSystems(){ - $ccpDB = $this->getDB('CCP'); - $query = $this->_getQuery(); - $rows = $ccpDB->exec($query, null, 60 * 60 * 24); - // format result $mapper = new Mapper\CcpSystemsMapper($rows); @@ -457,23 +453,3 @@ class System extends Controller\AccessController { } } - - - - - - - - - - - - - - - - - - - - diff --git a/app/main/controller/ccp/sso.php b/app/main/controller/ccp/sso.php index f64cc10c..be878ed6 100644 --- a/app/main/controller/ccp/sso.php +++ b/app/main/controller/ccp/sso.php @@ -167,7 +167,7 @@ class Sso extends Api\User{ $f3->clear(self::SESSION_KEY_SSO_STATE); $f3->clear(self::SESSION_KEY_SSO_FROM_MAP); - $accessData = $this->getCrestAccessData($getParams['code']); + $accessData = $this->getSsoAccessData($getParams['code']); if( isset($accessData->accessToken) && @@ -320,7 +320,7 @@ class Sso extends Api\User{ * @param bool $authCode * @return null|\stdClass */ - public function getCrestAccessData($authCode){ + public function getSsoAccessData($authCode){ $accessData = null; if( !empty($authCode) ){ diff --git a/app/main/controller/controller.php b/app/main/controller/controller.php index dd3e5e02..dec5af84 100644 --- a/app/main/controller/controller.php +++ b/app/main/controller/controller.php @@ -8,10 +8,8 @@ namespace Controller; use Controller\Api as Api; -use Controller\Ccp\Sso as Sso; use lib\Config; use lib\Socket; -use Lib\Util; use Model; use DB; @@ -486,7 +484,7 @@ class Controller { } /** - * get EVE server status from CREST + * get EVE server status from ESI * @param \Base $f3 */ public function getEveServerStatus(\Base $f3){ diff --git a/app/main/controller/setup.php b/app/main/controller/setup.php index f59e2e0c..be917841 100644 --- a/app/main/controller/setup.php +++ b/app/main/controller/setup.php @@ -39,7 +39,6 @@ class Setup extends Controller { 'CCP_SSO_SECRET_KEY', 'CCP_ESI_URL', 'CCP_ESI_DATASOURCE', - 'CCP_XML', 'SMTP_HOST', 'SMTP_PORT', 'SMTP_SCHEME', diff --git a/app/main/cron/ccpsystemsupdate.php b/app/main/cron/ccpsystemsupdate.php index d6177d60..28a6c692 100644 --- a/app/main/cron/ccpsystemsupdate.php +++ b/app/main/cron/ccpsystemsupdate.php @@ -9,7 +9,6 @@ namespace Cron; use Controller; use DB; -use lib\Config; class CcpSystemsUpdate { @@ -26,9 +25,9 @@ class CcpSystemsUpdate { */ protected $logTables = [ 'jumps' => 'system_jumps', - 'shipKills' => 'system_kills_ships', - 'podKills' => 'system_kills_pods', - 'factionKills' => 'system_kills_factions' + 'ship_kills' => 'system_kills_ships', + 'pod_kills' => 'system_kills_pods', + 'npc_kills' => 'system_kills_factions' ]; /** @@ -74,77 +73,37 @@ class CcpSystemsUpdate { */ function importSystemData($f3){ + // prepare system jump log table ------------------------------------------------------------------------------ $time_start = microtime(true); - // prepare system jump log table $systemsData = $this->prepareSystemLogTables(); $time_end = microtime(true); $execTimePrepareSystemLogTables = $time_end - $time_start; + // switch DB for data import.. $pfDB = DB\Database::instance()->getDB('PF'); - // get current jump Data ------------------------------------------------------- + // get current jump data -------------------------------------------------------------------------------------- $time_start = microtime(true); - $apiPath = Config::getEnvironmentData('CCP_XML') . '/map/Jumps.xml.aspx'; - - $apiResponse = \Web::instance()->request($apiPath, $this->apiRequestOptions ); - - $jumpData = []; - $updateJumps = false; - if($apiResponse['body']){ - $xml = simplexml_load_string($apiResponse['body']); - $rowApiData = $xml->result->rowset; - - foreach($rowApiData->children() as $systemApiData){ - $attributeApiData = $systemApiData->attributes(); - $systemId = $attributeApiData->solarSystemID->__toString(); - $shipJumps =$attributeApiData->shipJumps->__toString(); - - $jumpData[$systemId] = $shipJumps; - } - - $updateJumps = true; - } + $jumpData = $f3->ccpClient->getUniverseJumps(); $time_end = microtime(true); $execTimeGetJumpData = $time_end - $time_start; - // get current kill Data ------------------------------------------------------- + // get current kill data -------------------------------------------------------------------------------------- $time_start = microtime(true); - $apiPath = Config::getEnvironmentData('CCP_XML') . '/map/Kills.xml.aspx'; - - $apiResponse = \Web::instance()->request($apiPath, $this->apiRequestOptions ); - $killData = []; - $updateKills = false; - if($apiResponse['body']){ - $xml = simplexml_load_string($apiResponse['body']); - $rowApiData = $xml->result->rowset; - foreach($rowApiData->children() as $systemApiData){ - $attributeApiData = $systemApiData->attributes(); - $systemId = $attributeApiData->solarSystemID->__toString(); - $shipKills =$attributeApiData->shipKills->__toString(); - $podKills =$attributeApiData->podKills->__toString(); - $factionKills =$attributeApiData->factionKills->__toString(); - - $killData[$systemId] = [ - 'shipKills' => $shipKills, - 'podKills' => $podKills, - 'factionKills' => $factionKills, - ]; - } - - $updateKills = true; - - } + $killData = $f3->ccpClient->getUniverseKills(); $time_end = microtime(true); $execTimeGetKillData = $time_end - $time_start; - // update system log tables ----------------------------------------------------- + // merge both results + $systemValues = array_replace_recursive($jumpData, $killData); + + // update system log tables ----------------------------------------------------------------------------------- $time_start = microtime(true); - // make sure last update is (at least) 1h ago $pfDB->begin(); foreach($this->logTables as $key => $tableName){ $sql = "UPDATE - " . $tableName . " + $tableName SET updated = now(), value24 = value23, @@ -173,47 +132,30 @@ class CcpSystemsUpdate { value1 = :value WHERE systemId = :systemId - "; + "; foreach($systemsData as $systemData){ + $systemId = $systemData['systemId']; - if( - $key == 'jumps' && - $updateJumps - ){ - // update jump data (if available) - $currentJumps = 0; - if(array_key_exists($systemData['systemId'], $jumpData)){ - $currentJumps = $jumpData[$systemData['systemId']]; - } - - $pfDB->exec($sql, array( - ':systemId' => $systemData['systemId'], - ':value' => $currentJumps - ), 0, false); - }else if($updateKills){ - - // update kill data (if available) - $currentKills = 0; - if(array_key_exists($systemData['systemId'], $killData)){ - $currentKillData = $killData[$systemData['systemId']]; - - $currentKills = $currentKillData[$key]; - } - - $pfDB->exec($sql, array( - ':systemId' => $systemData['systemId'], - ':value' => $currentKills - ), 0, false); + // update data (if available) + $currentData = 0; + if( isset($systemValues[$systemId][$key]) ){ + $currentData = (int)$systemValues[$systemId][$key]; } + + $pfDB->exec($sql, [ + ':systemId' => $systemId, + ':value' => $currentData + ], 0, false); } } + $pfDB->commit(); $time_end = microtime(true); $execTimeUpdateTables = $time_end - $time_start; - // Log ------------------------ + // Log -------------------------------------------------------------------------------------------------------- $log = new \Log('cron_' . __FUNCTION__ . '.log'); $log->write( sprintf(self::LOG_TEXT, __FUNCTION__, $execTimePrepareSystemLogTables, $execTimeGetJumpData, $execTimeGetKillData, $execTimeUpdateTables) ); } diff --git a/app/main/data/mapper/crestservicestatus.php b/app/main/data/mapper/crestservicestatus.php deleted file mode 100644 index 07b39a76..00000000 --- a/app/main/data/mapper/crestservicestatus.php +++ /dev/null @@ -1,19 +0,0 @@ - 'dust', - 'eve' => 'eve', - 'server' => 'server' - ]; -} \ No newline at end of file diff --git a/app/main/data/mapper/crestusercounts.php b/app/main/data/mapper/crestusercounts.php deleted file mode 100644 index 7ed58b7b..00000000 --- a/app/main/data/mapper/crestusercounts.php +++ /dev/null @@ -1,18 +0,0 @@ - 'dust', - 'eve' => 'eve' - ]; -} \ No newline at end of file diff --git a/app/main/db/database.php b/app/main/db/database.php index baf3433b..bfda794f 100644 --- a/app/main/db/database.php +++ b/app/main/db/database.php @@ -98,17 +98,26 @@ class Database extends \Prefab { */ protected function connect($dns, $name, $user, $password){ $db = null; + $f3 = \Base::instance(); + + $options = [ + \PDO::MYSQL_ATTR_COMPRESS => true, + \PDO::ATTR_TIMEOUT => \Base::instance()->get('REQUIREMENTS.MYSQL.PDO_TIMEOUT'), + ]; + + // set ERRMODE depending on pathfinders global DEBUG level + if($f3->get('DEBUG') >= 3){ + $options[\PDO::ATTR_ERRMODE] = \PDO::ERRMODE_WARNING; + }else{ + $options[\PDO::ATTR_ERRMODE] = \PDO::ERRMODE_EXCEPTION; + } try { $db = new SQL( $dns . $name, $user, $password, - [ - \PDO::MYSQL_ATTR_COMPRESS => true, - \PDO::ATTR_ERRMODE => \PDO::ERRMODE_EXCEPTION, - \PDO::ATTR_TIMEOUT => \Base::instance()->get('REQUIREMENTS.MYSQL.PDO_TIMEOUT'), - ] + $options ); }catch(\PDOException $e){ // DB connection error diff --git a/app/main/lib/web.php b/app/main/lib/web.php index 9525599a..298a6af5 100644 --- a/app/main/lib/web.php +++ b/app/main/lib/web.php @@ -98,7 +98,7 @@ class Web extends \Web { * @param string $url * @param array|null $options * @param array $additionalOptions - * @param int $retryCount request counter for failed crest call + * @param int $retryCount request counter for failed call * @return array|FALSE|mixed */ public function request($url,array $options = null, $additionalOptions = [], $retryCount = 0 ) { diff --git a/app/requirements.ini b/app/requirements.ini index 36f727dc..10bd9834 100644 --- a/app/requirements.ini +++ b/app/requirements.ini @@ -39,7 +39,7 @@ ZMQ = 4.1.3 [REQUIREMENTS.MYSQL] ; min MySQL Version ; newer "deviation" of MySQL like "MariaDB" > 10.1 are recommended -VERSION = 5.6 +VERSION = 5.7 ; DB timeout (seconds) PDO_TIMEOUT = 2 diff --git a/js/app/ui/form_element.js b/js/app/ui/form_element.js index 0d8b5908..649fa592 100644 --- a/js/app/ui/form_element.js +++ b/js/app/ui/form_element.js @@ -148,7 +148,10 @@ define([ }).on('select2:open', function(){ // clear selected system (e.g. default system) // => improves usability (not necessary). There is a small "x" whe it could be cleared manually - if($(this).val() !== null){ + if( + options.maxSelectionLength === 1 && + $(this).val() !== null + ){ $(this).val('').trigger('change'); } }) diff --git a/public/js/v1.2.2/app/ui/form_element.js b/public/js/v1.2.2/app/ui/form_element.js index 0d8b5908..649fa592 100644 --- a/public/js/v1.2.2/app/ui/form_element.js +++ b/public/js/v1.2.2/app/ui/form_element.js @@ -148,7 +148,10 @@ define([ }).on('select2:open', function(){ // clear selected system (e.g. default system) // => improves usability (not necessary). There is a small "x" whe it could be cleared manually - if($(this).val() !== null){ + if( + options.maxSelectionLength === 1 && + $(this).val() !== null + ){ $(this).val('').trigger('change'); } })