- improved "Redis" cache handler. Switched to persistent connection

- improved "Redis" info panels on `/setup` page, added "connected clients" and "blocked clients" data - improved DB connection. Switched to persistent connection (configurable in `pathfinder.ini`) - fixed some PHP errors after "Cortex" upgrade
2019-03-24 15:42:49 +01:00
parent af59235b73
commit 66e843a479
16 changed files with 82 additions and 30 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -56,3 +56,4 @@ Temporary Items
 /public/js/vX.X.X/
 /vendor/
 /history/
+/package-lock.json
--- a/app/main/controller/api/setup.php
+++ b/app/main/controller/api/setup.php
@@ -76,9 +76,10 @@ class Setup extends Controller\Controller {
                $buildInfo = $controller->setupCategory($categoryId, $offset, $length);

                $categoryUniverseModel =  Model\Universe\BasicUniverseModel::getNew('CategoryModel');
+                $categoryUniverseModel->getById($categoryId, 0);
                $return->countAll = (int)$f3->get('REQUIREMENTS.DATA.STRUCTURES');
                $return->countBuild = array_reduce($buildInfo, $sum, 0);
-                $return->countBuildAll = $categoryUniverseModel->getById($categoryId, 0)->getTypesCount(false);
+                $return->countBuildAll = $categoryUniverseModel->getTypesCount(false);
                $return->progress = $percent($return->countAll, $return->countBuildAll);
                break;
            case 'Ships':
@@ -88,9 +89,10 @@ class Setup extends Controller\Controller {
                $buildInfo = $controller->setupCategory($categoryId, $offset, $length);

                $categoryUniverseModel =  Model\Universe\BasicUniverseModel::getNew('CategoryModel');
+                $categoryUniverseModel->getById($categoryId, 0);
                $return->countAll = (int)$f3->get('REQUIREMENTS.DATA.SHIPS');
                $return->countBuild = array_reduce($buildInfo, $sum, 0);
-                $return->countBuildAll = $categoryUniverseModel->getById($categoryId, 0)->getTypesCount(false);
+                $return->countBuildAll = $categoryUniverseModel->getTypesCount(false);
                $return->progress = $percent($return->countAll, $return->countBuildAll);
                break;
            case 'SystemNeighbour':
--- a/app/main/controller/controller.php
+++ b/app/main/controller/controller.php
@@ -152,7 +152,6 @@ class Controller {
                new DB\SQL\MySQL\Session($this->getDB('PF'), 'sessions', true, $onSuspect);
            }
        }
-
    }

    /**
--- a/app/main/controller/setup.php
+++ b/app/main/controller/setup.php
@@ -747,6 +747,7 @@ class Setup extends Controller {

                if($client->isConnected()){
                    $redisServerInfo = (array)$client->info('SERVER');
+                    $redisClientsInfo = (array)$client->info('CLIENTS');
                    $redisMemoryInfo = (array)$client->info('MEMORY');
                    $redisStatsInfo = (array)$client->info('STATS');

@@ -784,6 +785,18 @@ class Setup extends Controller {
                            'check' => $redisMemoryInfo['maxmemory_policy'] == $f3->get('REQUIREMENTS.REDIS.MAXMEMORY_POLICY'),
                            'tooltip' => 'How Redis behaves if \'maxmemory\' limit reached'
                        ],
+                        'connectedClients' => [
+                            'label' => 'connected_clients',
+                            'version' => $redisClientsInfo['connected_clients'],
+                            'check' => (bool)$redisClientsInfo['connected_clients'],
+                            'tooltip' => 'Number of client connections (excluding connections from replicas)'
+                        ],
+                        'blockedClients' => [
+                            'label' => 'blocked_clients',
+                            'version' => $redisClientsInfo['blocked_clients'],
+                            'check' => !(bool)$redisClientsInfo['blocked_clients'],
+                            'tooltip' => 'Number of clients pending on a blocking call (BLPOP, BRPOP, BRPOPLPUSH)'
+                        ],
                        'evictedKeys' => [
                            'label' => 'evicted_keys',
                            'version' => $redisStatsInfo['evicted_keys'],
@@ -846,7 +859,7 @@ class Setup extends Controller {
                    $client = new \Redis();

                    try{
-                        $client->connect($conf['host'], $conf['port'], 0.3);
+                        $client->pconnect($conf['host'], $conf['port'], 0.3);
                        if(isset($conf['db'])) {
                            $client->select($conf['db']);
                        }
@@ -1077,6 +1090,8 @@ class Setup extends Controller {

            // DB connection status
            $dbConnected = false;
+            // DB initialized as persistent connection
+            $dbPersistent = false;
            // DB type (e.g. MySql,..)
            $dbDriver = 'unknown';
            // enable database ::create() function on UI
@@ -1130,6 +1145,7 @@ class Setup extends Controller {

                // db connect was successful
                $dbConnected = true;
+                $dbPersistent = $db->pdo()->getAttribute(\PDO::ATTR_PERSISTENT);
                $dbDriver = $db->driver();
                $dbConfig = $this->checkDBConfig($f3, $db);

@@ -1377,6 +1393,7 @@ class Setup extends Controller {
                'dbCreate'          => $dbCreate,
                'setupEnable'       => $dbSetupEnable,
                'connected'         => $dbConnected,
+                'persistent'        => $dbPersistent,
                'statusCheckCount'  => $dbStatusCheckCount,
                'columnQueries'     => $dbColumnQueries,
                'tableData'         => $requiredTables,
@@ -1820,7 +1837,7 @@ class Setup extends Controller {
     */
    protected function flushRedisDb(string $host, int $port, int $db = 0){
        $client = new \Redis();
-        $client->connect($host, $port, 0.3);
+        $client->pconnect($host, $port, 0.3);
        $client->select($db);
        $client->flushDB();
        $client->close();
--- a/app/main/cron/ccpsystemsupdate.php
+++ b/app/main/cron/ccpsystemsupdate.php
@@ -13,11 +13,6 @@ class CcpSystemsUpdate extends AbstractCron {

    const LOG_TEXT = '%s prepare table (%.3F s), jump (%.3F s), kill (%.3F s), update all (%.3F s)';

-    protected  $apiRequestOptions = [
-        'timeout' => 5,
-        'follow_location' => false // otherwise CURLOPT_FOLLOWLOCATION will fail
-    ];
-
    /**
     * table names for all system log tables
     * @var array
--- a/app/main/db/database.php
+++ b/app/main/db/database.php
@@ -130,15 +130,19 @@ class Database extends \Prefab {
        $f3 = \Base::instance();

        $options = [
-            \PDO::MYSQL_ATTR_COMPRESS  => true,
-            \PDO::ATTR_TIMEOUT => \Base::instance()->get('REQUIREMENTS.MYSQL.PDO_TIMEOUT'),
+            \PDO::MYSQL_ATTR_COMPRESS       => true,
+            \PDO::ATTR_TIMEOUT              => \Base::instance()->get('REQUIREMENTS.MYSQL.PDO_TIMEOUT'),
        ];

+        if(Config::getPathfinderData('experiments.persistent_db_connections')){
+            $options[\PDO::ATTR_PERSISTENT] = true;
+        }
+
        // set ERRMODE depending on pathfinders global DEBUG level
        if($f3->get('DEBUG') >= 1){
-            $options[\PDO::ATTR_ERRMODE] = \PDO::ERRMODE_WARNING;
+            $options[\PDO::ATTR_ERRMODE]    = \PDO::ERRMODE_WARNING;
        }else{
-            $options[\PDO::ATTR_ERRMODE] = \PDO::ERRMODE_EXCEPTION;
+            $options[\PDO::ATTR_ERRMODE]    = \PDO::ERRMODE_EXCEPTION;
        }

        try {
--- a/app/main/lib/api/AbstractClient.php
+++ b/app/main/lib/api/AbstractClient.php
@@ -107,7 +107,7 @@ abstract class AbstractClient extends \Prefab {

        return function() use ($poolConfig) : ?CacheItemPoolInterface {
            // an active CachePool should be re-used
-            // -> no need for e.g. a new Redis->connect()
+            // -> no need for e.g. a new Redis->pconnect()
            //    and/or re-init when it is used the next time
            if(!is_null($this->cachePool)){
                return $this->cachePool;
@@ -122,7 +122,7 @@ abstract class AbstractClient extends \Prefab {
            ){
                $client = new \Redis();
                if(
-                    $client->connect(
+                    $client->pconnect(
                        $poolConfig['host'],
                        $poolConfig['port'],
                        Config::REDIS_OPT_TIMEOUT,
@@ -141,6 +141,8 @@ abstract class AbstractClient extends \Prefab {
                    //    This helps to separate keys by a namespace
                    // @see http://www.php-cache.com/en/latest/
                    $this->cachePool = new NamespacedCachePool($poolRedis, static::CLIENT_NAME);
+
+                    register_shutdown_function([$this,'unloadCache'], $client);
                }
            }

@@ -264,6 +266,16 @@ abstract class AbstractClient extends \Prefab {
        return $config;
    }

+    /**
+     * unload function
+     * @param \Redis $client
+     */
+    public function unloadCache(\Redis $client){
+        if($client->isConnected()){
+            $client->close();
+        }
+    }
+
    /**
     * call request API data
     * @param string $name
--- a/app/main/lib/config.php
+++ b/app/main/lib/config.php
@@ -240,7 +240,7 @@ class Config extends \Prefab {
     * @param string $dbKey
     * @return array
     */
-    static function getDatabaseConfig(string $dbKey  = 'PF'){
+    static function getDatabaseConfig(string $dbKey  = 'PF') : array {
        $dbKey = strtoupper($dbKey);
        return [
            'DNS'   => self::getEnvironmentData('DB_' . $dbKey . '_DNS'),
--- a/app/main/model/systemmodel.php
+++ b/app/main/model/systemmodel.php
@@ -32,7 +32,7 @@ class SystemModel extends AbstractMapTrackingModel {
    /**
     * TTL for history signature data
     */
-    const TTL_HISTORY_SIGNATURES        = 5000;
+    const TTL_HISTORY_SIGNATURES        = 7200;

    /**
     * cache key prefix for getData(); result WITH log data
--- a/app/main/model/universe/basicuniversemodel.php
+++ b/app/main/model/universe/basicuniversemodel.php
@@ -153,9 +153,9 @@ abstract class BasicUniverseModel extends BasicModel {
        /**
         * @var $model self
         */
-        $model = $this->getById($id, 0);
-        if($model->isOutdated()){
-            $model->loadData($id, $accessToken, $additionalOptions);
+        $this->getById($id, 0);
+        if($this->isOutdated()){
+            $this->loadData($id, $accessToken, $additionalOptions);
        }
    }

@@ -173,7 +173,7 @@ abstract class BasicUniverseModel extends BasicModel {
     * @param $value
     * @return string
     */
-    public static function generateHashKeyRow(string $table, $value){
+    public static function generateHashKeyRow(string $table, $value) : string {
        return self::generateHashKeyTable($table) . '_' .  md5(strtolower((string)$value));
    }

@@ -183,7 +183,7 @@ abstract class BasicUniverseModel extends BasicModel {
     * @param string $table
     * @return string
     */
-    public static function generateHashKeyTable(string $table){
+    public static function generateHashKeyTable(string $table) : string {
        return self::CACHE_KEY_PREFIX . strtolower($table);
    }

--- a/app/pathfinder.ini
+++ b/app/pathfinder.ini
@@ -378,3 +378,15 @@ CCP_IMAGE_SERVER            =   https://image.eveonline.com
 Z_KILLBOARD                 =   https://zkillboard.com/api
 ; GitHub Developer API
 GIT_HUB                     =   https://api.github.com
+
+; EXPERIMENTAL [BETA] =============================================================================
+; Use these settings with caution!
+; They are currently under testing and might be removed in further releases.
+[PATHFINDER.EXPERIMENTS]
+; Try to use persistent database connections
+;           PDO connections get initialized with ATTR_PERSISTENT => true .
+;           http://php.net/manual/en/pdo.connections.php#example-1030
+; Hint:     Set 'wait_timeout' to a high value in your my.conf to keep them open
+; Syntax:   0 | 1
+; Default:  0
+PERSISTENT_DB_CONNECTIONS   =   0
--- a/public/css/v1.5.1/pathfinder.css
+++ b/public/css/v1.5.1/pathfinder.css
--- a/public/css/v1.5.1/pathfinder.css.map
+++ b/public/css/v1.5.1/pathfinder.css.map
--- a/public/templates/modules/header.html
+++ b/public/templates/modules/header.html
@@ -39,7 +39,7 @@
                <i class="fas fa-fw fa-bolt"></i>
                <span>offline</span>
            </p>
-            <p class="navbar-text" title="map connection tracking">
+            <p class="navbar-text pf-head-map-tracking" title="map connection tracking">
                <i class="fas fa-fw fa-location-arrow"></i>
                <input id="{{mapTrackingId}}" type="checkbox" data-toggle="toggle" value="1" >
            </p>
--- a/public/templates/view/setup.html
+++ b/public/templates/view/setup.html
@@ -472,11 +472,12 @@
                                </tr>
                                <tr>
                                    <td>DB status</td>
-                                    <td class="text-right col-sm-3 col-md-3"></td>
-                                    <td class="text-right col-sm-3 col-md-3">
+                                    <td class="text-right col-sm-3 col-md-3" colspan="2">
                                        <check if="{{ @dbInformation.info.connected }}">
                                            <true>
-                                                <kbd class="txt-color txt-color-success">connected</kbd>
+                                                <kbd class="txt-color txt-color-success">
+                                                    connected<check if="{{ @dbInformation.info.persistent }}"> (persistent)</check>
+                                                </kbd>
                                            </true>
                                            <false>
                                                <kbd class="txt-color txt-color-danger">failed</kbd>
--- a/sass/layout/_main.scss
+++ b/sass/layout/_main.scss
@@ -716,7 +716,7 @@ table{

 // maps module ====================================================================================
 #pf-map-module{
-  margin: 20px 10px 0 10px;
+  margin: 15px 10px 0 10px;

  // Tabs (colored)
  #pf-map-tabs {
@@ -776,6 +776,10 @@ table{
  margin-top: 10px;
  padding-bottom: 40px; // space for footer

+  >.pf-map-content-col:first-child{
+    padding-right: 0; // overwrite default
+  }
+
  // all modules within a row
  .pf-module{
    font-family: $font-family-bold;
@@ -1501,6 +1505,11 @@ code {
    cursor: pointer;
  }

+  .pf-head-map-tracking{
+    margin-bottom: 4px;
+    margin-top: 4px;
+  }
+
  .navbar-text{
    min-width: 60px; // fixes a load-delay issue for "toggle" map-tracking
  }