- New "Intel module" for Citadel tracking, closed #246

- added some new tables (for SDE replacement), #628
- new "index build" functions added to `/setup`, #628
- updated "Cortex" PHP ORM lib `1.5.0` -> `1.6.0-dev`

app/main/model/basicmodel.php

@@ -111,7 +111,7 @@ abstract class BasicModel extends \DB\Cortex {
 
     const ERROR_INVALID_MODEL_CLASS                             = 'Model class (%s) not found';
 
-    public function __construct($db = NULL, $table = NULL, $fluid = NULL, $ttl = 0){
+    public function __construct($db = NULL, $table = NULL, $fluid = NULL, $ttl = self::DEFAULT_TTL){
 
         $this->addStaticFieldConfig();
 
@@ -318,6 +318,32 @@ abstract class BasicModel extends \DB\Cortex {
         return $valid;
     }
 
+    /**
+     * validates a model field to be a valid relational model
+     * @param $key
+     * @param $val
+     * @return bool
+     * @throws \Exception\ValidationException
+     */
+    protected function validate_notDry($key, $val): bool {
+        $valid = true;
+        if($colConf = $this->fieldConf[$key]){
+            if(isset($colConf['belongs-to-one'])){
+                if( (is_int($val) || ctype_digit($val)) && (int)$val > 0){
+                    $valid = true;
+                }elseif( is_a($val, $colConf['belongs-to-one']) && !$val->dry() ){
+                    $valid = true;
+                }else{
+                    $valid = false;
+                    $msg = 'Validation failed: "' . get_class($this) . '->' . $key . '" must be a valid instance of ' . $colConf['belongs-to-one'];
+                    $this->throwValidationException($key, $msg);
+                }
+            }
+        }
+
+        return $valid;
+    }
+
     /**
      * get key for for all objects in this table
      * @return string
@@ -619,15 +645,21 @@ abstract class BasicModel extends \DB\Cortex {
     /**
      * export and download table data as *.csv
      * this is primarily used for static tables
+     * @param array $fields
      * @return bool
      */
-    public function exportData(){
+    public function exportData(array $fields = []){
         $status = false;
 
         if(static::$enableDataExport){
             $tableModifier = static::getTableModifier();
             $headers = $tableModifier->getCols();
 
+            if($fields){
+                // columns to export -> reIndex keys
+                $headers = array_values(array_intersect($headers, $fields));
+            }
+
             // just get the records with existing columns
             // -> no "virtual" fields or "new" columns
             $this->fields($headers);
@@ -659,14 +691,16 @@ abstract class BasicModel extends \DB\Cortex {
 
     /**
      * import table data from a *.csv file
-     * @return bool
+     * @return array|bool
      */
     public function importData(){
         $status = false;
 
         // rtrim(); for arrays (removes empty values) from the end
-        $rtrim = function($array = []){
-          return array_slice($array, 0, key(array_reverse($array, 1))+1);
+        $rtrim = function($array = [], $lengthMin = false){
+            $length = key(array_reverse(array_diff($array, ['']), 1))+1;
+            $length = $length < $lengthMin ? $lengthMin : $length;
+            return array_slice($array, 0, $length);
         };
 
         if(static::$enableDataImport){
@@ -680,7 +714,7 @@ abstract class BasicModel extends \DB\Cortex {
                 if(count($keys) > 0){
                     $tableData = [];
                     while (!feof($handle)) {
-                        $tableData[] = array_combine($keys, $rtrim(fgetcsv($handle, 0, ';')));
+                        $tableData[] = array_combine($keys, $rtrim(fgetcsv($handle, 0, ';'), count($keys)));
                     }
                     // import row data
                     $status = $this->importStaticData($tableData);
@@ -704,20 +738,22 @@ abstract class BasicModel extends \DB\Cortex {
      */
     protected function importStaticData($tableData = []){
         $rowIDs = [];
-        $columnNames = array_merge(['id'], array_keys($this->fieldConf));
         $addedCount = 0;
         $updatedCount = 0;
         $deletedCount = 0;
 
+        $tableModifier = static::getTableModifier();
+        $fields = $tableModifier->getCols();
+
         foreach($tableData as $rowData){
             // search for existing record and update columns
-            $this->getById($rowData['id']);
+            $this->getById($rowData['id'], 0);
             if($this->dry()){
                 $addedCount++;
             }else{
                 $updatedCount++;
             }
-            $this->copyfrom($rowData, $columnNames);
+            $this->copyfrom($rowData, $fields);
             $this->save();
             $rowIDs[] = $this->id;
             $this->reset();
@@ -845,6 +881,21 @@ abstract class BasicModel extends \DB\Cortex {
         return \Base::instance();
     }
 
+    /**
+     * stores data direct into the Cache backend (e.g. Redis)
+     * $f3->set() used the same code. The difference is, that $f3->set()
+     * also loads data into the Hive.
+     * This can result in high RAM usage if a great number of key->values should be stored in Cache
+     * (like the search index for system data)
+     * @param string $key
+     * @param $data
+     * @param int $ttl
+     */
+    public static function setCacheValue(string $key, $data, int $ttl = 0){
+        $cache = \Cache::instance();
+        $cache->set(self::getF3()->hash($key).'.var', $data, $ttl);
+    }
+
     /**
      * debug log function
      * @param string $text