|
|
|
@@ -6,13 +6,11 @@ use App\Jobs\CheckAndStartSentinelJob;
|
|
|
|
use App\Jobs\CheckForUpdatesJob;
|
|
|
|
use App\Jobs\CheckForUpdatesJob;
|
|
|
|
use App\Jobs\CheckHelperImageJob;
|
|
|
|
use App\Jobs\CheckHelperImageJob;
|
|
|
|
use App\Jobs\CleanupInstanceStuffsJob;
|
|
|
|
use App\Jobs\CleanupInstanceStuffsJob;
|
|
|
|
use App\Jobs\CleanupStaleMultiplexedConnections;
|
|
|
|
|
|
|
|
use App\Jobs\DatabaseBackupJob;
|
|
|
|
use App\Jobs\DatabaseBackupJob;
|
|
|
|
use App\Jobs\DockerCleanupJob;
|
|
|
|
use App\Jobs\DockerCleanupJob;
|
|
|
|
use App\Jobs\PullTemplatesFromCDN;
|
|
|
|
use App\Jobs\PullTemplatesFromCDN;
|
|
|
|
use App\Jobs\ScheduledTaskJob;
|
|
|
|
use App\Jobs\ScheduledTaskJob;
|
|
|
|
use App\Jobs\ServerCheckJob;
|
|
|
|
use App\Jobs\ServerCheckJob;
|
|
|
|
use App\Jobs\ServerCleanupMux;
|
|
|
|
|
|
|
|
use App\Jobs\ServerStorageCheckJob;
|
|
|
|
use App\Jobs\ServerStorageCheckJob;
|
|
|
|
use App\Jobs\UpdateCoolifyJob;
|
|
|
|
use App\Jobs\UpdateCoolifyJob;
|
|
|
|
use App\Models\InstanceSettings;
|
|
|
|
use App\Models\InstanceSettings;
|
|
|
|
@@ -23,6 +21,7 @@ use App\Models\Team;
|
|
|
|
use Illuminate\Console\Scheduling\Schedule;
|
|
|
|
use Illuminate\Console\Scheduling\Schedule;
|
|
|
|
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
|
|
|
|
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
|
|
|
|
use Illuminate\Support\Carbon;
|
|
|
|
use Illuminate\Support\Carbon;
|
|
|
|
|
|
|
|
use Illuminate\Support\Facades\Log;
|
|
|
|
|
|
|
|
|
|
|
|
class Kernel extends ConsoleKernel
|
|
|
|
class Kernel extends ConsoleKernel
|
|
|
|
{
|
|
|
|
{
|
|
|
|
@@ -99,10 +98,14 @@ class Kernel extends ConsoleKernel
|
|
|
|
$servers = $this->allServers->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_reachable', true)->get();
|
|
|
|
$servers = $this->allServers->whereRelation('settings', 'is_usable', true)->whereRelation('settings', 'is_reachable', true)->get();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
foreach ($servers as $server) {
|
|
|
|
foreach ($servers as $server) {
|
|
|
|
if ($server->isSentinelEnabled()) {
|
|
|
|
try {
|
|
|
|
$this->scheduleInstance->job(function () use ($server) {
|
|
|
|
if ($server->isSentinelEnabled()) {
|
|
|
|
CheckAndStartSentinelJob::dispatch($server);
|
|
|
|
$this->scheduleInstance->job(function () use ($server) {
|
|
|
|
})->cron($this->updateCheckFrequency)->timezone($this->instanceTimezone)->onOneServer();
|
|
|
|
CheckAndStartSentinelJob::dispatch($server);
|
|
|
|
|
|
|
|
})->cron($this->updateCheckFrequency)->timezone($this->instanceTimezone)->onOneServer();
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
} catch (\Exception $e) {
|
|
|
|
|
|
|
|
Log::error('Error pulling images: '.$e->getMessage());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
$this->scheduleInstance->job(new CheckHelperImageJob)
|
|
|
|
$this->scheduleInstance->job(new CheckHelperImageJob)
|
|
|
|
@@ -138,35 +141,47 @@ class Kernel extends ConsoleKernel
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
foreach ($servers as $server) {
|
|
|
|
foreach ($servers as $server) {
|
|
|
|
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
|
|
|
|
try {
|
|
|
|
if (validate_timezone($serverTimezone) === false) {
|
|
|
|
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
|
|
|
|
$serverTimezone = config('app.timezone');
|
|
|
|
if (validate_timezone($serverTimezone) === false) {
|
|
|
|
}
|
|
|
|
$serverTimezone = config('app.timezone');
|
|
|
|
|
|
|
|
|
|
|
|
// Sentinel check
|
|
|
|
|
|
|
|
$lastSentinelUpdate = $server->sentinel_updated_at;
|
|
|
|
|
|
|
|
if (Carbon::parse($lastSentinelUpdate)->isBefore(now()->subSeconds($server->waitBeforeDoingSshCheck()))) {
|
|
|
|
|
|
|
|
// Check container status every minute if Sentinel does not activated
|
|
|
|
|
|
|
|
if (isCloud()) {
|
|
|
|
|
|
|
|
$this->scheduleInstance->job(new ServerCheckJob($server))->timezone($serverTimezone)->everyFiveMinutes()->onOneServer();
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
$this->scheduleInstance->job(new ServerCheckJob($server))->timezone($serverTimezone)->everyMinute()->onOneServer();
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// $this->scheduleInstance->job(new \App\Jobs\ServerCheckNewJob($server))->everyFiveMinutes()->onOneServer();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
$this->scheduleInstance->job(new ServerStorageCheckJob($server))->cron($server->settings->server_disk_usage_check_frequency)->timezone($serverTimezone)->onOneServer();
|
|
|
|
// Sentinel check
|
|
|
|
}
|
|
|
|
$lastSentinelUpdate = $server->sentinel_updated_at;
|
|
|
|
|
|
|
|
if (Carbon::parse($lastSentinelUpdate)->isBefore(now()->subSeconds($server->waitBeforeDoingSshCheck()))) {
|
|
|
|
|
|
|
|
// Check container status every minute if Sentinel does not activated
|
|
|
|
|
|
|
|
if (isCloud()) {
|
|
|
|
|
|
|
|
$this->scheduleInstance->job(new ServerCheckJob($server))->timezone($serverTimezone)->everyFiveMinutes()->onOneServer();
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
$this->scheduleInstance->job(new ServerCheckJob($server))->timezone($serverTimezone)->everyMinute()->onOneServer();
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
// $this->scheduleInstance->job(new \App\Jobs\ServerCheckNewJob($server))->everyFiveMinutes()->onOneServer();
|
|
|
|
|
|
|
|
|
|
|
|
$this->scheduleInstance->job(new DockerCleanupJob($server))->cron($server->settings->docker_cleanup_frequency)->timezone($serverTimezone)->onOneServer();
|
|
|
|
$serverDiskUsageCheckFrequency = data_get($server->settings, 'server_disk_usage_check_frequency', '0 * * * *');
|
|
|
|
|
|
|
|
if (isset(VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency])) {
|
|
|
|
|
|
|
|
$serverDiskUsageCheckFrequency = VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency];
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
$this->scheduleInstance->job(new ServerStorageCheckJob($server))->cron($serverDiskUsageCheckFrequency)->timezone($serverTimezone)->onOneServer();
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Cleanup multiplexed connections every hour
|
|
|
|
$dockerCleanupFrequency = data_get($server->settings, 'docker_cleanup_frequency', '0 * * * *');
|
|
|
|
// $this->scheduleInstance->job(new ServerCleanupMux($server))->hourly()->onOneServer();
|
|
|
|
if (isset(VALID_CRON_STRINGS[$dockerCleanupFrequency])) {
|
|
|
|
|
|
|
|
$dockerCleanupFrequency = VALID_CRON_STRINGS[$dockerCleanupFrequency];
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
$this->scheduleInstance->job(new DockerCleanupJob($server))->cron($dockerCleanupFrequency)->timezone($serverTimezone)->onOneServer();
|
|
|
|
|
|
|
|
|
|
|
|
// Temporary solution until we have better memory management for Sentinel
|
|
|
|
// Cleanup multiplexed connections every hour
|
|
|
|
if ($server->isSentinelEnabled()) {
|
|
|
|
// $this->scheduleInstance->job(new ServerCleanupMux($server))->hourly()->onOneServer();
|
|
|
|
$this->scheduleInstance->job(function () use ($server) {
|
|
|
|
|
|
|
|
$server->restartContainer('coolify-sentinel');
|
|
|
|
// Temporary solution until we have better memory management for Sentinel
|
|
|
|
})->daily()->onOneServer();
|
|
|
|
if ($server->isSentinelEnabled()) {
|
|
|
|
|
|
|
|
$this->scheduleInstance->job(function () use ($server) {
|
|
|
|
|
|
|
|
$server->restartContainer('coolify-sentinel');
|
|
|
|
|
|
|
|
})->daily()->onOneServer();
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
} catch (\Exception $e) {
|
|
|
|
|
|
|
|
Log::error('Error checking resources: '.$e->getMessage());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
@@ -200,24 +215,28 @@ class Kernel extends ConsoleKernel
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
foreach ($finalScheduledBackups as $scheduled_backup) {
|
|
|
|
foreach ($finalScheduledBackups as $scheduled_backup) {
|
|
|
|
if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) {
|
|
|
|
try {
|
|
|
|
$scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency];
|
|
|
|
if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) {
|
|
|
|
}
|
|
|
|
$scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency];
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
$server = $scheduled_backup->server();
|
|
|
|
|
|
|
|
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
|
|
|
|
|
|
|
|
|
|
|
|
$server = $scheduled_backup->server();
|
|
|
|
if (validate_timezone($serverTimezone) === false) {
|
|
|
|
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
|
|
|
|
$serverTimezone = config('app.timezone');
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if (validate_timezone($serverTimezone) === false) {
|
|
|
|
if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) {
|
|
|
|
$serverTimezone = config('app.timezone');
|
|
|
|
$scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency];
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
|
|
|
|
|
|
|
|
$this->scheduleInstance->job(new DatabaseBackupJob(
|
|
|
|
|
|
|
|
backup: $scheduled_backup
|
|
|
|
|
|
|
|
))->cron($scheduled_backup->frequency)->timezone($serverTimezone)->onOneServer();
|
|
|
|
|
|
|
|
} catch (\Exception $e) {
|
|
|
|
|
|
|
|
Log::error('Error scheduling backup: '.$e->getMessage());
|
|
|
|
|
|
|
|
Log::error($e->getTraceAsString());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) {
|
|
|
|
|
|
|
|
$scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency];
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
|
|
|
|
|
|
|
|
$this->scheduleInstance->job(new DatabaseBackupJob(
|
|
|
|
|
|
|
|
backup: $scheduled_backup
|
|
|
|
|
|
|
|
))->cron($scheduled_backup->frequency)->timezone($serverTimezone)->onOneServer();
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@@ -264,18 +283,23 @@ class Kernel extends ConsoleKernel
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
foreach ($finalScheduledTasks as $scheduled_task) {
|
|
|
|
foreach ($finalScheduledTasks as $scheduled_task) {
|
|
|
|
$server = $scheduled_task->server();
|
|
|
|
try {
|
|
|
|
if (isset(VALID_CRON_STRINGS[$scheduled_task->frequency])) {
|
|
|
|
$server = $scheduled_task->server();
|
|
|
|
$scheduled_task->frequency = VALID_CRON_STRINGS[$scheduled_task->frequency];
|
|
|
|
if (isset(VALID_CRON_STRINGS[$scheduled_task->frequency])) {
|
|
|
|
}
|
|
|
|
$scheduled_task->frequency = VALID_CRON_STRINGS[$scheduled_task->frequency];
|
|
|
|
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
|
|
|
|
}
|
|
|
|
|
|
|
|
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
|
|
|
|
|
|
|
|
|
|
|
|
if (validate_timezone($serverTimezone) === false) {
|
|
|
|
if (validate_timezone($serverTimezone) === false) {
|
|
|
|
$serverTimezone = config('app.timezone');
|
|
|
|
$serverTimezone = config('app.timezone');
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
$this->scheduleInstance->job(new ScheduledTaskJob(
|
|
|
|
|
|
|
|
task: $scheduled_task
|
|
|
|
|
|
|
|
))->cron($scheduled_task->frequency)->timezone($serverTimezone)->onOneServer();
|
|
|
|
|
|
|
|
} catch (\Exception $e) {
|
|
|
|
|
|
|
|
Log::error('Error scheduling task: '.$e->getMessage());
|
|
|
|
|
|
|
|
Log::error($e->getTraceAsString());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
$this->scheduleInstance->job(new ScheduledTaskJob(
|
|
|
|
|
|
|
|
task: $scheduled_task
|
|
|
|
|
|
|
|
))->cron($scheduled_task->frequency)->timezone($serverTimezone)->onOneServer();
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|