From 3be06ced92ea4fc01f542643f48ff43d63e36e16 Mon Sep 17 00:00:00 2001 From: Lorenzo Migliorero Date: Wed, 17 Jul 2024 21:06:56 +0200 Subject: [PATCH 01/84] feat: labels by proxy type --- bootstrap/helpers/docker.php | 88 ++++++++++++++++++-------------- bootstrap/helpers/shared.php | 99 +++++++++++++++++++++--------------- 2 files changed, 108 insertions(+), 79 deletions(-) diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index 21e946a9a..fa2a19c2a 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -1,5 +1,6 @@ fqdn) { $domains = str(data_get($application, 'fqdn'))->explode(','); - $labels = $labels->merge(fqdnLabelsForTraefik( - uuid: $appUuid, - domains: $domains, - onlyPort: $onlyPort, - is_force_https_enabled: $application->isForceHttpsEnabled(), - is_gzip_enabled: $application->isGzipEnabled(), - is_stripprefix_enabled: $application->isStripprefixEnabled(), - redirect_direction: $application->redirect - )); - // Add Caddy labels - $labels = $labels->merge(fqdnLabelsForCaddy( - network: $application->destination->network, - uuid: $appUuid, - domains: $domains, - onlyPort: $onlyPort, - is_force_https_enabled: $application->isForceHttpsEnabled(), - is_gzip_enabled: $application->isGzipEnabled(), - is_stripprefix_enabled: $application->isStripprefixEnabled(), - redirect_direction: $application->redirect - )); + switch($application->destination->server->proxyType()) { + case ProxyTypes::TRAEFIK_V2->value: + $labels = $labels->merge(fqdnLabelsForTraefik( + uuid: $appUuid, + domains: $domains, + onlyPort: $onlyPort, + is_force_https_enabled: $application->isForceHttpsEnabled(), + is_gzip_enabled: $application->isGzipEnabled(), + is_stripprefix_enabled: $application->isStripprefixEnabled(), + redirect_direction: $application->redirect + )); + break; + case ProxyTypes::CADDY->value: + $labels = $labels->merge(fqdnLabelsForCaddy( + network: $application->destination->network, + uuid: $appUuid, + domains: $domains, + onlyPort: $onlyPort, + is_force_https_enabled: $application->isForceHttpsEnabled(), + is_gzip_enabled: $application->isGzipEnabled(), + is_stripprefix_enabled: $application->isStripprefixEnabled(), + redirect_direction: $application->redirect + )); + break; + } } } else { if (data_get($preview, 'fqdn')) { @@ -562,24 +568,30 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview } else { $domains = collect([]); } - $labels = $labels->merge(fqdnLabelsForTraefik( - uuid: $appUuid, - domains: $domains, - onlyPort: $onlyPort, - is_force_https_enabled: $application->isForceHttpsEnabled(), - is_gzip_enabled: $application->isGzipEnabled(), - is_stripprefix_enabled: $application->isStripprefixEnabled() - )); - // Add Caddy labels - $labels = $labels->merge(fqdnLabelsForCaddy( - network: $application->destination->network, - uuid: $appUuid, - domains: $domains, - onlyPort: $onlyPort, - is_force_https_enabled: $application->isForceHttpsEnabled(), - is_gzip_enabled: $application->isGzipEnabled(), - is_stripprefix_enabled: $application->isStripprefixEnabled() - )); + + switch($application->destination->server->proxyType()) { + case ProxyTypes::TRAEFIK_V2->value: + $labels = $labels->merge(fqdnLabelsForTraefik( + uuid: $appUuid, + domains: $domains, + onlyPort: $onlyPort, + is_force_https_enabled: $application->isForceHttpsEnabled(), + is_gzip_enabled: $application->isGzipEnabled(), + is_stripprefix_enabled: $application->isStripprefixEnabled() + )); + break; + case ProxyTypes::CADDY->value: + $labels = $labels->merge(fqdnLabelsForCaddy( + network: $application->destination->network, + uuid: $appUuid, + domains: $domains, + onlyPort: $onlyPort, + is_force_https_enabled: $application->isForceHttpsEnabled(), + is_gzip_enabled: $application->isGzipEnabled(), + is_stripprefix_enabled: $application->isStripprefixEnabled() + )); + break; + } } diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index e3c8d5218..9600b13d2 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -1,6 +1,7 @@ merge($defaultLabels); if (! $isDatabase && $fqdns->count() > 0) { if ($fqdns) { - $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik( - uuid: $resource->uuid, - domains: $fqdns, - is_force_https_enabled: true, - serviceLabels: $serviceLabels, - is_gzip_enabled: $savedService->isGzipEnabled(), - is_stripprefix_enabled: $savedService->isStripprefixEnabled(), - service_name: $serviceName, - image: data_get($service, 'image') - )); - $serviceLabels = $serviceLabels->merge(fqdnLabelsForCaddy( - network: $resource->destination->network, - uuid: $resource->uuid, - domains: $fqdns, - is_force_https_enabled: true, - serviceLabels: $serviceLabels, - is_gzip_enabled: $savedService->isGzipEnabled(), - is_stripprefix_enabled: $savedService->isStripprefixEnabled(), - service_name: $serviceName, - image: data_get($service, 'image') - )); + switch($resource->destination->server->proxyType()) { + case ProxyTypes::TRAEFIK_V2->value: + $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik( + uuid: $resource->uuid, + domains: $fqdns, + is_force_https_enabled: true, + serviceLabels: $serviceLabels, + is_gzip_enabled: $savedService->isGzipEnabled(), + is_stripprefix_enabled: $savedService->isStripprefixEnabled(), + service_name: $serviceName, + image: data_get($service, 'image') + )); + break; + case ProxyTypes::CADDY->value: + $serviceLabels = $serviceLabels->merge(fqdnLabelsForCaddy( + network: $resource->destination->network, + uuid: $resource->uuid, + domains: $fqdns, + is_force_https_enabled: true, + serviceLabels: $serviceLabels, + is_gzip_enabled: $savedService->isGzipEnabled(), + is_stripprefix_enabled: $savedService->isStripprefixEnabled(), + service_name: $serviceName, + image: data_get($service, 'image') + )); + break; + } } } if ($resource->server->isLogDrainEnabled() && $savedService->isLogDrainEnabled()) { @@ -2023,26 +2030,36 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal }); } } - $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik( - uuid: $resource->uuid, - domains: $fqdns, - serviceLabels: $serviceLabels, - generate_unique_uuid: $resource->build_pack === 'dockercompose', - image: data_get($service, 'image'), - is_force_https_enabled: $resource->isForceHttpsEnabled(), - is_gzip_enabled: $resource->isGzipEnabled(), - is_stripprefix_enabled: $resource->isStripprefixEnabled(), - )); - $serviceLabels = $serviceLabels->merge(fqdnLabelsForCaddy( - network: $resource->destination->network, - uuid: $resource->uuid, - domains: $fqdns, - serviceLabels: $serviceLabels, - image: data_get($service, 'image'), - is_force_https_enabled: $resource->isForceHttpsEnabled(), - is_gzip_enabled: $resource->isGzipEnabled(), - is_stripprefix_enabled: $resource->isStripprefixEnabled(), - )); + switch ($server->proxyType()) { + case ProxyTypes::TRAEFIK_V2->value: + $serviceLabels = $serviceLabels->merge( + fqdnLabelsForTraefik( + uuid: $resource->uuid, + domains: $fqdns, + serviceLabels: $serviceLabels, + generate_unique_uuid: $resource->build_pack === 'dockercompose', + image: data_get($service, 'image'), + is_force_https_enabled: $resource->isForceHttpsEnabled(), + is_gzip_enabled: $resource->isGzipEnabled(), + is_stripprefix_enabled: $resource->isStripprefixEnabled(), + ) + ); + break; + case ProxyTypes::CADDY->value: + $serviceLabels = $serviceLabels->merge( + fqdnLabelsForCaddy( + network: $resource->destination->network, + uuid: $resource->uuid, + domains: $fqdns, + serviceLabels: $serviceLabels, + image: data_get($service, 'image'), + is_force_https_enabled: $resource->isForceHttpsEnabled(), + is_gzip_enabled: $resource->isGzipEnabled(), + is_stripprefix_enabled: $resource->isStripprefixEnabled(), + ) + ); + break; + } } } } From be6d74a6a3c3f680d6f69d61c030b7484a05640a Mon Sep 17 00:00:00 2001 From: Lorenzo Migliorero Date: Thu, 18 Jul 2024 11:42:41 +0200 Subject: [PATCH 02/84] style: linting --- bootstrap/helpers/docker.php | 8 ++++---- bootstrap/helpers/shared.php | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index fa2a19c2a..355127130 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -536,7 +536,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview if ($pull_request_id === 0) { if ($application->fqdn) { $domains = str(data_get($application, 'fqdn'))->explode(','); - switch($application->destination->server->proxyType()) { + switch ($application->destination->server->proxyType()) { case ProxyTypes::TRAEFIK_V2->value: $labels = $labels->merge(fqdnLabelsForTraefik( uuid: $appUuid, @@ -547,7 +547,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview is_stripprefix_enabled: $application->isStripprefixEnabled(), redirect_direction: $application->redirect )); - break; + break; case ProxyTypes::CADDY->value: $labels = $labels->merge(fqdnLabelsForCaddy( network: $application->destination->network, @@ -559,7 +559,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview is_stripprefix_enabled: $application->isStripprefixEnabled(), redirect_direction: $application->redirect )); - break; + break; } } } else { @@ -569,7 +569,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview $domains = collect([]); } - switch($application->destination->server->proxyType()) { + switch ($application->destination->server->proxyType()) { case ProxyTypes::TRAEFIK_V2->value: $labels = $labels->merge(fqdnLabelsForTraefik( uuid: $appUuid, diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 9600b13d2..a909b28e9 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -1306,7 +1306,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal $serviceLabels = $serviceLabels->merge($defaultLabels); if (! $isDatabase && $fqdns->count() > 0) { if ($fqdns) { - switch($resource->destination->server->proxyType()) { + switch ($resource->destination->server->proxyType()) { case ProxyTypes::TRAEFIK_V2->value: $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik( uuid: $resource->uuid, From 097ca209bc4cbc607440e83a47c2788c6aaa4d57 Mon Sep 17 00:00:00 2001 From: Toby Brancher Date: Sun, 21 Jul 2024 23:39:22 +0100 Subject: [PATCH 03/84] Support map fields in Directus To support the map fields in Directus with Postgres you need the Postgis extension installed, with docker we can just change the image to use the equivalent alpine postgis image, source latest stable alpine from here (perhaps there is a better ref to use than one with a specific version number?): https://registry.hub.docker.com/r/postgis/postgis/ If you are already running Directus without postgis, and would like to make use of the mapping fields after updating the docker compose file you will need to go to `execute command`, select the postgres container and run the following command filling in the appropriate env vars; `psql -U ${SERVICE_USER_POSTGRESQL} -d ${POSTGRESQL_DATABASE} -c "CREATE EXTENSION postgis;"` --- templates/compose/directus-with-postgresql.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/compose/directus-with-postgresql.yaml b/templates/compose/directus-with-postgresql.yaml index e7dd60921..e97c77af0 100644 --- a/templates/compose/directus-with-postgresql.yaml +++ b/templates/compose/directus-with-postgresql.yaml @@ -36,7 +36,7 @@ services: redis: condition: service_healthy postgresql: - image: postgres:16-alpine + image: postgis/postgis:16-3.4-alpine volumes: - directus-postgresql-data:/var/lib/postgresql/data environment: From 219b21767bdd6fef810376f86916ba5561815982 Mon Sep 17 00:00:00 2001 From: Filip Gornitzka Abelson Date: Fri, 26 Jul 2024 17:27:20 +0200 Subject: [PATCH 04/84] chore: update formbricks template --- templates/compose/formbricks.yaml | 50 ++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/templates/compose/formbricks.yaml b/templates/compose/formbricks.yaml index 129d53519..b0729284a 100644 --- a/templates/compose/formbricks.yaml +++ b/templates/compose/formbricks.yaml @@ -1,6 +1,6 @@ -# documentation: https://formbricks.com -# slogan: Open Source Experience Management -# tags: form, builder, forms, open source, experience, management, self-hosted, docker +# documentation: https://formbricks.com/docs/self-hosting/configuration +# slogan: Open Source Survey Platform +# tags: form, builder, forms, survey, open source, experience, management, self-hosted, docker # logo: svgs/formbricks.png # port: 3000 @@ -11,33 +11,55 @@ services: - SERVICE_FQDN_FORMBRICKS_3000 - WEBAPP_URL=$SERVICE_FQDN_FORMBRICKS - DATABASE_URL=postgres://$SERVICE_USER_POSTGRESQL:$SERVICE_PASSWORD_POSTGRESQL@postgresql:5432/${POSTGRESQL_DATABASE:-formbricks} + - POSTGRES_PASSWORD=${SERVICE_PASSWORD_POSTGRESQL} - NEXTAUTH_SECRET=$SERVICE_BASE64_64_NEXTAUTH - NEXTAUTH_URL=$SERVICE_FQDN_FORMBRICKS - ENCRYPTION_KEY=$SERVICE_BASE64_64_ENCRYPTION - - POSTGRES_PASSWORD=${SERVICE_PASSWORD_POSTGRESQL} + - CRON_SECRET=$SERVICE_BASE64_64_CRON + - ENTERPRISE_LICENSE_KEY=${ENTERPRISE_LICENSE_KEY} - MAIL_FROM=${MAIL_FROM:-test@example.com} - SMTP_HOST=${SMTP_HOST:-test.example.com} - SMTP_PORT=${SMTP_PORT:-587} - SMTP_USER=${SMTP_USER:-test} - SMTP_PASSWORD=${SMTP_PASSWORD:-test} - SMTP_SECURE_ENABLED=${SMTP_SECURE_ENABLED:-0} + - SMTP_REJECT_UNAUTHORIZED_TLS=${SMTP_REJECT_UNAUTHORIZED_TLS:-1} - SHORT_URL_BASE=${SHORT_URL_BASE} - EMAIL_VERIFICATION_DISABLED=${EMAIL_VERIFICATION_DISABLED:-1} - PASSWORD_RESET_DISABLED=${PASSWORD_RESET_DISABLED:-1} - - SIGNUP_DISABLED=${SIGNUP_DISABLED:-0} + - EMAIL_AUTH_DISABLED=${EMAIL_AUTH_DISABLED:-0} - INVITE_DISABLED=${INVITE_DISABLED:-0} + - ASSET_PREFIX_URL=${ASSET_PREFIX_URL} + - UNSPLASH_ACCESS_KEY=${UNSPLASH_ACCESS_KEY} + - GITHUB_ID=${GITHUB_ID} + - GITHUB_SECRET=${GITHUB_SECRET} + - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID} + - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET} + - AZUREAD_CLIENT_ID=${AZUREAD_CLIENT_ID} + - AZUREAD_CLIENT_SECRET=${AZUREAD_CLIENT_SECRET} + - AZUREAD_TENANT_ID=${AZUREAD_TENANT_ID} + - OIDC_CLIENT_ID=${OIDC_CLIENT_ID} + - OIDC_CLIENT_SECRET=${OIDC_CLIENT_SECRET} + - OIDC_ISSUER=${OIDC_ISSUER} + - OIDC_DISPLAY_NAME=${OIDC_DISPLAY_NAME} + - OIDC_SIGNING_ALGORITHM=${OIDC_SIGNING_ALGORITHM} + - NOTION_OAUTH_CLIENT_ID=${NOTION_OAUTH_CLIENT_ID} + - NOTION_OAUTH_CLIENT_SECRET=${NOTION_OAUTH_CLIENT_SECRET} + - GOOGLE_SHEETS_CLIENT_ID=${GOOGLE_SHEETS_CLIENT_ID} + - GOOGLE_SHEETS_CLIENT_SECRET=${GOOGLE_SHEETS_CLIENT_SECRET} + - GOOGLE_SHEETS_REDIRECT_URL=${GOOGLE_SHEETS_REDIRECT_URL} + - AIRTABLE_CLIENT_ID=${AIRTABLE_CLIENT_ID} + - SLACK_CLIENT_ID=${SLACK_CLIENT_ID} + - SLACK_CLIENT_SECRET=${SLACK_CLIENT_SECRET} - PRIVACY_URL=${PRIVACY_URL} - TERMS_URL=${TERMS_URL} - IMPRINT_URL=${IMPRINT_URL} - - GITHUB_AUTH_ENABLED=${GITHUB_AUTH_ENABLED:-0} - - GITHUB_ID=${GITHUB_ID} - - GITHUB_SECRET=${GITHUB_SECRET} - - GOOGLE_AUTH_ENABLED=${GOOGLE_AUTH_ENABLED:-0} - - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID} - - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET} - - ASSET_PREFIX_URL=${ASSET_PREFIX_URL} - - CRON_SECRET=$SERVICE_BASE64_64_CRON - - ENCRYPTION_KEY=$SERVICE_BASE64_64_ENCRYPTION + - RATE_LIMITING_DISABLED=${RATE_LIMITING_DISABLED:-0} + - OPENTELEMETRY_LISTENER_URL=${OPENTELEMETRY_LISTENER_URL} + - REDIS_URL=${REDIS_URL} + - REDIS_HTTP_URL=${REDIS_HTTP_URL} + - DEFAULT_ORGANIZATION_ID=${DEFAULT_ORGANIZATION_ID} + - DEFAULT_ORGANIZATION_ROLE=${DEFAULT_ORGANIZATION_ROLE:-admin} volumes: - formbricks-uploads:/apps/web/uploads/ depends_on: From a2e439686d12a76f98cb0bef56a814386a4877d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= <88948175+J-o-n-a-t-h-a-n-M-u-e-l-l-e-r@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:30:36 +0000 Subject: [PATCH 05/84] Fixes Plausible TOTP key by changing it from BASE64 to REALBASE64_32 --- templates/compose/plausible.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/compose/plausible.yaml b/templates/compose/plausible.yaml index e73421d57..14a80ad3f 100644 --- a/templates/compose/plausible.yaml +++ b/templates/compose/plausible.yaml @@ -12,7 +12,7 @@ services: - "DATABASE_URL=postgres://postgres:$SERVICE_PASSWORD_POSTGRES@plausible_db/plausible" - BASE_URL=$SERVICE_FQDN_PLAUSIBLE - SECRET_KEY_BASE=$SERVICE_BASE64_64_PLAUSIBLE - - TOTP_VAULT_KEY=$SERVICE_BASE64_TOTP + - TOTP_VAULT_KEY=$SERVICE_REALBASE64_32_TOTP depends_on: - plausible_db - plausible_events_db From 342ef4d36782efbd365c632b15720d0901f3d2f7 Mon Sep 17 00:00:00 2001 From: Alexzvn Date: Mon, 29 Jul 2024 07:57:13 +0000 Subject: [PATCH 06/84] add graceful shutdown --- app/Jobs/ApplicationDeploymentJob.php | 33 +++++++++++++++++++-------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index d7b1a57da..54a80a92a 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -2024,6 +2024,26 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); $this->application_deployment_queue->addLogEntry('Building docker image completed.'); } + /** + * @param integer $timeout in seconds + */ + private function graceful_shutdown_container(string $containerName, int $timeout = 300) { + try { + $this->execute_remote_command( + ["docker stop --time=$timeout $containerName > /dev/null 2>&1", 'hidden' => true], + ["docker rm $containerName > /dev/null 2>&1", 'hidden' => true] + ); + + return; + } catch (\Exception $error) { + // report error if needed + } finally { + $this->execute_remote_command( + ["docker rm -f $containerName >/dev/null 2>&1", 'hidden' => true, 'ignore_errors' => true] + ); + } + } + private function stop_running_container(bool $force = false) { $this->application_deployment_queue->addLogEntry('Removing old containers.'); @@ -2035,15 +2055,10 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); }); } $containers->each(function ($container) { - $containerName = data_get($container, 'Names'); - $this->execute_remote_command( - ["docker rm -f $containerName >/dev/null 2>&1", 'hidden' => true, 'ignore_errors' => true], - ); + $this->graceful_shutdown_container(data_get($container, 'Names')); }); if ($this->application->settings->is_consistent_container_name_enabled || isset($this->application->settings->custom_internal_name)) { - $this->execute_remote_command( - ["docker rm -f $this->container_name >/dev/null 2>&1", 'hidden' => true, 'ignore_errors' => true], - ); + $this->graceful_shutdown_container($this->container_name); } } else { if ($this->application->dockerfile || $this->application->build_pack === 'dockerfile' || $this->application->build_pack === 'dockerimage') { @@ -2055,9 +2070,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); $this->application_deployment_queue->update([ 'status' => ApplicationDeploymentStatus::FAILED->value, ]); - $this->execute_remote_command( - ["docker rm -f $this->container_name >/dev/null 2>&1", 'hidden' => true, 'ignore_errors' => true], - ); + $this->graceful_shutdown_container($this->container_name); } } From 1feb8488a38ca003745b397d67c4335501711546 Mon Sep 17 00:00:00 2001 From: Alexzvn Date: Mon, 29 Jul 2024 08:09:27 +0000 Subject: [PATCH 07/84] cleanup --- app/Jobs/ApplicationDeploymentJob.php | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 54a80a92a..2d3b0d5a6 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -2029,19 +2029,17 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); */ private function graceful_shutdown_container(string $containerName, int $timeout = 300) { try { - $this->execute_remote_command( + return $this->execute_remote_command( ["docker stop --time=$timeout $containerName > /dev/null 2>&1", 'hidden' => true], ["docker rm $containerName > /dev/null 2>&1", 'hidden' => true] ); - - return; } catch (\Exception $error) { // report error if needed - } finally { - $this->execute_remote_command( - ["docker rm -f $containerName >/dev/null 2>&1", 'hidden' => true, 'ignore_errors' => true] - ); } + + $this->execute_remote_command( + ["docker rm -f $containerName >/dev/null 2>&1", 'hidden' => true, 'ignore_errors' => true] + ); } private function stop_running_container(bool $force = false) From 9887d5eedd6679e7c691c648ab96ed8103c005d7 Mon Sep 17 00:00:00 2001 From: Gabriel Peralta Date: Wed, 31 Jul 2024 00:50:13 -0400 Subject: [PATCH 08/84] Added minecraft-server (java) as template --- public/svgs/minecraft.svg | 1 + templates/compose/minecraft.yaml | 44 ++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 public/svgs/minecraft.svg create mode 100644 templates/compose/minecraft.yaml diff --git a/public/svgs/minecraft.svg b/public/svgs/minecraft.svg new file mode 100644 index 000000000..10ae9042c --- /dev/null +++ b/public/svgs/minecraft.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/templates/compose/minecraft.yaml b/templates/compose/minecraft.yaml new file mode 100644 index 000000000..e00f891e9 --- /dev/null +++ b/templates/compose/minecraft.yaml @@ -0,0 +1,44 @@ +# documentation: https://github.com/itzg/docker-minecraft-server +# slogan: Docker image that provides a Minecraft Server that will automatically download selected version at startup. +# tags: minecraft +# logo: svgs/minecraft.svg +# port: 25565 + +services: + mc: + image: itzg/minecraft-server + ports: + - 25565:25565 + environment: + EULA: true + VERSIOn: latest + TYPE: vanilla + SERVER_NAME: Minecraft Server + MOTD: Minecraft Server powered by §aCoolify§r + DIFFICULTY: normal + MAX_PLAYERS: 10 + MAX_WORLD_SIZE: 10000 + VIEW_DISTANCE: 10 + MAX_BUILD_HEIGHT: 256 + MAX_TICK_TIME: 60000 + ALLOW_NETHER: true + ANNOUNCE_PLAYER_ACHIEVEMENTS: true + GENERATE_STRUCTURES: true + PVP: true + FORCE_GAMEMODE: false + HARDCORE: false + ENABLE_COMMAND_BLOCK: false + SPAWN_ANIMALS: true + SPAWN_MONSTERS: true + SPAWN_NPCS: true + SNOOPER_ENABLED: true + ONLINE_MODE: true + PLAYER_IDLE_TIMEOUT: 0 + MEMORY: 1G + ENABLE_AUTOPAUSE: false + tty: true + stdin_open: true + restart: always + volumes: + # attach a directory relative to the directory containing this compose file + - /minecraft-data:/data \ No newline at end of file From cb2fc68dde24abc91ab917d0ef2a8f3a10901e4f Mon Sep 17 00:00:00 2001 From: Gabriel Peralta Date: Sat, 3 Aug 2024 21:13:42 -0400 Subject: [PATCH 09/84] Update minecraft.yaml --- templates/compose/minecraft.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/compose/minecraft.yaml b/templates/compose/minecraft.yaml index e00f891e9..ce8d20827 100644 --- a/templates/compose/minecraft.yaml +++ b/templates/compose/minecraft.yaml @@ -11,7 +11,7 @@ services: - 25565:25565 environment: EULA: true - VERSIOn: latest + VERSION: latest TYPE: vanilla SERVER_NAME: Minecraft Server MOTD: Minecraft Server powered by §aCoolify§r @@ -41,4 +41,4 @@ services: restart: always volumes: # attach a directory relative to the directory containing this compose file - - /minecraft-data:/data \ No newline at end of file + - /minecraft-data:/data From d7b45a6dd205e183d2cbb4f1fddc5e3de9de9557 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 5 Aug 2024 11:30:48 +0200 Subject: [PATCH 10/84] fix: remove lazy load from scheduled tasks --- .../views/livewire/project/service/configuration.blade.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/views/livewire/project/service/configuration.blade.php b/resources/views/livewire/project/service/configuration.blade.php index d65c20f88..b9282dc91 100644 --- a/resources/views/livewire/project/service/configuration.blade.php +++ b/resources/views/livewire/project/service/configuration.blade.php @@ -183,7 +183,7 @@ @endforeach
- +
From 820693ac22316b0d55914ffbfb8f6c5c1012be83 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 5 Aug 2024 11:50:52 +0200 Subject: [PATCH 11/84] fix: plausible template --- templates/compose/plausible.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/compose/plausible.yaml b/templates/compose/plausible.yaml index e73421d57..408d96598 100644 --- a/templates/compose/plausible.yaml +++ b/templates/compose/plausible.yaml @@ -6,13 +6,13 @@ services: plausible: - image: "ghcr.io/plausible/community-edition:v2.1.0" + image: "ghcr.io/plausible/community-edition:v2.1" command: 'sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run"' environment: - "DATABASE_URL=postgres://postgres:$SERVICE_PASSWORD_POSTGRES@plausible_db/plausible" - BASE_URL=$SERVICE_FQDN_PLAUSIBLE - SECRET_KEY_BASE=$SERVICE_BASE64_64_PLAUSIBLE - - TOTP_VAULT_KEY=$SERVICE_BASE64_TOTP + - TOTP_VAULT_KEY=$SERVICE_REALBASE64_32_TOTP depends_on: - plausible_db - plausible_events_db From 1d8c496906408c4da2e21057a714bd5583d23ea7 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 5 Aug 2024 12:03:36 +0200 Subject: [PATCH 12/84] feat: delete team in cloud without subscription --- app/Livewire/NavbarDeleteTeam.php | 35 ++++++++++++++++ resources/views/components/navbar.blade.php | 42 ++----------------- .../livewire/navbar-delete-team.blade.php | 5 +++ 3 files changed, 43 insertions(+), 39 deletions(-) create mode 100644 app/Livewire/NavbarDeleteTeam.php create mode 100644 resources/views/livewire/navbar-delete-team.blade.php diff --git a/app/Livewire/NavbarDeleteTeam.php b/app/Livewire/NavbarDeleteTeam.php new file mode 100644 index 000000000..ec196c154 --- /dev/null +++ b/app/Livewire/NavbarDeleteTeam.php @@ -0,0 +1,35 @@ +delete(); + + $currentTeam->members->each(function ($user) use ($currentTeam) { + if ($user->id === auth()->user()->id) { + return; + } + $user->teams()->detach($currentTeam); + $session = DB::table('sessions')->where('user_id', $user->id)->first(); + if ($session) { + DB::table('sessions')->where('id', $session->id)->delete(); + } + }); + + refreshSession(); + + return redirect()->route('team.index'); + } + + public function render() + { + return view('livewire.navbar-delete-team'); + } +} diff --git a/resources/views/components/navbar.blade.php b/resources/views/components/navbar.blade.php index fbb5291fb..7da619377 100644 --- a/resources/views/components/navbar.blade.php +++ b/resources/views/components/navbar.blade.php @@ -351,6 +351,9 @@ @endif + @if (!isSubscribed() && isCloud() && auth()->user()->teams()->get()->count() > 1) + + @endif
  • @@ -380,44 +383,5 @@
    • - {{--
  • -
    Your teams
    - -
    • -
  • - - - Your profile - - -
    • --}} diff --git a/resources/views/livewire/navbar-delete-team.blade.php b/resources/views/livewire/navbar-delete-team.blade.php new file mode 100644 index 000000000..b660a0dd4 --- /dev/null +++ b/resources/views/livewire/navbar-delete-team.blade.php @@ -0,0 +1,5 @@ +
    + + This team be deleted. It is not reversible.
    Please think again. +     +
    From 76c39a987c613bfe7ebf05dcafee2ecf5cb2ebdd Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 5 Aug 2024 13:42:51 +0200 Subject: [PATCH 13/84] fix: service_url should not have a trailing slash --- bootstrap/helpers/services.php | 48 ++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/bootstrap/helpers/services.php b/bootstrap/helpers/services.php index d4e9ebcde..ea743b8cd 100644 --- a/bootstrap/helpers/services.php +++ b/bootstrap/helpers/services.php @@ -122,14 +122,22 @@ function updateCompose(ServiceApplication|ServiceDatabase $resource) $path = $fqdn->getPath(); $fqdn = $fqdn->getScheme().'://'.$fqdn->getHost(); if ($generatedEnv) { - $generatedEnv->value = $fqdn.$path; + if ($path === '/') { + $generatedEnv->value = $fqdn; + } else { + $generatedEnv->value = $fqdn.$path; + } $generatedEnv->save(); } if ($port) { $variableName = $variableName."_$port"; $generatedEnv = EnvironmentVariable::where('service_id', $resource->service_id)->where('key', $variableName)->first(); if ($generatedEnv) { - $generatedEnv->value = $fqdn.$path; + if ($path === '/') { + $generatedEnv->value = $fqdn; + } else { + $generatedEnv->value = $fqdn.$path; + } $generatedEnv->save(); } } @@ -141,14 +149,22 @@ function updateCompose(ServiceApplication|ServiceDatabase $resource) $url = $url->getHost(); if ($generatedEnv) { $url = str($fqdn)->after('://'); - $generatedEnv->value = $url.$path; + if ($path === '/') { + $generatedEnv->value = $url; + } else { + $generatedEnv->value = $url.$path; + } $generatedEnv->save(); } if ($port) { $variableName = $variableName."_$port"; $generatedEnv = EnvironmentVariable::where('service_id', $resource->service_id)->where('key', $variableName)->first(); if ($generatedEnv) { - $generatedEnv->value = $url.$path; + if ($path === '/') { + $generatedEnv->value = $url; + } else { + $generatedEnv->value = $url.$path; + } $generatedEnv->save(); } } @@ -165,10 +181,18 @@ function updateCompose(ServiceApplication|ServiceDatabase $resource) $service_fqdn = str($port_env->key)->beforeLast('_')->after('SERVICE_FQDN_'); $env = EnvironmentVariable::where('service_id', $resource->service_id)->where('key', 'SERVICE_FQDN_'.$service_fqdn)->first(); if ($env) { - $env->value = $host.$path; + if ($path === '/') { + $env->value = $host; + } else { + $env->value = $host.$path; + } $env->save(); } - $port_env->value = $host.$path; + if ($path === '/') { + $port_env->value = $host; + } else { + $port_env->value = $host.$path; + } $port_env->save(); } $port_envs_url = EnvironmentVariable::where('service_id', $resource->service_id)->where('key', 'like', "SERVICE_URL_%_$port")->get(); @@ -176,10 +200,18 @@ function updateCompose(ServiceApplication|ServiceDatabase $resource) $service_url = str($port_env_url->key)->beforeLast('_')->after('SERVICE_URL_'); $env = EnvironmentVariable::where('service_id', $resource->service_id)->where('key', 'SERVICE_URL_'.$service_url)->first(); if ($env) { - $env->value = $url.$path; + if ($path === '/') { + $env->value = $url; + } else { + $env->value = $url.$path; + } $env->save(); } - $port_env_url->value = $url.$path; + if ($path === '/') { + $port_env_url->value = $url; + } else { + $port_env_url->value = $url.$path; + } $port_env_url->save(); } } else { From b38a651a08862b1e2576257f8c700dea947eac97 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 5 Aug 2024 13:45:24 +0200 Subject: [PATCH 14/84] feat: coolify init should cleanup stuck networks in proxy --- app/Console/Commands/Init.php | 39 +++++++++++++++++++++++++++++++++-- bootstrap/helpers/proxy.php | 15 ++++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/app/Console/Commands/Init.php b/app/Console/Commands/Init.php index 4a276cfc4..a2e31e779 100644 --- a/app/Console/Commands/Init.php +++ b/app/Console/Commands/Init.php @@ -16,7 +16,7 @@ use Illuminate\Support\Facades\Http; class Init extends Command { - protected $signature = 'app:init {--full-cleanup} {--cleanup-deployments}'; + protected $signature = 'app:init {--full-cleanup} {--cleanup-deployments} {--cleanup-proxy-networks}'; protected $description = 'Cleanup instance related stuffs'; @@ -36,7 +36,7 @@ class Init extends Command $full_cleanup = $this->option('full-cleanup'); $cleanup_deployments = $this->option('cleanup-deployments'); - + $cleanup_proxy_networks = $this->option('cleanup-proxy-networks'); $this->replace_slash_in_environment_name(); if ($cleanup_deployments) { echo "Running cleanup deployments.\n"; @@ -44,9 +44,16 @@ class Init extends Command return; } + if ($cleanup_proxy_networks) { + echo "Running cleanup proxy networks.\n"; + $this->cleanup_unused_network_from_coolify_proxy(); + + return; + } if ($full_cleanup) { // Required for falsely deleted coolify db $this->restore_coolify_db_backup(); + $this->cleanup_unused_network_from_coolify_proxy(); $this->cleanup_in_progress_application_deployments(); $this->cleanup_stucked_helper_containers(); $this->call('cleanup:queue'); @@ -75,6 +82,34 @@ class Init extends Command $this->call('cleanup:stucked-resources'); } + private function cleanup_unused_network_from_coolify_proxy() + { + ray()->clearAll(); + $servers = Server::all(); + foreach ($servers as $server) { + if (! $server->isFunctional()) { + continue; + } + if (! $server->isProxyShouldRun()) { + continue; + } + ['networks' => $networks, 'allNetworks' => $allNetworks] = collectDockerNetworksByServer($server); + $removeNetworks = $allNetworks->diff($networks); + $commands = collect(); + foreach ($removeNetworks as $network) { + $out = instant_remote_process(["docker network inspect -f json $network | jq '.[].Containers | if . == {} then null else . end'"], $server, false); + if (empty($out)) { + $commands->push("docker network disconnect $network coolify-proxy >/dev/null 2>&1 || true"); + $commands->push("docker network rm $network >/dev/null 2>&1 || true"); + } + } + if ($commands->isNotEmpty()) { + echo "Cleaning up unused networks from coolify proxy\n"; + instant_remote_process($commands, $server, false); + } + } + } + private function restore_coolify_db_backup() { try { diff --git a/bootstrap/helpers/proxy.php b/bootstrap/helpers/proxy.php index dccfaeb38..c8202af2d 100644 --- a/bootstrap/helpers/proxy.php +++ b/bootstrap/helpers/proxy.php @@ -24,6 +24,7 @@ function collectProxyDockerNetworksByServer(Server $server) } function collectDockerNetworksByServer(Server $server) { + $allNetworks = collect([]); if ($server->isSwarm()) { $networks = collect($server->swarmDockers)->map(function ($docker) { return $docker['network']; @@ -34,11 +35,13 @@ function collectDockerNetworksByServer(Server $server) return $docker['network']; }); } + $allNetworks = $allNetworks->merge($networks); // Service networks foreach ($server->services()->get() as $service) { if ($service->isRunning()) { $networks->push($service->networks()); } + $allNetworks->push($service->networks()); } // Docker compose based apps $docker_compose_apps = $server->dockerComposeBasedApplications(); @@ -46,6 +49,7 @@ function collectDockerNetworksByServer(Server $server) if ($app->isRunning()) { $networks->push($app->uuid); } + $allNetworks->push($app->uuid); } // Docker compose based preview deployments $docker_compose_previews = $server->dockerComposeBasedPreviewDeployments(); @@ -61,23 +65,30 @@ function collectDockerNetworksByServer(Server $server) } $network = "{$application->uuid}-{$pullRequestId}"; $networks->push($network); + $allNetworks->push($network); } $networks = collect($networks)->flatten()->unique(); + $allNetworks = $allNetworks->flatten()->unique(); if ($server->isSwarm()) { if ($networks->count() === 0) { $networks = collect(['coolify-overlay']); + $allNetworks = collect(['coolify-overlay']); } } else { if ($networks->count() === 0) { $networks = collect(['coolify']); + $allNetworks = collect(['coolify']); } } - return $networks; + return [ + 'networks' => $networks, + 'allNetworks' => $allNetworks, + ]; } function connectProxyToNetworks(Server $server) { - $networks = collectDockerNetworksByServer($server); + ['networks' => $networks] = collectDockerNetworksByServer($server); if ($server->isSwarm()) { $commands = $networks->map(function ($network) { return [ From e2bc3f48417c90849d14f1b03a0d33f8d14f7122 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 5 Aug 2024 13:45:33 +0200 Subject: [PATCH 15/84] refactor: Remove commented out code for cleaning up networks in CleanupDocker.php --- app/Actions/Server/CleanupDocker.php | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/app/Actions/Server/CleanupDocker.php b/app/Actions/Server/CleanupDocker.php index 35cab4d11..0009e001d 100644 --- a/app/Actions/Server/CleanupDocker.php +++ b/app/Actions/Server/CleanupDocker.php @@ -11,7 +11,6 @@ class CleanupDocker public function handle(Server $server, bool $force = true) { - // cleanup docker images, containers, and builder caches if ($force) { instant_remote_process(['docker image prune -af'], $server, false); @@ -22,15 +21,5 @@ class CleanupDocker instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $server, false); instant_remote_process(['docker builder prune -f'], $server, false); } - // cleanup networks - // $networks = collectDockerNetworksByServer($server); - // $proxyNetworks = collectProxyDockerNetworksByServer($server); - // $diff = $proxyNetworks->diff($networks); - // if ($diff->count() > 0) { - // $diff->map(function ($network) use ($server) { - // instant_remote_process(["docker network disconnect $network coolify-proxy"], $server); - // instant_remote_process(["docker network rm $network"], $server); - // }); - // } } } From 465b254813e902a1bcc18914ce6f84a576d23a2f Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 5 Aug 2024 13:45:53 +0200 Subject: [PATCH 16/84] fix: if usagebefore cannot be determined, cleanup docker with force --- app/Jobs/DockerCleanupJob.php | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php index f99a65b5d..c798cd807 100644 --- a/app/Jobs/DockerCleanupJob.php +++ b/app/Jobs/DockerCleanupJob.php @@ -26,17 +26,6 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue public function handle(): void { try { - // $isInprogress = false; - // $this->server->applications()->each(function ($application) use (&$isInprogress) { - // if ($application->isDeploymentInprogress()) { - // $isInprogress = true; - - // return; - // } - // }); - // if ($isInprogress) { - // throw new RuntimeException('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...'); - // } if (! $this->server->isFunctional()) { return; } @@ -48,6 +37,12 @@ class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue } $this->usageBefore = $this->server->getDiskUsage(); + if ($this->usageBefore === null) { + Log::info('DockerCleanupJob force cleanup on '.$this->server->name); + CleanupDocker::run(server: $this->server, force: true); + + return; + } if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) { CleanupDocker::run(server: $this->server, force: false); $usageAfter = $this->server->getDiskUsage(); From 39e3ea9f07b5948d9d56f149173ca2818a28e7a4 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 5 Aug 2024 13:51:34 +0200 Subject: [PATCH 17/84] fix: async remote command --- app/Console/Commands/Init.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/Console/Commands/Init.php b/app/Console/Commands/Init.php index a2e31e779..f397d8861 100644 --- a/app/Console/Commands/Init.php +++ b/app/Console/Commands/Init.php @@ -3,6 +3,7 @@ namespace App\Console\Commands; use App\Actions\Server\StopSentinel; +use App\Enums\ActivityTypes; use App\Enums\ApplicationDeploymentStatus; use App\Jobs\CleanupHelperContainersJob; use App\Models\ApplicationDeploymentQueue; @@ -27,7 +28,9 @@ class Init extends Command if (version_compare('4.0.0-beta.312', config('version'), '<=')) { $servers = Server::all(); foreach ($servers as $server) { - $server->settings->update(['is_metrics_enabled' => false]); + if ($server->settings->is_metrics_enabled === true) { + $server->settings->update(['is_metrics_enabled' => false]); + } if ($server->isFunctional()) { StopSentinel::dispatch($server); } @@ -105,7 +108,7 @@ class Init extends Command } if ($commands->isNotEmpty()) { echo "Cleaning up unused networks from coolify proxy\n"; - instant_remote_process($commands, $server, false); + remote_process(command: $commands, type: ActivityTypes::INLINE->value, server: $server, ignore_errors: false); } } } From 7d6a8954498d46c6b5891dee59224a0936c42f2f Mon Sep 17 00:00:00 2001 From: ayntk-ai <122374094+ayntk-ai@users.noreply.github.com> Date: Mon, 5 Aug 2024 14:44:20 +0200 Subject: [PATCH 18/84] Updated Configuration Settings --- .../livewire/settings/configuration.blade.php | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/resources/views/livewire/settings/configuration.blade.php b/resources/views/livewire/settings/configuration.blade.php index b5fb49d3e..e2f375037 100644 --- a/resources/views/livewire/settings/configuration.blade.php +++ b/resources/views/livewire/settings/configuration.blade.php @@ -10,23 +10,25 @@
    +

    Instance Settings

    +

    DNS Validation

    +
    + +
    -
    - -
    {{--
    --}} -
    -

    API

    +
    +

    API

    @@ -42,6 +44,10 @@ id="is_auto_update_enabled" label="Auto Update Coolify" /> @else + @if($is_auto_update_enabled) + + @endif + @endif From 94b2d67a6e5d9b1b834e1d83d9f7d44510c01165 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 5 Aug 2024 15:02:23 +0200 Subject: [PATCH 19/84] refactor: Update livewire polling interval in heading.blade.php --- .../views/livewire/project/application/heading.blade.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/resources/views/livewire/project/application/heading.blade.php b/resources/views/livewire/project/application/heading.blade.php index 2f0bba535..2704cc1f4 100644 --- a/resources/views/livewire/project/application/heading.blade.php +++ b/resources/views/livewire/project/application/heading.blade.php @@ -1,4 +1,4 @@ -