[Fixes GeoNode#7428] Port Celery optimizations from geonode-project (GeoNode#7438)

Author: Alessio Fabiani

.env

@@ -173,3 +173,18 @@ OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZK
 # GeoNode APIs
 API_LOCKDOWN=False
 TASTYPIE_APIKEY=
+
+# CELERY
+
+# expressed in KB
+# CELERY__MAX_MEMORY_PER_CHILD="200000"
+# ## 
+# Note right autoscale value must coincide with worker concurrency value
+# CELERY__AUTOSCALE_VALUES="1,4" 
+# CELERY__WORKER_CONCURRENCY="4"
+# ##
+# CELERY__OPTS="--without-gossip --without-mingle -Ofair -B -E"
+# CELERY__BEAT_SCHEDULE="/mnt/volumes/statics/celerybeat-schedule"
+# CELERY__LOG_LEVEL="INFO"
+# CELERY__LOG_FILE="/var/log/celery.log"
+# CELERY__WORKER_NAME="worker1@%h"

Dockerfile

@@ -47,6 +47,9 @@ RUN chmod +x /usr/src/geonode/tasks.py \
 COPY celery.sh /usr/bin/celery-commands
 RUN chmod +x /usr/bin/celery-commands
 
+COPY celery-cmd /usr/bin/celery-cmd
+RUN chmod +x /usr/bin/celery-cmd
+
 # Prepraing dependencies
 RUN apt-get update && apt-get install -y devscripts build-essential debhelper pkg-kde-tools sharutils
 # RUN git clone https://salsa.debian.org/debian-gis-team/proj.git /tmp/proj

celery-cmd

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# A configurable celery command.
+# Luca Pasquali <luca.pasquali@geo-solutions.it>
+CELERY_BIN=${CELERY_BIN:-"$(which celery||echo celery)"}
+CELERY_APP=${CELERY_APP:-"geonode.celery_app:app"}
+CELERY__STATE_DB=${CELERY__STATE_DB:-"/mnt/volumes/statics/worker.state"}
+# expressed in KB
+CELERY__MAX_MEMORY_PER_CHILD=${CELERY__MAX_MEMORY_PER_CHILD:-"200000"}
+CELERY__AUTOSCALE_VALUES=${CELERY__AUTOSCALE_VALUES:-"2,4"}
+CELERY__MAX_TASKS_PER_CHILD=${CELERY__MAX_TASKS_PER_CHILD:-"10"}
+CELERY__OPTS=${CELERY__OPTS:-"--without-gossip --without-mingle -Ofair -B -E"}
+CELERY__BEAT_SCHEDULE=${CELERY__BEAT_SCHEDULE:-"/mnt/volumes/statics/celerybeat-schedule"}
+CELERY__LOG_LEVEL=${CELERY__LOG_LEVEL:-"INFO"}
+CELERY__LOG_FILE=${CELERY__LOG_FILE:-"/var/log/celery.log"}
+CELERY__WORKER_NAME=${CELERY__WORKER_NAME:-"worker1@%h"}
+CELERY__WORKER_CONCURRENCY=${CELERY__WORKER_CONCURRENCY:-"4"}
+
+$CELERY_BIN -A $CELERY_APP worker --autoscale=$CELERY__AUTOSCALE_VALUES \
+    --max-memory-per-child=$CELERY__MAX_MEMORY_PER_CHILD $CELERY__OPTS \
+    --statedb=$CELERY__STATE_DB -s $CELERY__BEAT_SCHEDULE \
+    --loglevel=$CELERY__LOG_LEVEL -n $CELERY__WORKER_NAME -f $CELERY__LOG_FILE \
+    --concurrency=$CELERY__WORKER_CONCURRENCY --max-tasks-per-child=$CELERY__MAX_TASKS_PER_CHILD

docker-compose-test.yml

@@ -1,14 +1,11 @@
 version: '3.4'
 services:
 
-# Common Django template for Geonode, Celery services below
+# Common Django template for GeoNode and Celery services below
 x-common-django:
   &default-common-django
   image: geonode/geonode:latest
   restart: on-failure
-  build:
-    context: ./
-    dockerfile: Dockerfile
   env_file:
     - .env_test
   volumes:
@@ -24,6 +21,9 @@ services:
   # Our custom django application. It includes Geonode.
   django:
     << : *default-common-django
+    build:
+      context: ./
+      dockerfile: Dockerfile
     container_name: django4${COMPOSE_PROJECT_NAME}
     healthcheck:
       test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://127.0.0.1:8001/"
@@ -39,11 +39,14 @@ services:
   # Celery worker that executes celery tasks created by Django.
   celery:
     << : *default-common-django
+    image: geonode/geonode:latest
     container_name: celery4${COMPOSE_PROJECT_NAME}
+    depends_on:
+      - django
     environment:
       - IS_CELERY=True
     entrypoint: ["/usr/src/geonode/entrypoint.sh"]
-    command: "celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=/mnt/volumes/statics/worker.state -s /mnt/volumes/statics/celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log"
+    command: "celery-cmd"
 
   # Nginx is serving django static and media files and proxies to django and geonode
   geonode:
@@ -110,7 +113,7 @@ services:
 
   # PostGIS database.
   db:
-    # use geonode official postgis 11 image
+    # use geonode official postgis 13 image
     image: geonode/postgis:13
     container_name: db4${COMPOSE_PROJECT_NAME}
     env_file:

docker-compose.yml

@@ -1,14 +1,11 @@
 version: '3.4'
 services:
 
-# Common Django template for Geonode, Celery services below
+# Common Django template for GeoNode and Celery services below
 x-common-django:
   &default-common-django
   image: geonode/geonode:latest
   restart: on-failure
-  build:
-    context: ./
-    dockerfile: Dockerfile
   env_file:
     - .env
   volumes:
@@ -24,6 +21,9 @@ services:
   # Our custom django application. It includes Geonode.
   django:
     << : *default-common-django
+    build:
+      context: ./
+      dockerfile: Dockerfile
     container_name: django4${COMPOSE_PROJECT_NAME}
     healthcheck:
       test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://127.0.0.1:8001/"
@@ -39,11 +39,14 @@ services:
   # Celery worker that executes celery tasks created by Django.
   celery:
     << : *default-common-django
+    image: geonode/geonode:latest
     container_name: celery4${COMPOSE_PROJECT_NAME}
+    depends_on:
+      - django
     environment:
       - IS_CELERY=True
     entrypoint: ["/usr/src/geonode/entrypoint.sh"]
-    command: "celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=/mnt/volumes/statics/worker.state -s /mnt/volumes/statics/celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log"
+    command: "celery-cmd"
 
   # Nginx is serving django static and media files and proxies to django and geonode
   geonode: