Skip to content

Commit fa88b0d

Browse files
author
Alessio Fabiani
authored
[Fixes GeoNode#7428] Port Celery optimizations from geonode-project (GeoNode#7438)
1 parent 9e85041 commit fa88b0d

5 files changed

+57
-11
lines changed

.env

+15
Original file line numberDiff line numberDiff line change
@@ -173,3 +173,18 @@ OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZK
173173
# GeoNode APIs
174174
API_LOCKDOWN=False
175175
TASTYPIE_APIKEY=
176+
177+
# CELERY
178+
179+
# expressed in KB
180+
# CELERY__MAX_MEMORY_PER_CHILD="200000"
181+
# ##
182+
# Note right autoscale value must coincide with worker concurrency value
183+
# CELERY__AUTOSCALE_VALUES="1,4"
184+
# CELERY__WORKER_CONCURRENCY="4"
185+
# ##
186+
# CELERY__OPTS="--without-gossip --without-mingle -Ofair -B -E"
187+
# CELERY__BEAT_SCHEDULE="/mnt/volumes/statics/celerybeat-schedule"
188+
# CELERY__LOG_LEVEL="INFO"
189+
# CELERY__LOG_FILE="/var/log/celery.log"
190+
# CELERY__WORKER_NAME="worker1@%h"

Dockerfile

+3
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ RUN chmod +x /usr/src/geonode/tasks.py \
4747
COPY celery.sh /usr/bin/celery-commands
4848
RUN chmod +x /usr/bin/celery-commands
4949

50+
COPY celery-cmd /usr/bin/celery-cmd
51+
RUN chmod +x /usr/bin/celery-cmd
52+
5053
# Prepraing dependencies
5154
RUN apt-get update && apt-get install -y devscripts build-essential debhelper pkg-kde-tools sharutils
5255
# RUN git clone https://salsa.debian.org/debian-gis-team/proj.git /tmp/proj

celery-cmd

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env bash
2+
# A configurable celery command.
3+
# Luca Pasquali <luca.pasquali@geo-solutions.it>
4+
CELERY_BIN=${CELERY_BIN:-"$(which celery||echo celery)"}
5+
CELERY_APP=${CELERY_APP:-"geonode.celery_app:app"}
6+
CELERY__STATE_DB=${CELERY__STATE_DB:-"/mnt/volumes/statics/worker.state"}
7+
# expressed in KB
8+
CELERY__MAX_MEMORY_PER_CHILD=${CELERY__MAX_MEMORY_PER_CHILD:-"200000"}
9+
CELERY__AUTOSCALE_VALUES=${CELERY__AUTOSCALE_VALUES:-"2,4"}
10+
CELERY__MAX_TASKS_PER_CHILD=${CELERY__MAX_TASKS_PER_CHILD:-"10"}
11+
CELERY__OPTS=${CELERY__OPTS:-"--without-gossip --without-mingle -Ofair -B -E"}
12+
CELERY__BEAT_SCHEDULE=${CELERY__BEAT_SCHEDULE:-"/mnt/volumes/statics/celerybeat-schedule"}
13+
CELERY__LOG_LEVEL=${CELERY__LOG_LEVEL:-"INFO"}
14+
CELERY__LOG_FILE=${CELERY__LOG_FILE:-"/var/log/celery.log"}
15+
CELERY__WORKER_NAME=${CELERY__WORKER_NAME:-"worker1@%h"}
16+
CELERY__WORKER_CONCURRENCY=${CELERY__WORKER_CONCURRENCY:-"4"}
17+
18+
$CELERY_BIN -A $CELERY_APP worker --autoscale=$CELERY__AUTOSCALE_VALUES \
19+
--max-memory-per-child=$CELERY__MAX_MEMORY_PER_CHILD $CELERY__OPTS \
20+
--statedb=$CELERY__STATE_DB -s $CELERY__BEAT_SCHEDULE \
21+
--loglevel=$CELERY__LOG_LEVEL -n $CELERY__WORKER_NAME -f $CELERY__LOG_FILE \
22+
--concurrency=$CELERY__WORKER_CONCURRENCY --max-tasks-per-child=$CELERY__MAX_TASKS_PER_CHILD

docker-compose-test.yml

+9-6
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,11 @@
11
version: '3.4'
22
services:
33

4-
# Common Django template for Geonode, Celery services below
4+
# Common Django template for GeoNode and Celery services below
55
x-common-django:
66
&default-common-django
77
image: geonode/geonode:latest
88
restart: on-failure
9-
build:
10-
context: ./
11-
dockerfile: Dockerfile
129
env_file:
1310
- .env_test
1411
volumes:
@@ -24,6 +21,9 @@ services:
2421
# Our custom django application. It includes Geonode.
2522
django:
2623
<< : *default-common-django
24+
build:
25+
context: ./
26+
dockerfile: Dockerfile
2727
container_name: django4${COMPOSE_PROJECT_NAME}
2828
healthcheck:
2929
test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://127.0.0.1:8001/"
@@ -39,11 +39,14 @@ services:
3939
# Celery worker that executes celery tasks created by Django.
4040
celery:
4141
<< : *default-common-django
42+
image: geonode/geonode:latest
4243
container_name: celery4${COMPOSE_PROJECT_NAME}
44+
depends_on:
45+
- django
4346
environment:
4447
- IS_CELERY=True
4548
entrypoint: ["/usr/src/geonode/entrypoint.sh"]
46-
command: "celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=/mnt/volumes/statics/worker.state -s /mnt/volumes/statics/celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log"
49+
command: "celery-cmd"
4750

4851
# Nginx is serving django static and media files and proxies to django and geonode
4952
geonode:
@@ -110,7 +113,7 @@ services:
110113

111114
# PostGIS database.
112115
db:
113-
# use geonode official postgis 11 image
116+
# use geonode official postgis 13 image
114117
image: geonode/postgis:13
115118
container_name: db4${COMPOSE_PROJECT_NAME}
116119
env_file:

docker-compose.yml

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,11 @@
11
version: '3.4'
22
services:
33

4-
# Common Django template for Geonode, Celery services below
4+
# Common Django template for GeoNode and Celery services below
55
x-common-django:
66
&default-common-django
77
image: geonode/geonode:latest
88
restart: on-failure
9-
build:
10-
context: ./
11-
dockerfile: Dockerfile
129
env_file:
1310
- .env
1411
volumes:
@@ -24,6 +21,9 @@ services:
2421
# Our custom django application. It includes Geonode.
2522
django:
2623
<< : *default-common-django
24+
build:
25+
context: ./
26+
dockerfile: Dockerfile
2727
container_name: django4${COMPOSE_PROJECT_NAME}
2828
healthcheck:
2929
test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://127.0.0.1:8001/"
@@ -39,11 +39,14 @@ services:
3939
# Celery worker that executes celery tasks created by Django.
4040
celery:
4141
<< : *default-common-django
42+
image: geonode/geonode:latest
4243
container_name: celery4${COMPOSE_PROJECT_NAME}
44+
depends_on:
45+
- django
4346
environment:
4447
- IS_CELERY=True
4548
entrypoint: ["/usr/src/geonode/entrypoint.sh"]
46-
command: "celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=/mnt/volumes/statics/worker.state -s /mnt/volumes/statics/celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log"
49+
command: "celery-cmd"
4750

4851
# Nginx is serving django static and media files and proxies to django and geonode
4952
geonode:

0 commit comments

Comments
 (0)