Skip to content

Commit 87548b2

Browse files
committed
[Hardening] Simple Celery options to [try] to prevent pre-forked workers delay and hang-up
1 parent 9c84a2d commit 87548b2

8 files changed

+10
-7
lines changed

celery.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
#!/bin/bash
22
nohup celery -A geonode.celery_app:app beat -l DEBUG -f /var/log/celery.log &>/dev/null &
3-
nohup celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log &>/dev/null &
3+
nohup celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log &>/dev/null &
44
nohup celery -A geonode.celery_app:app flower --auto_refresh=True --debug=False --broker=${BROKER_URL} --basic_auth=${ADMIN_USERNAME}:${ADMIN_PASSWORD} --address=0.0.0.0 --port=5555 &>/dev/null &

celery_dev.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ set -a
22
. ./.env_dev
33
set +a
44

5-
celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG --concurrency=10 -n worker1@%h
5+
celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG --concurrency=10 -n worker1@%h

docker-compose-test.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ services:
4343
environment:
4444
- IS_CELERY=True
4545
entrypoint: ["/usr/src/geonode/entrypoint.sh"]
46-
command: "celery -A geonode.celery_app:app worker -B -E --statedb=/mnt/volumes/statics/worker.state -s /mnt/volumes/statics/celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log"
46+
command: "celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=/mnt/volumes/statics/worker.state -s /mnt/volumes/statics/celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log"
4747

4848
# Nginx is serving django static and media files and proxies to django and geonode
4949
geonode:

docker-compose.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ services:
4343
environment:
4444
- IS_CELERY=True
4545
entrypoint: ["/usr/src/geonode/entrypoint.sh"]
46-
command: "celery -A geonode.celery_app:app worker -B -E --statedb=/mnt/volumes/statics/worker.state -s /mnt/volumes/statics/celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log"
46+
command: "celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=/mnt/volumes/statics/worker.state -s /mnt/volumes/statics/celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log"
4747

4848
# Nginx is serving django static and media files and proxies to django and geonode
4949
geonode:

geonode/settings.py

+3
Original file line numberDiff line numberDiff line change
@@ -1745,6 +1745,9 @@ def get_geonode_catalogue_service():
17451745
# Allow to recover from any unknown crash.
17461746
CELERY_ACKS_LATE = ast.literal_eval(os.environ.get('CELERY_ACKS_LATE', 'True'))
17471747

1748+
# Add a ten-minutes timeout to all Celery tasks.
1749+
CELERYD_TASK_SOFT_TIME_LIMIT = 600
1750+
17481751
# Set this to False in order to run async
17491752
_EAGER_FLAG = 'False' if ASYNC_SIGNALS else 'True'
17501753
CELERY_TASK_ALWAYS_EAGER = ast.literal_eval(os.environ.get('CELERY_TASK_ALWAYS_EAGER', _EAGER_FLAG))

pavement.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -612,7 +612,7 @@ def start_django(options):
612612
sh('%s python -W ignore manage.py runserver %s %s' % (settings, bind, foreground))
613613

614614
if 'django_celery_beat' not in INSTALLED_APPS:
615-
sh("{} celery -A geonode.celery_app:app worker -B -E \
615+
sh("{} celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E \
616616
--statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG \
617617
--concurrency=10 -n worker1@%h -f celery.log {}".format( # noqa
618618
settings,

scripts/spcgeonode/docker-compose.override.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ services:
1818
command: "uwsgi --chdir=/spcgeonode --module=geonode.wsgi --socket=:8000 --http=127.0.0.1:8001 --processes=5 --py-autoreload=2"
1919
celery:
2020
<< : *default-common-django
21-
command: 'celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG --concurrency=10 -n worker1@%h'
21+
command: 'celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG --concurrency=10 -n worker1@%h'
2222

2323
nginx:
2424
image: geonode/spcgeonode:nginx-3.1

scripts/spcgeonode/docker-compose.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ services:
7272
celery:
7373
<< : *default-common-django
7474
entrypoint: []
75-
command: 'celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h'
75+
command: 'celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h'
7676

7777
# Nginx is serving django static and media files and proxies to django and geonode
7878
nginx:

0 commit comments

Comments
 (0)