Skip to content

Commit e357f55

Browse files
author
Alessio Fabiani
authored
[Fixes GeoNode#4521] Migrate Tests to CircleCI: smoke, apps, upload, analytics and CSW (GeoNode#6760)
* - Test Fixes: CircleCI * - Test Fixes: CircleCI * - Test Fixes: CircleCI - Moving "upload" tests from travis to circleci * - Update .sh modes * - Test Fixes: CircleCI - Moving "upload" tests from travis to circleci * - Test Fixes: CircleCI - Moving "upload" tests from travis to circleci * - Test Fixes: CircleCI - Moving "upload" tests from travis to circleci * - TESTING: upload tests progress request errors * - TESTING: upload tests progress request errors * - TESTING: upload tests progress request errors * - Restore the whole suite: monitoring, integration and upload * - TESTING: upload tests progress request errors
1 parent 87b991d commit e357f55

16 files changed

+511
-138
lines changed

‎.circleci/config.yml

+34-18
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
3535
- run:
3636
name: Build the stack
37-
command: docker-compose -f docker-compose.yml build
37+
command: docker-compose -f docker-compose-test.yml build
3838
working_directory: ./
3939

4040
- when:
@@ -54,7 +54,7 @@ jobs:
5454

5555
- run:
5656
name: Start the stack
57-
command: docker-compose -f docker-compose.yml up -d
57+
command: docker-compose -f docker-compose-test.yml up -d
5858
working_directory: ./
5959

6060
- run:
@@ -99,65 +99,81 @@ jobs:
9999
- run:
100100
name: Run test suite
101101
command: |
102-
docker-compose -f docker-compose.yml exec db psql -U postgres -c 'SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();'
103-
docker-compose -f docker-compose.yml exec db createdb -U postgres -T postgres test_postgres
104-
docker-compose -f docker-compose.yml exec db createdb -U postgres -T postgres test_geonode
105-
docker-compose -f docker-compose.yml exec db createdb -U postgres -T postgres test_geonode_data
106-
docker-compose -f docker-compose.yml exec db psql -U postgres -d test_geonode -c 'CREATE EXTENSION IF NOT EXISTS postgis;'
107-
docker-compose -f docker-compose.yml exec db psql -U postgres -d test_geonode_data -c 'CREATE EXTENSION IF NOT EXISTS postgis;'
108-
docker-compose -f docker-compose.yml exec django bash -c '<<parameters.test_suite>>'
102+
docker-compose -f docker-compose-test.yml exec db psql -U postgres -c 'SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();'
103+
docker-compose -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_postgres
104+
docker-compose -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_geonode
105+
docker-compose -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_geonode_data
106+
docker-compose -f docker-compose-test.yml exec db psql -U postgres -d test_geonode -c 'CREATE EXTENSION IF NOT EXISTS postgis;'
107+
docker-compose -f docker-compose-test.yml exec db psql -U postgres -d test_geonode_data -c 'CREATE EXTENSION IF NOT EXISTS postgis;'
108+
docker-compose -f docker-compose-test.yml exec django bash -c '<<parameters.test_suite>>'
109109
working_directory: ./
110110
- run:
111111
name: Run pep8 checks
112112
command: |
113-
docker-compose -f docker-compose.yml exec django bash -c 'flake8 geonode'
114-
docker-compose -f docker-compose.yml exec django bash -c 'codecov; bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8'
113+
docker-compose -f docker-compose-test.yml exec django bash -c 'flake8 geonode'
114+
docker-compose -f docker-compose-test.yml exec django bash -c 'codecov; bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8'
115115
working_directory: ./
116116

117117
workflows:
118118

119119
commit:
120120
jobs:
121121
- build:
122-
name: geonode_test_suite
122+
name: geonode_test_suite_smoke
123123
load_docker_cache: true
124124
save_docker_cache: true
125-
test_suite: coverage run --branch --source=geonode manage.py test -v 3 --keepdb geonode.tests.smoke $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a+'\''.tests'\'' for a in settings.GEONODE_APPS]))")
125+
test_suite: coverage run --branch --source=geonode manage.py test -v 3 --keepdb geonode.tests.smoke
126+
- build:
127+
name: geonode_test_suite
128+
load_docker_cache: true
129+
save_docker_cache: false
130+
test_suite: coverage run --branch --source=geonode manage.py test -v 3 --keepdb $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a+'\''.tests'\'' for a in settings.GEONODE_APPS]))")
131+
requires:
132+
- geonode_test_suite_smoke
126133
- build:
127134
name: geonode_test_integration_csw
128135
load_docker_cache: true
129136
save_docker_cache: false
130137
test_suite: ./test_csw.sh
131138
requires:
132-
- geonode_test_suite
139+
- geonode_test_suite_smoke
140+
- build:
141+
name: geonode_test_integration_upload
142+
load_docker_cache: true
143+
save_docker_cache: false
144+
test_suite: ./test_upload.sh
145+
requires:
146+
- geonode_test_suite_smoke
133147
- build:
134148
name: geonode_test_integration_monitoring
135149
load_docker_cache: true
136150
save_docker_cache: false
137151
test_suite: coverage run --branch --source=geonode manage.py test -v 3 --keepdb geonode.tests.smoke geonode.monitoring.tests.integration
138152
requires:
139-
- geonode_test_suite
153+
- geonode_test_suite_smoke
154+
155+
# TODO
140156
# - build:
141157
# name: tests_integration
142158
# load_docker_cache: true
143159
# save_docker_cache: false
144160
# test_suite: 'geonode.tests.integration'
145161
# requires:
146-
# - geonode_test_suite
162+
# - geonode_test_suite_smoke
147163
# - build:
148164
# name: tests_geoserver_integration
149165
# load_docker_cache: true
150166
# save_docker_cache: false
151167
# test_suite: 'geonode.geoserver.tests.integration'
152168
# requires:
153-
# - geonode_test_suite
169+
# - geonode_test_suite_smoke
154170
# - build:
155171
# name: tests_upload_integration
156172
# load_docker_cache: true
157173
# save_docker_cache: false
158174
# test_suite: 'geonode.upload.tests.integration'
159175
# requires:
160-
# - geonode_test_suite
176+
# - geonode_test_suite_smoke
161177

162178
# nightly:
163179
# triggers:

‎.env_test

+170
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
COMPOSE_PROJECT_NAME=geonode
2+
DOCKER_HOST_IP=
3+
DOCKER_ENV=production
4+
# See https://github.com/geosolutions-it/geonode-generic/issues/28
5+
# to see why we force API version to 1.24
6+
DOCKER_API_VERSION="1.24"
7+
BACKUPS_VOLUME_DRIVER=local
8+
9+
C_FORCE_ROOT=1
10+
11+
DEBUG=False
12+
13+
DJANGO_SETTINGS_MODULE=geonode.settings
14+
GEONODE_INSTANCE_NAME=geonode
15+
GEONODE_LB_HOST_IP
16+
GEONODE_LB_PORT
17+
18+
19+
POSTGRES_USER=postgres
20+
POSTGRES_PASSWORD=postgres
21+
GEONODE_DATABASE=geonode
22+
GEONODE_DATABASE_PASSWORD=geonode
23+
GEONODE_GEODATABASE=geonode_data
24+
GEONODE_GEODATABASE_PASSWORD=geonode_data
25+
GEONODE_DATABASE_SCHEMA=public
26+
GEONODE_GEODATABASE_SCHEMA=public
27+
DATABASE_URL=postgis://geonode:geonode@db:5432/geonode
28+
GEODATABASE_URL=postgis://geonode_data:geonode_data@db:5432/geonode_data
29+
GEONODE_DB_CONN_MAX_AGE=0
30+
GEONODE_DB_CONN_TOUT=5
31+
DEFAULT_BACKEND_DATASTORE=datastore
32+
BROKER_URL=amqp://guest:guest@rabbitmq:5672/
33+
ASYNC_SIGNALS=True
34+
35+
36+
SITEURL=http://localhost:8001/
37+
38+
STATIC_ROOT=/mnt/volumes/statics/static/
39+
MEDIA_ROOT=/mnt/volumes/statics/uploaded/
40+
GEOIP_PATH=/mnt/volumes/statics/geoip.db
41+
42+
ALLOWED_HOSTS=['django', '*']
43+
44+
DEFAULT_BACKEND_UPLOADER=geonode.importer
45+
TIME_ENABLED=True
46+
MOSAIC_ENABLED=False
47+
HAYSTACK_SEARCH=False
48+
HAYSTACK_ENGINE_URL=http://elasticsearch:9200/
49+
HAYSTACK_ENGINE_INDEX_NAME=haystack
50+
HAYSTACK_SEARCH_RESULTS_PER_PAGE=200
51+
52+
CACHE_BUSTING_STATIC_ENABLED=False
53+
CACHE_BUSTING_MEDIA_ENABLED=False
54+
55+
MEMCACHED_ENABLED=False
56+
MEMCACHED_BACKEND=django.core.cache.backends.memcached.MemcachedCache
57+
MEMCACHED_LOCATION=127.0.0.1:11211
58+
MEMCACHED_LOCK_EXPIRE=3600
59+
MEMCACHED_LOCK_TIMEOUT=10
60+
61+
MAX_DOCUMENT_SIZE=2
62+
CLIENT_RESULTS_LIMIT=5
63+
API_LIMIT_PER_PAGE=1000
64+
65+
# HTTPD Server
66+
GEONODE_LB_HOST_IP=localhost
67+
GEONODE_LB_PORT=80
68+
69+
# IP or domain name and port where the server can be reached on HTTPS (leave HOST empty if you want to use HTTP only)
70+
# port where the server can be reached on HTTPS
71+
HTTP_HOST=localhost
72+
HTTPS_HOST=
73+
74+
HTTP_PORT=8001
75+
HTTPS_PORT=443
76+
77+
# Let's Encrypt certificates for https encryption. You must have a domain name as HTTPS_HOST (doesn't work
78+
# with an ip) and it must be reachable from the outside. This can be one of the following :
79+
# disabled : we do not get a certificate at all (a placeholder certificate will be used)
80+
# staging : we get staging certificates (are invalid, but allow to test the process completely and have much higher limit rates)
81+
# production : we get a normal certificate (default)
82+
LETSENCRYPT_MODE=disabled
83+
# LETSENCRYPT_MODE=staging
84+
# LETSENCRYPT_MODE=production
85+
86+
RESOLVER=127.0.0.11
87+
88+
# GIS Server
89+
GEOSERVER_WEB_UI_LOCATION=http://localhost:8001/geoserver/
90+
GEOSERVER_PUBLIC_LOCATION=http://localhost:8001/geoserver/
91+
GEOSERVER_LOCATION=http://geoserver:8080/geoserver/
92+
GEOSERVER_ADMIN_USER=admin
93+
GEOSERVER_ADMIN_PASSWORD=geoserver
94+
95+
OGC_REQUEST_TIMEOUT=30
96+
OGC_REQUEST_MAX_RETRIES=1
97+
OGC_REQUEST_BACKOFF_FACTOR=0.3
98+
OGC_REQUEST_POOL_MAXSIZE=10
99+
OGC_REQUEST_POOL_CONNECTIONS=10
100+
101+
# GIS Client
102+
GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore
103+
MAPBOX_ACCESS_TOKEN=
104+
BING_API_KEY=
105+
GOOGLE_API_KEY=
106+
107+
# Monitoring
108+
MONITORING_ENABLED=True
109+
MONITORING_DATA_TTL=365
110+
USER_ANALYTICS_ENABLED=True
111+
USER_ANALYTICS_GZIP=True
112+
CENTRALIZED_DASHBOARD_ENABLED=False
113+
MONITORING_SERVICE_NAME=local-geonode
114+
MONITORING_HOST_NAME=geonode
115+
116+
# Other Options/Contribs
117+
MODIFY_TOPICCATEGORY=True
118+
AVATAR_GRAVATAR_SSL=True
119+
AVATAR_DEFAULT_URL=/geonode/img/avatar.png
120+
121+
EXIF_ENABLED=True
122+
CREATE_LAYER=True
123+
FAVORITE_ENABLED=True
124+
125+
# #################
126+
# Security
127+
# #################
128+
# Admin Settings
129+
ADMIN_USERNAME=admin
130+
ADMIN_PASSWORD=admin
131+
ADMIN_EMAIL=admin@localhost
132+
133+
# EMAIL Notifications
134+
EMAIL_ENABLE=False
135+
DJANGO_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend
136+
DJANGO_EMAIL_HOST=localhost
137+
DJANGO_EMAIL_PORT=25
138+
DJANGO_EMAIL_HOST_USER=
139+
DJANGO_EMAIL_HOST_PASSWORD=
140+
DJANGO_EMAIL_USE_TLS=False
141+
DJANGO_EMAIL_USE_SSL=False
142+
DEFAULT_FROM_EMAIL='GeoNode <no-reply@geonode.org>'
143+
144+
# Session/Access Control
145+
LOCKDOWN_GEONODE=False
146+
CORS_ORIGIN_ALLOW_ALL=True
147+
X_FRAME_OPTIONS=ALLOW-FROM ALL
148+
SESSION_EXPIRED_CONTROL_ENABLED=True
149+
DEFAULT_ANONYMOUS_VIEW_PERMISSION=True
150+
DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True
151+
152+
# Users Registration
153+
ACCOUNT_OPEN_SIGNUP=True
154+
ACCOUNT_EMAIL_REQUIRED=True
155+
ACCOUNT_APPROVAL_REQUIRED=False
156+
ACCOUNT_CONFIRM_EMAIL_ON_GET=False
157+
ACCOUNT_EMAIL_VERIFICATION=none
158+
ACCOUNT_EMAIL_CONFIRMATION_EMAIL=False
159+
ACCOUNT_EMAIL_CONFIRMATION_REQUIRED=False
160+
ACCOUNT_AUTHENTICATION_METHOD=username_email
161+
AUTO_ASSIGN_REGISTERED_MEMBERS_TO_REGISTERED_MEMBERS_GROUP_NAME=True
162+
163+
# OAuth2
164+
OAUTH2_API_KEY=
165+
OAUTH2_CLIENT_ID=Jrchz2oPY3akmzndmgUTYrs9gczlgoV20YPSvqaV
166+
OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZKFfePBHYXCLd7B8NlkfDBY9HKeIQPcy5Cp08KQNpRHQbjpLItDHv12GvkSeXp6OxaUETv3
167+
168+
# GeoNode APIs
169+
API_LOCKDOWN=False
170+
TASTYPIE_APIKEY=

‎.travis.yml

-43
Original file line numberDiff line numberDiff line change
@@ -105,49 +105,6 @@ jobs:
105105
- SESSION_EXPIRED_CONTROL_ENABLED: 'True'
106106
- CELERY_ALWAYS_EAGER: 'True'
107107

108-
- stage: upload-tests
109-
name: "GeoServer-backend Upload Tests"
110-
virtualenv:
111-
system_site_packages: false
112-
env:
113-
- BACKEND: 'geonode.geoserver'
114-
- DOCKER_COMPOSE_VERSION: 1.19.0
115-
- GEOSERVER_SERVER_URL: http://localhost:8080/geoserver/
116-
- GEOSERVER_SERVER_PORT: 8080
117-
- ON_TRAVIS: 'True'
118-
- TEST_RUNNER_KEEPDB: 'True'
119-
- TEST_RUN_INTEGRATION: 'True'
120-
- TEST_RUN_INTEGRATION_SERVER: 'False'
121-
- TEST_RUN_INTEGRATION_UPLOAD: 'True'
122-
- TEST_RUN_INTEGRATION_MONITORING: 'False'
123-
- TEST_RUN_INTEGRATION_CSW: 'False'
124-
- TEST_RUN_INTEGRATION_BDD: 'False'
125-
- MONITORING_ENABLED: 'False'
126-
- SESSION_EXPIRED_CONTROL_ENABLED: 'True'
127-
- CELERY_ALWAYS_EAGER: 'True'
128-
129-
- stage: anlytics-tests
130-
name: "GeoServer-backend Monitoring Tests"
131-
virtualenv:
132-
system_site_packages: false
133-
env:
134-
- BACKEND: 'geonode.geoserver'
135-
- DOCKER_COMPOSE_VERSION: 1.19.0
136-
- GEOSERVER_SERVER_URL: http://localhost:8080/geoserver/
137-
- GEOSERVER_SERVER_PORT: 8080
138-
- ON_TRAVIS: 'True'
139-
- TEST_RUNNER_KEEPDB: 'True'
140-
- TEST_RUN_INTEGRATION: 'True'
141-
- TEST_RUN_INTEGRATION_SERVER: 'False'
142-
- TEST_RUN_INTEGRATION_UPLOAD: 'False'
143-
- TEST_RUN_INTEGRATION_MONITORING: 'True'
144-
- TEST_RUN_INTEGRATION_CSW: 'False'
145-
- TEST_RUN_INTEGRATION_BDD: 'False'
146-
- MONITORING_ENABLED: 'True'
147-
- USER_ANALYTICS_ENABLED: 'True'
148-
- SESSION_EXPIRED_CONTROL_ENABLED: 'True'
149-
- CELERY_ALWAYS_EAGER: 'True'
150-
151108
- stage: integration-tests
152109
name: "Backend Integration Tests"
153110
virtualenv:

‎celery.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
#!/bin/bash
22
nohup celery -A geonode.celery_app:app beat -l DEBUG -f /var/log/celery.log &>/dev/null &
3-
nohup celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log &>/dev/null &
3+
nohup celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=INFO --concurrency=2 -n worker1@%h -f /var/log/celery.log &>/dev/null &
44
nohup celery -A geonode.celery_app:app flower --auto_refresh=True --debug=False --broker=${BROKER_URL} --basic_auth=${ADMIN_USERNAME}:${ADMIN_PASSWORD} --address=0.0.0.0 --port=5555 &>/dev/null &

‎celery_dev.sh

100644100755
+1-1
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ set -a
22
. ./.env_dev
33
set +a
44

5-
celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG --concurrency=10 -n worker1@%h
5+
celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG --concurrency=2 -n worker1@%h

0 commit comments

Comments
 (0)