Skip to content

Commit ef89866

Browse files
[Fixes #9799] Thesaurus selectbox performance in metadata editor is very slow (#9800) (#9836)
* [Fixes #9799] Thesaurus selectbox performance in metadata editor is very slow Co-authored-by: mattiagiupponi <51856725+mattiagiupponi@users.noreply.github.com> Co-authored-by: Marcel Wallschläger <marcel.wallschlaeger@zalf.de>
1 parent 7244bdc commit ef89866

File tree

4 files changed

+78
-30
lines changed

4 files changed

+78
-30
lines changed

geonode/base/forms.py

+25-12
Original file line numberDiff line numberDiff line change
@@ -40,17 +40,18 @@
4040
from taggit.forms import TagField
4141
from tinymce.widgets import TinyMCE
4242
from django.contrib.admin.utils import flatten
43+
from django.utils.translation import get_language
44+
4345
from geonode.base.enumerations import ALL_LANGUAGES
4446
from geonode.base.models import (HierarchicalKeyword,
4547
License, Region, ResourceBase, Thesaurus,
4648
ThesaurusKeyword, ThesaurusKeywordLabel, ThesaurusLabel,
4749
TopicCategory)
4850
from geonode.base.widgets import TaggitSelect2Custom
51+
from geonode.base.fields import MultiThesauriField
4952
from geonode.documents.models import Document
5053
from geonode.layers.models import Dataset
51-
from django.utils.translation import get_language
52-
from .fields import MultiThesauriField
53-
from geonode.base.utils import validate_extra_metadata
54+
from geonode.base.utils import validate_extra_metadata, remove_country_from_lanugecode
5455

5556
logger = logging.getLogger(__name__)
5657

@@ -290,7 +291,14 @@ class Meta:
290291
)
291292

292293

294+
THESAURUS_RESULT_LIST_SEPERATOR = ("", "-------")
295+
296+
293297
class ThesaurusAvailableForm(forms.Form):
298+
299+
# seperator at beginning of thesaurus search result and between
300+
# results found in local language and alt label
301+
294302
def __init__(self, *args, **kwargs):
295303
super().__init__(*args, **kwargs)
296304
lang = get_language()
@@ -337,16 +345,21 @@ def _define_choicefield(self, item, required, tname, lang):
337345

338346
@staticmethod
339347
def _get_thesauro_keyword_label(item, lang):
340-
qs_local = []
341-
qs_non_local = [("", "------")]
342-
for key in ThesaurusKeyword.objects.filter(thesaurus_id=item.id):
343-
label = ThesaurusKeywordLabel.objects.filter(keyword=key).filter(lang=lang)
344-
if label.exists():
345-
qs_local.append((label.get().keyword.id, label.get().label))
346-
else:
347-
qs_non_local.append((key.id, key.alt_label))
348348

349-
return qs_non_local + qs_local
349+
keyword_id_for_given_thesaurus = ThesaurusKeyword.objects.filter(thesaurus_id=item)
350+
351+
# try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again
352+
qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id")
353+
if len(qs_keyword_ids) == 0:
354+
lang = remove_country_from_lanugecode(lang)
355+
qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id")
356+
357+
not_qs_ids = ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids).order_by("keyword_id").distinct("keyword_id").values("keyword_id")
358+
359+
qs_local = list(ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values_list("keyword_id", "label"))
360+
qs_non_local = list(keyword_id_for_given_thesaurus.filter(id__in=not_qs_ids).values_list("id", "alt_label"))
361+
362+
return [THESAURUS_RESULT_LIST_SEPERATOR] + qs_local + [THESAURUS_RESULT_LIST_SEPERATOR] + qs_non_local
350363

351364
@staticmethod
352365
def _get_thesauro_title_label(item, lang):

geonode/base/tests.py

+20-1
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
from geonode.storage.manager import storage_manager
5959
from django.test import Client, TestCase, override_settings, SimpleTestCase
6060
from django.shortcuts import reverse
61+
from django.utils import translation
6162

6263
from geonode.base.middleware import ReadOnlyMiddleware, MaintenanceMiddleware
6364
from geonode.base.templatetags.base_tags import get_visibile_resources, facets
@@ -73,7 +74,7 @@
7374
from django.core.files import File
7475
from django.core.management import call_command
7576
from django.core.management.base import CommandError
76-
from geonode.base.forms import ThesaurusAvailableForm
77+
from geonode.base.forms import ThesaurusAvailableForm, THESAURUS_RESULT_LIST_SEPERATOR
7778

7879

7980
test_image = Image.new('RGBA', size=(50, 50), color=(155, 0, 0))
@@ -923,6 +924,24 @@ def test_will_return_thesaurus_with_the_defaul_order_as_0(self):
923924
# will check if the second element of the tuple is the thesaurus_id = 1
924925
self.assertEqual(fields[1][0], '2')
925926

927+
def test_get_thesuro_key_label_with_cmd_language_code(self):
928+
# in python test language code look like 'en' this test checks if key label result function
929+
# returns correct results
930+
tid = 1
931+
translation.activate("en")
932+
t_available_form = ThesaurusAvailableForm(data={"1": tid})
933+
results = t_available_form._get_thesauro_keyword_label(tid, translation.get_language())
934+
self.assertNotEqual(results[1], THESAURUS_RESULT_LIST_SEPERATOR)
935+
936+
def test_get_thesuro_key_label_with_browser_language_code(self):
937+
# in browser scenario language does not look like "it", but rather include coutry code
938+
# like "it-it" this test checks if _get_thesauro_keyword_label can handle this
939+
tid = 1
940+
translation.activate("en-us")
941+
t_available_form = ThesaurusAvailableForm(data={"1": tid})
942+
results = t_available_form._get_thesauro_keyword_label(tid, translation.get_language())
943+
self.assertNotEqual(results[1], THESAURUS_RESULT_LIST_SEPERATOR)
944+
926945

927946
class TestFacets(TestCase):
928947

geonode/base/utils.py

+13
Original file line numberDiff line numberDiff line change
@@ -189,3 +189,16 @@ def validate_extra_metadata(data, instance):
189189
raise ValidationError(f"{e} at index {_index} for input json: {json.dumps(_metadata)}")
190190
# conerted because in this case, we can store a well formated json instead of the user input
191191
return data
192+
193+
194+
@staticmethod
195+
def remove_country_from_lanugecode(language: str):
196+
""" Remove country code (us) from language name (en-us)
197+
>>> remove_country_from_lanugecode("en-us")
198+
'en'
199+
"""
200+
if "-" not in language:
201+
return language
202+
203+
lang, _, _ = language.lower().partition("-")
204+
return lang

geonode/base/views.py

+20-17
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
from django.contrib.auth.decorators import login_required
3636
from django.contrib.auth.mixins import LoginRequiredMixin
3737
from django.urls import reverse
38+
from django.utils.translation import get_language
3839

3940
# Geonode dependencies
4041
from geonode.maps.models import Map
@@ -47,7 +48,7 @@
4748
from geonode.resource.manager import resource_manager
4849
from geonode.security.utils import get_visible_resources
4950
from geonode.notifications_helper import send_notification
50-
from geonode.base.utils import OwnerRightsRequestViewUtils
51+
from geonode.base.utils import OwnerRightsRequestViewUtils, remove_country_from_lanugecode
5152
from geonode.base.forms import UserAndGroupPermissionsForm
5253

5354
from geonode.base.forms import (
@@ -301,23 +302,25 @@ def get_results(self, context):
301302

302303
class ThesaurusAvailable(autocomplete.Select2QuerySetView):
303304
def get_queryset(self):
305+
304306
tid = self.request.GET.get("sysid")
305-
lang = self.request.GET.get("lang")
306-
qs_local = []
307-
qs_non_local = []
308-
for key in ThesaurusKeyword.objects.filter(thesaurus_id=tid):
309-
label = ThesaurusKeywordLabel.objects.filter(keyword=key).filter(lang=lang)
310-
if self.q:
311-
label = label.filter(label__icontains=self.q)
312-
if label.exists():
313-
qs_local.append(label.get())
314-
else:
315-
if self.q in key.alt_label:
316-
qs_non_local.append(key)
317-
elif not self.q:
318-
qs_non_local.append(key)
319-
320-
return qs_non_local + qs_local
307+
lang = get_language()
308+
keyword_id_for_given_thesaurus = ThesaurusKeyword.objects.filter(thesaurus_id=tid)
309+
310+
# try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again
311+
qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id")
312+
if len(qs_keyword_ids) == 0:
313+
lang = remove_country_from_lanugecode(lang)
314+
qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id")
315+
316+
not_qs_ids = ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids).order_by("keyword_id").distinct("keyword_id").values("keyword_id")
317+
qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus)
318+
if self.q:
319+
qs = qs.filter(label__istartswith=self.q)
320+
321+
qs_local = list(qs)
322+
qs_non_local = list(keyword_id_for_given_thesaurus.filter(id__in=not_qs_ids))
323+
return qs_local + qs_non_local
321324

322325
def get_results(self, context):
323326
return [

0 commit comments

Comments
 (0)