[Fixes: #10841] Serializing and deserializing ResourceBase and Dataset objects breaks validation check (#10843)

* issue10841 add convertion to internal types for DynamicModelSerializer

* issue10841 add convertion to internal types for DynamicModelSerializer

* issue10841 add convertion to internal types for DynamicModelSerializer

* issue10841 add convertion to internal types for DynamicModelSerializer

* issue10841 add convertion to internal types for DynamicModelSerializer

* - Fix formatting

---------

Co-authored-by: Giovanni Allegri <giohappy@gmail.com>
Co-authored-by: Alessio Fabiani <alessio.fabiani@geosolutionsgroup.com>

Author: 3 people

geonode/base/api/fields.py

@@ -0,0 +1,54 @@
+#########################################################################
+#
+# Copyright (C) 2016 OSGeo
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+#########################################################################
+
+import json
+
+from django.core.exceptions import ValidationError
+from dynamic_rest.fields.fields import DynamicRelationField
+
+
+class ComplexDynamicRelationField(DynamicRelationField):
+    def to_internal_value_single(self, data, serializer):
+        """Overwrite of DynamicRelationField implementation to handle complex data structure initialization
+
+        Args:
+            data (Optional[str, Dict]}): serialized or deserialized data from http calls (POST, GET ...)
+            serializer (DynamicModelSerializer): Serializer for the given data
+
+        Raises:
+            ValidationError: raised when requested data does not exist
+
+            django.db.models.QuerySet: return QuerySet object of the request or set data
+        """
+        related_model = serializer.Meta.model
+        if isinstance(data, str):
+            data = json.loads(data)
+
+        if isinstance(data, dict):
+            try:
+                if hasattr(serializer, "many") and serializer.many is True:
+                    return [serializer.get_model().objects.get(**d) for d in data]
+                return serializer.get_model().objects.get(**data)
+            except related_model.DoesNotExist:
+                raise ValidationError(
+                    "Invalid value for '%s': %s object with ID=%s not found"
+                    % (self.field_name, related_model.__name__, data)
+                )
+        else:
+            return super().to_internal_value_single(data, serializer)

geonode/base/api/serializers.py

@@ -16,16 +16,18 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 #########################################################################
-import json
+import logging
 from slugify import slugify
 from urllib.parse import urljoin
+import json
 
 from django.db.models import Q
 from django.conf import settings
 from django.contrib.auth.models import Group
 from django.forms.models import model_to_dict
 from django.contrib.auth import get_user_model
 from django.db.models.query import QuerySet
+from django.http import QueryDict
 
 from rest_framework import serializers
 from rest_framework_gis import fields
@@ -51,13 +53,11 @@
     ExtraMetadata,
 )
 from geonode.groups.models import GroupCategory, GroupProfile
-
+from geonode.base.api.fields import ComplexDynamicRelationField
 from geonode.utils import build_absolute_uri
 from geonode.security.utils import get_resources_with_perms
 from geonode.resource.models import ExecutionRequest
 
-import logging
-
 logger = logging.getLogger(__name__)
 
 
@@ -351,9 +351,6 @@ class Meta:
         model = ResourceBase
         fields = ("pk", "blob")
 
-    def to_internal_value(self, data):
-        return data
-
     def to_representation(self, value):
         data = ResourceBase.objects.filter(id=value)
         if data.exists() and data.count() == 1:
@@ -444,7 +441,7 @@ def __init__(self, *args, **kwargs):
         self.fields["bbox_polygon"] = fields.GeometryField(read_only=True, required=False)
         self.fields["ll_bbox_polygon"] = fields.GeometryField(read_only=True, required=False)
         self.fields["srid"] = serializers.CharField(required=False)
-        self.fields["group"] = DynamicRelationField(GroupSerializer, embed=True, many=False)
+        self.fields["group"] = ComplexDynamicRelationField(GroupSerializer, embed=True, many=False)
         self.fields["popular_count"] = serializers.CharField(required=False)
         self.fields["share_count"] = serializers.CharField(required=False)
         self.fields["rating"] = serializers.CharField(required=False)
@@ -463,28 +460,27 @@ def __init__(self, *args, **kwargs):
         self.fields["processed"] = serializers.BooleanField(read_only=True)
         self.fields["state"] = serializers.CharField(read_only=True)
         self.fields["sourcetype"] = serializers.CharField(read_only=True)
-
         self.fields["embed_url"] = EmbedUrlField(required=False)
         self.fields["thumbnail_url"] = ThumbnailUrlField(read_only=True)
-        self.fields["keywords"] = DynamicRelationField(SimpleHierarchicalKeywordSerializer, embed=False, many=True)
-        self.fields["tkeywords"] = DynamicRelationField(SimpleThesaurusKeywordSerializer, embed=False, many=True)
+        self.fields["keywords"] = ComplexDynamicRelationField(
+            SimpleHierarchicalKeywordSerializer, embed=False, many=True
+        )
+        self.fields["tkeywords"] = ComplexDynamicRelationField(SimpleThesaurusKeywordSerializer, embed=False, many=True)
         self.fields["regions"] = DynamicRelationField(SimpleRegionSerializer, embed=True, many=True, read_only=True)
-        self.fields["category"] = DynamicRelationField(SimpleTopicCategorySerializer, embed=True, many=False)
-        self.fields["restriction_code_type"] = DynamicRelationField(
+        self.fields["category"] = ComplexDynamicRelationField(SimpleTopicCategorySerializer, embed=True, many=False)
+        self.fields["restriction_code_type"] = ComplexDynamicRelationField(
             RestrictionCodeTypeSerializer, embed=True, many=False
         )
-        self.fields["license"] = DynamicRelationField(LicenseSerializer, embed=True, many=False)
-        self.fields["spatial_representation_type"] = DynamicRelationField(
+        self.fields["license"] = ComplexDynamicRelationField(LicenseSerializer, embed=True, many=False)
+        self.fields["spatial_representation_type"] = ComplexDynamicRelationField(
             SpatialRepresentationTypeSerializer, embed=True, many=False
         )
         self.fields["blob"] = serializers.JSONField(required=False, write_only=True)
         self.fields["is_copyable"] = serializers.BooleanField(read_only=True)
-
         self.fields["download_url"] = DownloadLinkField(read_only=True)
-
         self.fields["favorite"] = FavoriteField(read_only=True)
 
-    metadata = DynamicRelationField(ExtraMetadataSerializer, embed=False, many=True, deferred=True)
+    metadata = ComplexDynamicRelationField(ExtraMetadataSerializer, embed=False, many=True, deferred=True)
 
     class Meta:
         model = ResourceBase
@@ -596,6 +592,8 @@ def to_internal_value(self, data):
             data = json.loads(data)
         if "data" in data:
             data["blob"] = data.pop("data")
+        if isinstance(data, QueryDict):
+            data = data.dict()
         data = super(ResourceBaseSerializer, self).to_internal_value(data)
         return data
 

geonode/base/api/tests.py

@@ -32,20 +32,23 @@
 from uuid import uuid4
 from unittest.mock import patch
 from urllib.parse import urljoin
+from datetime import date, timedelta
 
 from django.urls import reverse
 from django.conf import settings
 from django.core.files.uploadedfile import SimpleUploadedFile
-from django.contrib.auth.models import Group
 from django.contrib.auth import get_user_model
 
 from rest_framework.test import APITestCase
+from rest_framework.renderers import JSONRenderer
+from rest_framework.parsers import JSONParser
 
 from guardian.shortcuts import get_anonymous_user
 from geonode.maps.models import Map, MapLayer
 from geonode.tests.base import GeoNodeBaseTestSupport
 
 from geonode.base import enumerations
+from geonode.base.api.serializers import ResourceBaseSerializer
 from geonode.groups.models import GroupMember, GroupProfile
 from geonode.thumbs.exceptions import ThumbnailError
 from geonode.layers.utils import get_files
@@ -56,6 +59,9 @@
     TopicCategory,
     ThesaurusKeyword,
     ExtraMetadata,
+    RestrictionCodeType,
+    License,
+    Group,
 )
 
 from geonode.layers.models import Dataset
@@ -707,6 +713,52 @@ def test_write_resources(self):
             self.assertEqual("321-12345-987654321", response.data["resource"]["doi"], response.data["resource"]["doi"])
             self.assertEqual(True, response.data["resource"]["is_published"], response.data["resource"]["is_published"])
 
+    def test_resource_serializer_validation(self):
+        """
+        Testing serializing and deserializing of a Dataset base on django-rest description:
+        https://www.django-rest-framework.org/api-guide/serializers/#deserializing-objects
+        """
+        owner, _ = get_user_model().objects.get_or_create(username="delet-owner")
+        title = "TEST DS TITLE"
+        HierarchicalKeyword.add_root(name="a")
+        keyword = HierarchicalKeyword.objects.get(slug="a")
+        keyword.add_child(name="a1")
+
+        Dataset(
+            title=title,
+            abstract="abstract",
+            name="test dataset",
+            alternate="Test Remove User",
+            attribution="Test Attribution",
+            uuid=str(uuid4()),
+            doi="test DOI",
+            edition=1,
+            maintenance_frequency=enumerations.UPDATE_FREQUENCIES[0],
+            constraints_other="Test Constrains other",
+            temporal_extent_start=date.today() - timedelta(days=1),
+            temporal_extent_end=date.today(),
+            data_quality_statement="Test data quality statement",
+            purpose="Test Purpose",
+            owner=owner,
+            subtype="raster",
+            category=TopicCategory.objects.get(identifier="elevation"),
+            resource_type="dataset",
+            license=License.objects.all().first(),
+            restriction_code_type=RestrictionCodeType.objects.all().first(),
+            group=Group.objects.all().first(),
+        ).save()
+
+        ds = ResourceBase.objects.get(title=title)
+        ds.keywords.add(HierarchicalKeyword.objects.get(slug="a1"))
+
+        serialized = ResourceBaseSerializer(ds)
+        json = JSONRenderer().render(serialized.data)
+        stream = BytesIO(json)
+        data = JSONParser().parse(stream)
+        self.assertIsInstance(data, dict)
+        se = ResourceBaseSerializer(data=data)
+        self.assertTrue(se.is_valid())
+
     def test_delete_user_with_resource(self):
         owner, created = get_user_model().objects.get_or_create(username="delet-owner")
         Dataset(