Skip to content

Commit 216e3dd

Browse files
mattiagiupponietj
authored and
giohappy
committed
[Fixes GeoNode#12124] GNIP 100: Assets (GeoNode#12335)
* [Fixes GeoNode#12124] GNIP 100: Assets --------- Co-authored-by: etj <e.tajariol@gmail.com>
1 parent eb9bfe0 commit 216e3dd

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1460
-178
lines changed

‎.env.sample

+1
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ SECRET_KEY='{secret_key}'
168168

169169
STATIC_ROOT=/mnt/volumes/statics/static/
170170
MEDIA_ROOT=/mnt/volumes/statics/uploaded/
171+
ASSETS_ROOT=/mnt/volumes/statics/assets/
171172
GEOIP_PATH=/mnt/volumes/statics/geoip.db
172173

173174
CACHE_BUSTING_STATIC_ENABLED=False

‎geonode/assets/__init__.py

Whitespace-only changes.

‎geonode/assets/apps.py

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#########################################################################
2+
#
3+
# Copyright (C) 2016 OSGeo
4+
#
5+
# This program is free software: you can redistribute it and/or modify
6+
# it under the terms of the GNU General Public License as published by
7+
# the Free Software Foundation, either version 3 of the License, or
8+
# (at your option) any later version.
9+
#
10+
# This program is distributed in the hope that it will be useful,
11+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
# GNU General Public License for more details.
14+
#
15+
# You should have received a copy of the GNU General Public License
16+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
#
18+
#########################################################################
19+
from django.apps import AppConfig
20+
21+
from geonode.notifications_helper import NotificationsAppConfigBase
22+
23+
24+
class BaseAppConfig(NotificationsAppConfigBase, AppConfig):
25+
name = "geonode.assets"
26+
27+
def ready(self):
28+
super().ready()
29+
run_setup_hooks()
30+
31+
32+
def run_setup_hooks(*args, **kwargs):
33+
from geonode.assets.handlers import asset_handler_registry
34+
35+
asset_handler_registry.init_registry()

‎geonode/assets/handlers.py

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import logging
2+
3+
from django.conf import settings
4+
from django.http import HttpResponse
5+
from django.utils.module_loading import import_string
6+
7+
from geonode.assets.models import Asset
8+
9+
logger = logging.getLogger(__name__)
10+
11+
12+
class AssetHandlerInterface:
13+
14+
def handled_asset_class(self):
15+
raise NotImplementedError()
16+
17+
def create(self, title, description, type, owner, *args, **kwargs):
18+
raise NotImplementedError()
19+
20+
def remove_data(self, asset: Asset, **kwargs):
21+
raise NotImplementedError()
22+
23+
def replace_data(self, asset: Asset, files: list):
24+
raise NotImplementedError()
25+
26+
def clone(self, asset: Asset) -> Asset:
27+
"""
28+
Creates a copy in the DB and copies the underlying data as well
29+
"""
30+
raise NotImplementedError()
31+
32+
def create_link_url(self, asset: Asset) -> str:
33+
raise NotImplementedError()
34+
35+
def get_download_handler(self, asset: Asset):
36+
raise NotImplementedError()
37+
38+
def get_storage_manager(self, asset):
39+
raise NotImplementedError()
40+
41+
42+
class AssetDownloadHandlerInterface:
43+
44+
def create_response(self, asset: Asset, attachment: bool = False, basename=None) -> HttpResponse:
45+
raise NotImplementedError()
46+
47+
48+
class AssetHandlerRegistry:
49+
_registry = {}
50+
_default_handler = None
51+
52+
def init_registry(self):
53+
self.register_asset_handlers()
54+
self.set_default_handler()
55+
56+
def register_asset_handlers(self):
57+
for module_path in settings.ASSET_HANDLERS:
58+
handler = import_string(module_path)
59+
self.register(handler)
60+
logger.info(f"Registered Asset handlers: {', '.join(settings.ASSET_HANDLERS)}")
61+
62+
def set_default_handler(self):
63+
# check if declared class is registered
64+
for handler in self._registry.values():
65+
if ".".join([handler.__class__.__module__, handler.__class__.__name__]) == settings.DEFAULT_ASSET_HANDLER:
66+
self._default_handler = handler
67+
break
68+
69+
if self._default_handler is None:
70+
logger.error(f"Could not set default asset handler class {settings.DEFAULT_ASSET_HANDLER}")
71+
else:
72+
logger.info(f"Default Asset handler {settings.DEFAULT_ASSET_HANDLER}")
73+
74+
def register(self, asset_handler_class):
75+
self._registry[asset_handler_class.handled_asset_class()] = asset_handler_class()
76+
77+
def get_default_handler(self) -> AssetHandlerInterface:
78+
return self._default_handler
79+
80+
def get_handler(self, asset):
81+
asset_cls = asset if isinstance(asset, type) else asset.__class__
82+
ret = self._registry.get(asset_cls, None)
83+
if not ret:
84+
logger.warning(f"Could not find asset handler for {asset_cls}::{asset.__class__}")
85+
logger.warning("Available asset types:")
86+
for k, v in self._registry.items():
87+
logger.warning(f"{k} --> {v.__class__.__name__}")
88+
return ret
89+
90+
91+
asset_handler_registry = AssetHandlerRegistry()

‎geonode/assets/local.py

+159
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
import datetime
2+
import logging
3+
import os
4+
5+
from django.conf import settings
6+
from django.http import HttpResponse
7+
from django.urls import reverse
8+
from django_downloadview import DownloadResponse
9+
10+
from geonode.assets.handlers import asset_handler_registry, AssetHandlerInterface, AssetDownloadHandlerInterface
11+
from geonode.assets.models import LocalAsset
12+
from geonode.storage.manager import DefaultStorageManager, StorageManager
13+
from geonode.utils import build_absolute_uri, mkdtemp
14+
15+
logger = logging.getLogger(__name__)
16+
17+
_asset_storage_manager = StorageManager(
18+
concrete_storage_manager=DefaultStorageManager(location=os.path.dirname(settings.ASSETS_ROOT))
19+
)
20+
21+
22+
class LocalAssetHandler(AssetHandlerInterface):
23+
@staticmethod
24+
def handled_asset_class():
25+
return LocalAsset
26+
27+
def get_download_handler(self, asset):
28+
return LocalAssetDownloadHandler()
29+
30+
def get_storage_manager(self, asset):
31+
return _asset_storage_manager
32+
33+
def _create_asset_dir(self):
34+
return os.path.normpath(
35+
mkdtemp(dir=settings.ASSETS_ROOT, prefix=datetime.datetime.now().strftime("%Y%m%d%H%M%S"))
36+
)
37+
38+
def create(self, title, description, type, owner, files=None, clone_files=False, *args, **kwargs):
39+
if not files:
40+
raise ValueError("File(s) expected")
41+
42+
if clone_files:
43+
prefix = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
44+
files = _asset_storage_manager.copy_files_list(files, dir=settings.ASSETS_ROOT, dir_prefix=prefix)
45+
# TODO: please note the copy_files_list will make flat any directory structure
46+
47+
asset = LocalAsset(
48+
title=title,
49+
description=description,
50+
type=type,
51+
owner=owner,
52+
created=datetime.datetime.now(),
53+
location=files,
54+
)
55+
asset.save()
56+
return asset
57+
58+
def remove_data(self, asset: LocalAsset):
59+
"""
60+
Removes the files related to an Asset.
61+
Only files within the Assets directory are removed
62+
"""
63+
removed_dir = set()
64+
for file in asset.location:
65+
is_managed = self._is_file_managed(file)
66+
if is_managed:
67+
logger.info(f"Removing asset file {file}")
68+
_asset_storage_manager.delete(file)
69+
removed_dir.add(os.path.dirname(file))
70+
else:
71+
logger.info(f"Not removing asset file outside asset directory {file}")
72+
73+
# TODO: in case of subdirs, make sure that all the tree is removed in the proper order
74+
for dir in removed_dir:
75+
if not os.path.exists(dir):
76+
logger.warning(f"Trying to remove not existing asset directory {dir}")
77+
continue
78+
if not os.listdir(dir):
79+
logger.info(f"Removing empty asset directory {dir}")
80+
os.rmdir(dir)
81+
82+
def replace_data(self, asset: LocalAsset, files: list):
83+
self.remove_data(asset)
84+
asset.location = files
85+
asset.save()
86+
87+
def clone(self, source: LocalAsset) -> LocalAsset:
88+
# get a new asset instance to be edited and stored back
89+
asset = LocalAsset.objects.get(pk=source.pk)
90+
# only copy files if they are managed
91+
if self._are_files_managed(asset.location):
92+
asset.location = _asset_storage_manager.copy_files_list(
93+
asset.location, dir=settings.ASSETS_ROOT, dir_prefix=datetime.datetime.now().strftime("%Y%m%d%H%M%S")
94+
)
95+
# it's a polymorphic object, we need to null both IDs
96+
# https://django-polymorphic.readthedocs.io/en/stable/advanced.html#copying-polymorphic-objects
97+
asset.pk = None
98+
asset.id = None
99+
asset.save()
100+
asset.refresh_from_db()
101+
return asset
102+
103+
def create_download_url(self, asset) -> str:
104+
return build_absolute_uri(reverse("assets-download", args=(asset.pk,)))
105+
106+
def create_link_url(self, asset) -> str:
107+
return build_absolute_uri(reverse("assets-link", args=(asset.pk,)))
108+
109+
def _is_file_managed(self, file) -> bool:
110+
assets_root = os.path.normpath(settings.ASSETS_ROOT)
111+
return file.startswith(assets_root)
112+
113+
def _are_files_managed(self, files: list) -> bool:
114+
"""
115+
:param files: files to be checked
116+
:return: True if all files are managed, False is no file is managed
117+
:raise: ValueError if both managed and unmanaged files are in the list
118+
"""
119+
managed = unmanaged = None
120+
for file in files:
121+
if self._is_file_managed(file):
122+
managed = True
123+
else:
124+
unmanaged = True
125+
if managed and unmanaged:
126+
logger.error(f"Both managed and unmanaged files are present: {files}")
127+
raise ValueError("Both managed and unmanaged files are present")
128+
129+
return bool(managed)
130+
131+
132+
class LocalAssetDownloadHandler(AssetDownloadHandlerInterface):
133+
134+
def create_response(self, asset: LocalAsset, attachment: bool = False, basename=None) -> HttpResponse:
135+
if not asset.location:
136+
return HttpResponse("Asset does not contain any data", status=500)
137+
138+
if len(asset.location) > 1:
139+
logger.warning("TODO: Asset contains more than one file. Download needs to be implemented")
140+
141+
file0 = asset.location[0]
142+
filename = os.path.basename(file0)
143+
orig_base, ext = os.path.splitext(filename)
144+
outname = f"{basename or orig_base}{ext}"
145+
146+
if _asset_storage_manager.exists(file0):
147+
logger.info(f"Returning file {file0} with name {outname}")
148+
149+
return DownloadResponse(
150+
_asset_storage_manager.open(file0).file,
151+
basename=f"{outname}",
152+
attachment=attachment,
153+
)
154+
else:
155+
logger.warning(f"Internal file {file0} not found for asset {asset.id}")
156+
return HttpResponse(f"Internal file not found for asset {asset.id}", status=500)
157+
158+
159+
asset_handler_registry.register(LocalAssetHandler)

‎geonode/assets/migrations/0001_initial.py

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Generated by Django 4.2.9 on 2024-04-24 10:02
2+
3+
from django.conf import settings
4+
from django.db import migrations, models
5+
import django.utils.timezone
6+
7+
8+
class Migration(migrations.Migration):
9+
10+
initial = True
11+
12+
dependencies = [
13+
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
14+
("base", "0091_alter_hierarchicalkeyword_slug"),
15+
]
16+
17+
operations = [
18+
migrations.CreateModel(
19+
name="Asset",
20+
fields=[
21+
("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
22+
("title", models.CharField(max_length=255)),
23+
("description", models.TextField(blank=True, null=True)),
24+
("type", models.CharField(max_length=255)),
25+
("created", models.DateTimeField(auto_now_add=True)),
26+
("owner", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
27+
(
28+
"polymorphic_ctype",
29+
models.ForeignKey(
30+
editable=False,
31+
null=True,
32+
on_delete=django.db.models.deletion.CASCADE,
33+
related_name="polymorphic_%(app_label)s.%(class)s_set+",
34+
to="contenttypes.contenttype",
35+
),
36+
),
37+
],
38+
options={
39+
"verbose_name_plural": "Assets",
40+
},
41+
),
42+
migrations.CreateModel(
43+
name="LocalAsset",
44+
fields=[
45+
(
46+
"asset_ptr",
47+
models.OneToOneField(
48+
auto_created=True,
49+
on_delete=django.db.models.deletion.CASCADE,
50+
parent_link=True,
51+
primary_key=True,
52+
serialize=False,
53+
to="assets.asset",
54+
),
55+
),
56+
("location", models.JSONField(blank=True, default=list)),
57+
],
58+
options={
59+
"verbose_name_plural": "Local assets",
60+
},
61+
bases=("assets.asset",),
62+
),
63+
]

‎geonode/assets/migrations/__init__.py

Whitespace-only changes.

0 commit comments

Comments
 (0)