From 8fbc977b6487217f5ba30d91c2b91e9a65b52ee1 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Tue, 30 Jun 2020 17:09:17 +0200 Subject: [PATCH 1/9] Ensure markdown fields cannot contain HTML or images Since images can be used to drop things like tracking pixels, simply disallow them in all submissions. --- pgweb/util/helpers.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pgweb/util/helpers.py b/pgweb/util/helpers.py index 6b766a89..d2d71dd0 100644 --- a/pgweb/util/helpers.py +++ b/pgweb/util/helpers.py @@ -1,5 +1,6 @@ from django.shortcuts import render, get_object_or_404 from django.core.exceptions import PermissionDenied +from django.core.validators import ValidationError from django.http import HttpResponseRedirect, Http404 from django.template.loader import get_template import django.utils.xmlutils @@ -8,11 +9,31 @@ from pgweb.util.contexts import render_pgweb import io +import re import difflib +import markdown from pgweb.mailqueue.util import send_simple_mail +_re_img = re.compile(']*)>') + + +def MarkdownValidator(val): + if _re_html_open.search(val): + raise ValidationError('Embedding HTML in markdown is not allowed') + + out = markdown.markdown(val) + + # We find images with a regexp, because it works... For now, nothing more advanced + # is needed. + if _re_img.search(out): + raise ValidationError('Image references are not allowed in this field') + + return val + + def simple_form(instancetype, itemid, request, formclass, formtemplate='base/form.html', redirect='/account/', navsection='account', fixedfields=None, createifempty=False): if itemid == 'new': instance = instancetype() @@ -38,6 +59,9 @@ def simple_form(instancetype, itemid, request, formclass, formtemplate='base/for if request.method == 'POST': # Process this form form = formclass(data=request.POST, instance=instance) + for fn in form.fields: + if fn in getattr(instancetype, 'markdown_fields', []): + form.fields[fn].validators.append(MarkdownValidator) # Save away the old value from the instance before it's saved if not is_new: From e4872d9aeeab86ca2bdf17876f1375fdb5e6e2d9 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Sun, 12 Jul 2020 13:46:28 +0200 Subject: [PATCH 2/9] Re-work moderation of submitted items This includes a number of new features: * Move some moderation functionality into shared places, so we don't keep re-inventing the wheel. * Implement three-state moderation, where the submitter can edit their item and then explicitly say "i'm done, please moderate this now". This is currently only implemented for News, but done in a reusable way. * Move moderation workflow to it's own set of URLs instead of overloading it on the general admin interface. Admin interface remains for editing things, but these are now separated out into separate things. * Do proper stylesheet clearing for moderation of markdown fields, using a dynamic sandboxed iframe, so it's not ruined by the /admin/ css. * Move moderation email notification into dedicated moderation code, thereby simplifying the admin subclassing we did which was in some places quite fragile. * Reset date of news postings to the date of their approval, when approved. This avoids some annoying ordering issues. --- media/css/admin_pgweb.css | 55 ++++++ media/css/main.css | 5 + media/js/admin_pgweb.js | 24 ++- pgweb/account/forms.py | 8 + pgweb/account/urls.py | 12 +- pgweb/account/views.py | 179 ++++++++++++++++-- pgweb/core/forms.py | 16 ++ pgweb/core/models.py | 18 +- pgweb/core/views.py | 153 ++++++++++++++- pgweb/downloads/forms.py | 3 - pgweb/downloads/models.py | 16 +- pgweb/downloads/views.py | 10 +- pgweb/events/models.py | 13 +- pgweb/events/views.py | 9 - pgweb/news/admin.py | 13 +- pgweb/news/feeds.py | 5 +- pgweb/news/forms.py | 7 +- .../news/management/commands/twitter_post.py | 3 +- pgweb/news/migrations/0004_modstate.py | 28 +++ pgweb/news/models.py | 24 ++- pgweb/news/struct.py | 4 +- pgweb/news/views.py | 16 +- pgweb/profserv/forms.py | 3 - pgweb/profserv/models.py | 20 +- pgweb/profserv/views.py | 10 - pgweb/urls.py | 1 + pgweb/util/admin.py | 84 -------- pgweb/util/helpers.py | 32 +++- pgweb/util/moderation.py | 141 ++++++++++++-- pgweb/util/signals.py | 19 +- templates/account/index.html | 54 +----- templates/account/objectlist.html | 50 ++++- templates/account/submit_form.html | 22 +++ templates/account/submit_preview.html | 28 +++ templates/admin/change_form_pgweb.html | 42 ++-- .../admin/news/newsarticle/change_form.html | 10 - templates/core/admin_moderation_form.html | 113 +++++++++++ 37 files changed, 929 insertions(+), 321 deletions(-) create mode 100644 pgweb/news/migrations/0004_modstate.py create mode 100644 templates/account/submit_form.html create mode 100644 templates/account/submit_preview.html delete mode 100644 templates/admin/news/newsarticle/change_form.html create mode 100644 templates/core/admin_moderation_form.html diff --git a/media/css/admin_pgweb.css b/media/css/admin_pgweb.css index 3b91f3a6..e50ea200 100644 --- a/media/css/admin_pgweb.css +++ b/media/css/admin_pgweb.css @@ -1,3 +1,58 @@ +a.admbutton { + padding: 10px 15px; +} + +div.modadmfield input, +div.modadmfield select, +div.modadmfield textarea { + width: 500px; +} + +.moderation-form-row div { + display: inline-block; + vertical-align: top; +} + +.moderation-form-row div.txtpreview { + border: 1px solid gray; + padding: 5px; + border-radius: 5px; + white-space: pre; + width: 500px; + overflow-x: auto; + margin-right: 20px; +} + +.moderation-form-row iframe.mdpreview { + border: 1px solid gray; + padding: 5px; + border-radius: 5px; + width: 500px; + overflow-x: auto; +} + +.moderation-form-row div.mdpreview-data { + display: none; +} + +.moderation-form-row div.simplepreview { + max-width: 800px; +} + +.moderror { + color: red !important; +} + +div.modhelp { + display: block; + color: #999; + font-size: 11px; +} + #new_notification { width: 400px; } + +.wspre { + white-space: pre; +} diff --git a/media/css/main.css b/media/css/main.css index 1dc828a9..db18a092 100644 --- a/media/css/main.css +++ b/media/css/main.css @@ -197,6 +197,11 @@ p, ul, ol, dl, table { padding: 1em 2em; } +/* Utility */ +.ws-pre { + white-space: pre; +} + /* #BLOCKQUOTE */ blockquote { diff --git a/media/js/admin_pgweb.js b/media/js/admin_pgweb.js index cf83ee45..c920b5f1 100644 --- a/media/js/admin_pgweb.js +++ b/media/js/admin_pgweb.js @@ -1,8 +1,28 @@ window.onload = function() { - tael = document.getElementsByTagName('textarea'); - for (i = 0; i < tael.length; i++) { + /* Preview in the pure admin views */ + let tael = document.getElementsByTagName('textarea'); + for (let i = 0; i < tael.length; i++) { if (tael[i].className.indexOf('markdown_preview') >= 0) { attach_showdown_preview(tael[i].id, 1); } } + + /* Preview in the moderation view */ + let previews = document.getElementsByClassName('mdpreview'); + for (let i = 0; i < previews.length; i++) { + let iframe = previews[i]; + let textdiv = iframe.previousElementSibling; + let hiddendiv = iframe.nextElementSibling; + + /* Copy the HTML into the iframe */ + iframe.srcdoc = hiddendiv.innerHTML; + + /* Maybe we should apply *some* stylesheet here? */ + + /* Resize the height to to be the same */ + if (textdiv.offsetHeight > iframe.offsetHeight) + iframe.style.height = textdiv.offsetHeight + 'px'; + if (iframe.offsetHeight > textdiv.offsetHeight) + textdiv.style.height = iframe.offsetHeight + 'px'; + } } diff --git a/pgweb/account/forms.py b/pgweb/account/forms.py index a9f322ee..9d2a0f7f 100644 --- a/pgweb/account/forms.py +++ b/pgweb/account/forms.py @@ -171,3 +171,11 @@ def clean_email2(self): class PgwebPasswordResetForm(forms.Form): email = forms.EmailField() + + +class ConfirmSubmitForm(forms.Form): + confirm = forms.BooleanField(required=True, help_text='Confirm') + + def __init__(self, objtype, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['confirm'].help_text = 'Confirm that you are ready to submit this {}.'.format(objtype) diff --git a/pgweb/account/urls.py b/pgweb/account/urls.py index 6cd1463a..703673ed 100644 --- a/pgweb/account/urls.py +++ b/pgweb/account/urls.py @@ -24,20 +24,14 @@ # List of items to edit url(r'^edit/(.*)/$', pgweb.account.views.listobjects), - # News & Events - url(r'^news/(.*)/$', pgweb.news.views.form), - url(r'^events/(.*)/$', pgweb.events.views.form), - - # Software catalogue + # Submitted items + url(r'^(?Pnews)/(?P\d+)/(?Psubmit|withdraw)/$', pgweb.account.views.submitted_item_submitwithdraw), + url(r'^(?Pnews|events|products|organisations|services)/(?P\d+|new)/$', pgweb.account.views.submitted_item_form), url(r'^organisations/(.*)/$', pgweb.core.views.organisationform), - url(r'^products/(.*)/$', pgweb.downloads.views.productform), # Organisation information url(r'^orglist/$', pgweb.account.views.orglist), - # Professional services - url(r'^services/(.*)/$', pgweb.profserv.views.profservform), - # Docs comments url(r'^comments/(new)/([^/]+)/([^/]+)/$', pgweb.docs.views.commentform), url(r'^comments/(new)/([^/]+)/([^/]+)/done/$', pgweb.docs.views.commentform_done), diff --git a/pgweb/account/views.py b/pgweb/account/views.py index 3a11428d..31955a5e 100644 --- a/pgweb/account/views.py +++ b/pgweb/account/views.py @@ -2,6 +2,7 @@ from django.contrib.auth import login as django_login import django.contrib.auth.views as authviews from django.http import HttpResponseRedirect, Http404, HttpResponse +from django.core.exceptions import PermissionDenied from django.shortcuts import get_object_or_404 from pgweb.util.decorators import login_required, script_sources, frame_sources from django.utils.encoding import force_bytes @@ -23,23 +24,28 @@ from pgweb.util.contexts import render_pgweb from pgweb.util.misc import send_template_mail, generate_random_token, get_client_ip -from pgweb.util.helpers import HttpSimpleResponse +from pgweb.util.helpers import HttpSimpleResponse, simple_form +from pgweb.util.moderation import ModerationState from pgweb.news.models import NewsArticle from pgweb.events.models import Event -from pgweb.core.models import Organisation, UserProfile +from pgweb.core.models import Organisation, UserProfile, ModerationNotification from pgweb.contributors.models import Contributor from pgweb.downloads.models import Product from pgweb.profserv.models import ProfessionalService from .models import CommunityAuthSite, CommunityAuthConsent, EmailChangeToken -from .forms import PgwebAuthenticationForm +from .forms import PgwebAuthenticationForm, ConfirmSubmitForm from .forms import CommunityAuthConsentForm from .forms import SignupForm, SignupOauthForm from .forms import UserForm, UserProfileForm, ContributorForm from .forms import ChangeEmailForm, PgwebPasswordResetForm import logging + +from pgweb.util.moderation import get_moderation_model_from_suburl +from pgweb.mailqueue.util import send_simple_mail + log = logging.getLogger(__name__) # The value we store in user.password for oauth logins. This is @@ -47,43 +53,69 @@ OAUTH_PASSWORD_STORE = 'oauth_signin_account_no_password' +def _modobjs(qs): + l = list(qs) + if l: + return { + 'title': l[0]._meta.verbose_name_plural.capitalize(), + 'objects': l, + 'editurl': l[0].account_edit_suburl, + } + else: + return None + + @login_required def home(request): - myarticles = NewsArticle.objects.filter(org__managers=request.user, approved=False) - myevents = Event.objects.filter(org__managers=request.user, approved=False) - myorgs = Organisation.objects.filter(managers=request.user, approved=False) - myproducts = Product.objects.filter(org__managers=request.user, approved=False) - myprofservs = ProfessionalService.objects.filter(org__managers=request.user, approved=False) return render_pgweb(request, 'account', 'account/index.html', { - 'newsarticles': myarticles, - 'events': myevents, - 'organisations': myorgs, - 'products': myproducts, - 'profservs': myprofservs, + 'modobjects': [ + { + 'title': 'not submitted yet', + 'objects': [ + _modobjs(NewsArticle.objects.filter(org__managers=request.user, modstate=ModerationState.CREATED)), + ], + }, + { + 'title': 'waiting for moderator approval', + 'objects': [ + _modobjs(NewsArticle.objects.filter(org__managers=request.user, modstate=ModerationState.PENDING)), + _modobjs(Event.objects.filter(org__managers=request.user, approved=False)), + _modobjs(Organisation.objects.filter(managers=request.user, approved=False)), + _modobjs(Product.objects.filter(org__managers=request.user, approved=False)), + _modobjs(ProfessionalService.objects.filter(org__managers=request.user, approved=False)) + ], + }, + ], }) objtypes = { 'news': { - 'title': 'News Article', + 'title': 'news article', 'objects': lambda u: NewsArticle.objects.filter(org__managers=u), + 'tristate': True, + 'editapproved': False, }, 'events': { - 'title': 'Event', + 'title': 'event', 'objects': lambda u: Event.objects.filter(org__managers=u), + 'editapproved': True, }, 'products': { - 'title': 'Product', + 'title': 'product', 'objects': lambda u: Product.objects.filter(org__managers=u), + 'editapproved': True, }, 'services': { - 'title': 'Professional Service', + 'title': 'professional service', 'objects': lambda u: ProfessionalService.objects.filter(org__managers=u), + 'editapproved': True, }, 'organisations': { - 'title': 'Organisation', + 'title': 'organisation', 'objects': lambda u: Organisation.objects.filter(managers=u), 'submit_header': 'Before submitting a new Organisation, please verify on the list of current organisations if the organisation already exists. If it does, please contact the manager of the organisation to gain permissions.', + 'editapproved': True, }, } @@ -208,14 +240,25 @@ def listobjects(request, objtype): raise Http404("Object type not found") o = objtypes[objtype] - return render_pgweb(request, 'account', 'account/objectlist.html', { - 'objects': { + if o.get('tristate', False): + objects = { + 'approved': o['objects'](request.user).filter(modstate=ModerationState.APPROVED), + 'unapproved': o['objects'](request.user).filter(modstate=ModerationState.PENDING), + 'inprogress': o['objects'](request.user).filter(modstate=ModerationState.CREATED), + } + else: + objects = { 'approved': o['objects'](request.user).filter(approved=True), 'unapproved': o['objects'](request.user).filter(approved=False), - }, + } + + return render_pgweb(request, 'account', 'account/objectlist.html', { + 'objects': objects, 'title': o['title'], + 'editapproved': o['editapproved'], 'submit_header': o.get('submit_header', None), 'suburl': objtype, + 'tristate': o.get('tristate', False), }) @@ -228,6 +271,100 @@ def orglist(request): }) +@login_required +def submitted_item_form(request, objtype, item): + model = get_moderation_model_from_suburl(objtype) + + if item == 'new': + extracontext = {} + else: + extracontext = { + 'notices': ModerationNotification.objects.filter( + objecttype=model.__name__, + objectid=item, + ).order_by('-date') + } + + return simple_form(model, item, request, model.get_formclass(), + redirect='/account/edit/{}/'.format(objtype), + formtemplate='account/submit_form.html', + extracontext=extracontext) + + +def _submitted_item_submit(request, objtype, model, obj): + if obj.modstate != ModerationState.CREATED: + # Can only submit if state is created + return HttpResponseRedirect("/account/edit/{}/".format(objtype)) + + if request.method == 'POST': + form = ConfirmSubmitForm(obj._meta.verbose_name, data=request.POST) + if form.is_valid(): + with transaction.atomic(): + obj.modstate = ModerationState.PENDING + obj.send_notification = False + obj.save() + + send_simple_mail(settings.NOTIFICATION_FROM, + settings.NOTIFICATION_EMAIL, + "{} {} submitted".format(obj._meta.verbose_name.capitalize(), obj.id), + "{} {} with title {} submitted for moderation by {}".format( + obj._meta.verbose_name.capitalize(), + obj.id, + obj.title, + request.user.username + ), + ) + return HttpResponseRedirect("/account/edit/{}/".format(objtype)) + else: + form = ConfirmSubmitForm(obj._meta.verbose_name) + + return render_pgweb(request, 'account', 'account/submit_preview.html', { + 'obj': obj, + 'form': form, + 'objtype': obj._meta.verbose_name, + 'preview': obj.get_preview_fields(), + }) + + +def _submitted_item_withdraw(request, objtype, model, obj): + # XXX: should we do a confirmation step? But it's easy enough to resubmit. + if obj.modstate != ModerationState.PENDING: + # Can only withdraw if it's in pending state + return HttpResponseRedirect("/account/edit/{}/".format(objtype)) + + obj.modstate = ModerationState.CREATED + obj.send_notification = False + obj.save() + + send_simple_mail( + settings.NOTIFICATION_FROM, + settings.NOTIFICATION_EMAIL, + "{} {} withdrawn from moderation".format(model._meta.verbose_name.capitalize(), obj.id), + "{} {} with title {} withdrawn from moderation by {}".format( + model._meta.verbose_name.capitalize(), + obj.id, + obj.title, + request.user.username + ), + ) + return HttpResponseRedirect("/account/edit/{}/".format(objtype)) + + +@login_required +@transaction.atomic +def submitted_item_submitwithdraw(request, objtype, item, what): + model = get_moderation_model_from_suburl(objtype) + + obj = get_object_or_404(model, pk=item) + if not obj.verify_submitter(request.user): + raise PermissionDenied("You are not the owner of this item!") + + if what == 'submit': + return _submitted_item_submit(request, objtype, model, obj) + else: + return _submitted_item_withdraw(request, objtype, model, obj) + + def login(request): return authviews.LoginView.as_view(template_name='account/login.html', authentication_form=PgwebAuthenticationForm, diff --git a/pgweb/core/forms.py b/pgweb/core/forms.py index 65d1026d..334f9791 100644 --- a/pgweb/core/forms.py +++ b/pgweb/core/forms.py @@ -6,6 +6,7 @@ from django.contrib.auth.models import User from pgweb.util.middleware import get_current_user +from pgweb.util.moderation import ModerationState from pgweb.mailqueue.util import send_simple_mail @@ -79,3 +80,18 @@ def clean(self): if self.cleaned_data['merge_into'] == self.cleaned_data['merge_from']: raise ValidationError("The two organisations selected must be different!") return self.cleaned_data + + +class ModerationForm(forms.Form): + modnote = forms.CharField(label='Moderation notice', widget=forms.Textarea, required=False, + help_text="This note will be sent to the creator of the object regardless of if the moderation state has changed.") + oldmodstate = forms.CharField(label='Current moderation state', disabled=True) + modstate = forms.ChoiceField(label='New moderation status', choices=ModerationState.CHOICES + ( + (ModerationState.REJECTED, 'Reject and delete'), + )) + + def __init__(self, *args, **kwargs): + self.twostate = kwargs.pop('twostate') + super().__init__(*args, **kwargs) + if self.twostate: + self.fields['modstate'].choices = [(k, v) for k, v in self.fields['modstate'].choices if int(k) != 1] diff --git a/pgweb/core/models.py b/pgweb/core/models.py index c4f8a4cd..63b70a7a 100644 --- a/pgweb/core/models.py +++ b/pgweb/core/models.py @@ -5,6 +5,8 @@ import base64 +from pgweb.util.moderation import TwostateModerateModel + TESTING_CHOICES = ( (0, 'Release'), (1, 'Release candidate'), @@ -121,9 +123,8 @@ def __str__(self): return self.typename -class Organisation(models.Model): +class Organisation(TwostateModerateModel): name = models.CharField(max_length=100, null=False, blank=False, unique=True) - approved = models.BooleanField(null=False, default=False) address = models.TextField(null=False, blank=True) url = models.URLField(null=False, blank=False) email = models.EmailField(null=False, blank=True) @@ -132,15 +133,24 @@ class Organisation(models.Model): managers = models.ManyToManyField(User, blank=False) lastconfirmed = models.DateTimeField(null=False, blank=False, auto_now_add=True) - send_notification = True - send_m2m_notification = True + account_edit_suburl = 'organisations' + moderation_fields = ['address', 'url', 'email', 'phone', 'orgtype', 'managers'] def __str__(self): return self.name + @property + def title(self): + return self.name + class Meta: ordering = ('name',) + @classmethod + def get_formclass(self): + from pgweb.core.forms import OrganisationForm + return OrganisationForm + # Basic classes for importing external RSS feeds, such as planet class ImportedRSSFeed(models.Model): diff --git a/pgweb/core/views.py b/pgweb/core/views.py index a835cadb..42ae37f1 100644 --- a/pgweb/core/views.py +++ b/pgweb/core/views.py @@ -20,28 +20,29 @@ from pgweb.util.decorators import cache, nocache from pgweb.util.contexts import render_pgweb, get_nav_menu, PGWebContextProcessor from pgweb.util.helpers import simple_form, PgXmlHelper -from pgweb.util.moderation import get_all_pending_moderations +from pgweb.util.moderation import get_all_pending_moderations, get_moderation_model, ModerationState from pgweb.util.misc import get_client_ip, varnish_purge, varnish_purge_expr, varnish_purge_xkey from pgweb.util.sitestruct import get_all_pages_struct +from pgweb.mailqueue.util import send_simple_mail # models needed for the pieces on the frontpage from pgweb.news.models import NewsArticle, NewsTag from pgweb.events.models import Event from pgweb.quotes.models import Quote -from .models import Version, ImportedRSSItem +from .models import Version, ImportedRSSItem, ModerationNotification # models needed for the pieces on the community page from pgweb.survey.models import Survey # models and forms needed for core objects from .models import Organisation -from .forms import OrganisationForm, MergeOrgsForm +from .forms import OrganisationForm, MergeOrgsForm, ModerationForm # Front page view @cache(minutes=10) def home(request): - news = NewsArticle.objects.filter(approved=True)[:5] + news = NewsArticle.objects.filter(modstate=ModerationState.APPROVED)[:5] today = date.today() # get up to seven events to display on the homepage event_base_queryset = Event.objects.select_related('country').filter( @@ -288,6 +289,150 @@ def admin_pending(request): }) +def _send_moderation_message(request, obj, message, notice, what): + if message and notice: + msg = "{}\n\nThe following further information was provided:\n{}".format(message, notice) + elif notice: + msg = notice + else: + msg = message + + n = ModerationNotification( + objectid=obj.id, + objecttype=type(obj).__name__, + text=msg, + author=request.user, + ) + n.save() + + # In the email, add a link back to the item in the bottom + msg += "\n\nYou can view your {} by going to\n{}/account/edit/{}/".format( + obj._meta.verbose_name, + settings.SITE_ROOT, + obj.account_edit_suburl, + ) + + # Send message to org admin + if isinstance(obj, Organisation): + orgemail = obj.email + else: + orgemail = obj.org.email + + send_simple_mail( + settings.NOTIFICATION_FROM, + orgemail, + "Your submitted {} with title {}".format(obj._meta.verbose_name, obj.title), + msg, + suppress_auto_replies=False, + ) + + # Send notification to admins + if what: + admmsg = message + if obj.is_approved: + admmsg += "\n\nNOTE! This {} was previously approved!!".format(obj._meta.verbose_name) + + if notice: + admmsg += "\n\nModeration notice:\n{}".format(notice) + + admmsg += "\n\nEdit at: {}/admin/_moderate/{}/{}/\n".format(settings.SITE_ROOT, obj._meta.model_name, obj.id) + + send_simple_mail(settings.NOTIFICATION_FROM, + settings.NOTIFICATION_EMAIL, + "{} {} by {}".format(obj._meta.verbose_name.capitalize(), what, request.user), + admmsg) + + +# Moderate a single item +@login_required +@user_passes_test(lambda u: u.groups.filter(name='pgweb moderators').exists()) +@transaction.atomic +def admin_moderate(request, objtype, objid): + model = get_moderation_model(objtype) + obj = get_object_or_404(model, pk=objid) + + initdata = { + 'oldmodstate': obj.modstate_string, + 'modstate': obj.modstate, + } + # Else deal with it as a form + if request.method == 'POST': + form = ModerationForm(request.POST, twostate=hasattr(obj, 'approved'), initial=initdata) + if form.is_valid(): + # Ok, do something! + modstate = int(form.cleaned_data['modstate']) + modnote = form.cleaned_data['modnote'] + if modstate == obj.modstate: + # No change in moderation state, but did we want to send a message? + if modnote: + _send_moderation_message(request, obj, None, modnote, None) + messages.info(request, "Moderation message sent, no state changed.") + return HttpResponseRedirect("/admin/pending/") + else: + messages.warning(request, "Moderation state not changed and no moderation note added.") + return HttpResponseRedirect(".") + # Ok, we have a moderation state change! + if modstate == ModerationState.CREATED: + # Returned to editing again (for two-state, this means de-moderated) + _send_moderation_message(request, + obj, + "The {} with title {}\nhas been returned for further editing.\nPlease re-submit when you have adjusted it.".format( + obj._meta.verbose_name, + obj.title + ), + modnote, + "returned") + elif modstate == ModerationState.PENDING: + # Pending moderation should never happen if we actually *change* the value + messages.warning(request, "Cannot change state to 'pending moderation'") + return HttpResponseRedirect(".") + elif modstate == ModerationState.APPROVED: + _send_moderation_message(request, + obj, + "The {} with title {}\nhas been approved and is now published.".format(obj._meta.verbose_name, obj.title), + modnote, + "approved") + elif modstate == ModerationState.REJECTED: + _send_moderation_message(request, + obj, + "The {} with title {}\nhas been rejected and is now deleted.".format(obj._meta.verbose_name, obj.title), + modnote, + "rejected") + messages.info(request, "{} rejected and deleted".format(obj._meta.verbose_name)) + obj.send_notification = False + obj.delete() + return HttpResponseRedirect("/admin/pending") + else: + raise Exception("Can't happen.") + + if hasattr(obj, 'approved'): + # This is a two-state one! + obj.approved = (modstate == ModerationState.APPROVED) + else: + # Three-state moderation + obj.modstate = modstate + + # Suppress notifications as we're sending our own + obj.send_notification = False + obj.save() + messages.info(request, "Moderation state changed to {}".format(obj.modstate_string)) + return HttpResponseRedirect("/admin/pending/") + else: + form = ModerationForm(twostate=hasattr(obj, 'approved'), initial=initdata) + + return render(request, 'core/admin_moderation_form.html', { + 'obj': obj, + 'form': form, + 'app': obj._meta.app_label, + 'model': obj._meta.model_name, + 'itemtype': obj._meta.verbose_name, + 'itemtypeplural': obj._meta.verbose_name_plural, + 'notices': ModerationNotification.objects.filter(objectid=obj.id, objecttype=type(obj).__name__).order_by('date'), + 'previous': hasattr(obj, 'org') and type(obj).objects.filter(org=obj.org).exclude(id=obj.id).order_by('-id')[:10] or None, + 'object_fields': obj.get_moderation_preview_fields(), + }) + + # Purge objects from varnish, for the admin pages @login_required @user_passes_test(lambda u: u.is_staff) diff --git a/pgweb/downloads/forms.py b/pgweb/downloads/forms.py index 4f6ea15c..1f3d7113 100644 --- a/pgweb/downloads/forms.py +++ b/pgweb/downloads/forms.py @@ -5,9 +5,6 @@ class ProductForm(forms.ModelForm): - form_intro = """Note that in order to register a new product, you must first register an organisation. -If you have not done so, use this form.""" - def __init__(self, *args, **kwargs): super(ProductForm, self).__init__(*args, **kwargs) diff --git a/pgweb/downloads/models.py b/pgweb/downloads/models.py index 37176259..1e3999fe 100644 --- a/pgweb/downloads/models.py +++ b/pgweb/downloads/models.py @@ -1,6 +1,7 @@ from django.db import models from pgweb.core.models import Organisation +from pgweb.util.moderation import TwostateModerateModel class Category(models.Model): @@ -24,9 +25,8 @@ class Meta: ordering = ('typename',) -class Product(models.Model): +class Product(TwostateModerateModel): name = models.CharField(max_length=100, null=False, blank=False, unique=True) - approved = models.BooleanField(null=False, default=False) org = models.ForeignKey(Organisation, db_column="publisher_id", null=False, verbose_name="Organisation", on_delete=models.CASCADE) url = models.URLField(null=False, blank=False) category = models.ForeignKey(Category, null=False, on_delete=models.CASCADE) @@ -35,18 +35,28 @@ class Product(models.Model): price = models.CharField(max_length=200, null=False, blank=True) lastconfirmed = models.DateTimeField(null=False, blank=False, auto_now_add=True) - send_notification = True + account_edit_suburl = 'products' markdown_fields = ('description', ) + moderation_fields = ('org', 'url', 'category', 'licencetype', 'description', 'price') def __str__(self): return self.name + @property + def title(self): + return self.name + def verify_submitter(self, user): return (len(self.org.managers.filter(pk=user.pk)) == 1) class Meta: ordering = ('name',) + @classmethod + def get_formclass(self): + from pgweb.downloads.forms import ProductForm + return ProductForm + class StackBuilderApp(models.Model): textid = models.CharField(max_length=100, null=False, blank=False) diff --git a/pgweb/downloads/views.py b/pgweb/downloads/views.py index 3d507658..7370193c 100644 --- a/pgweb/downloads/views.py +++ b/pgweb/downloads/views.py @@ -1,7 +1,6 @@ from django.shortcuts import render, get_object_or_404 from django.http import HttpResponse, Http404, HttpResponseRedirect from django.core.exceptions import PermissionDenied -from pgweb.util.decorators import login_required from django.views.decorators.csrf import csrf_exempt from django.conf import settings @@ -11,12 +10,11 @@ from pgweb.util.decorators import nocache from pgweb.util.contexts import render_pgweb -from pgweb.util.helpers import simple_form, PgXmlHelper, HttpServerError +from pgweb.util.helpers import PgXmlHelper, HttpServerError from pgweb.util.misc import varnish_purge, version_sort from pgweb.core.models import Version from .models import Category, Product, StackBuilderApp -from .forms import ProductForm ####### @@ -224,12 +222,6 @@ def productlist(request, catid, junk=None): }) -@login_required -def productform(request, itemid): - return simple_form(Product, itemid, request, ProductForm, - redirect='/account/edit/products/') - - ####### # Stackbuilder ####### diff --git a/pgweb/events/models.py b/pgweb/events/models.py index 92168c7e..185a6a7c 100644 --- a/pgweb/events/models.py +++ b/pgweb/events/models.py @@ -1,11 +1,10 @@ from django.db import models from pgweb.core.models import Country, Language, Organisation +from pgweb.util.moderation import TwostateModerateModel -class Event(models.Model): - approved = models.BooleanField(null=False, blank=False, default=False) - +class Event(TwostateModerateModel): org = models.ForeignKey(Organisation, null=False, blank=False, verbose_name="Organisation", help_text="If no organisations are listed, please check the organisation list and contact the organisation manager or webmaster@postgresql.org if none are listed.", on_delete=models.CASCADE) title = models.CharField(max_length=100, null=False, blank=False) isonline = models.BooleanField(null=False, default=False, verbose_name="Online event") @@ -22,8 +21,9 @@ class Event(models.Model): summary = models.TextField(blank=False, null=False, help_text="A short introduction (shown on the events listing page)") details = models.TextField(blank=False, null=False, help_text="Complete event description") - send_notification = True + account_edit_suburl = 'events' markdown_fields = ('details', 'summary', ) + moderation_fields = ['org', 'title', 'isonline', 'city', 'state', 'country', 'language', 'badged', 'description_for_badged', 'startdate', 'enddate', 'summary', 'details'] def purge_urls(self): yield '/about/event/%s/' % self.pk @@ -69,3 +69,8 @@ def locationstring(self): class Meta: ordering = ('-startdate', '-enddate', ) + + @classmethod + def get_formclass(self): + from pgweb.events.forms import EventForm + return EventForm diff --git a/pgweb/events/views.py b/pgweb/events/views.py index fe331e90..95fbf257 100644 --- a/pgweb/events/views.py +++ b/pgweb/events/views.py @@ -1,14 +1,11 @@ from django.shortcuts import get_object_or_404 from django.http import Http404 -from pgweb.util.decorators import login_required from datetime import date from pgweb.util.contexts import render_pgweb -from pgweb.util.helpers import simple_form from .models import Event -from .forms import EventForm def main(request): @@ -39,9 +36,3 @@ def item(request, itemid, throwaway=None): return render_pgweb(request, 'about', 'events/item.html', { 'obj': event, }) - - -@login_required -def form(request, itemid): - return simple_form(Event, itemid, request, EventForm, - redirect='/account/edit/events/') diff --git a/pgweb/news/admin.py b/pgweb/news/admin.py index 0e935860..21d9b580 100644 --- a/pgweb/news/admin.py +++ b/pgweb/news/admin.py @@ -5,18 +5,11 @@ class NewsArticleAdmin(PgwebAdmin): - list_display = ('title', 'org', 'date', 'approved', ) - list_filter = ('approved', ) + list_display = ('title', 'org', 'date', 'modstate', ) + list_filter = ('modstate', ) filter_horizontal = ('tags', ) search_fields = ('content', 'title', ) - change_form_template = 'admin/news/newsarticle/change_form.html' - - def change_view(self, request, object_id, extra_context=None): - newsarticle = NewsArticle.objects.get(pk=object_id) - my_context = { - 'latest': NewsArticle.objects.filter(org=newsarticle.org)[:10] - } - return super(NewsArticleAdmin, self).change_view(request, object_id, extra_context=my_context) + exclude = ('modstate', ) class NewsTagAdmin(PgwebAdmin): diff --git a/pgweb/news/feeds.py b/pgweb/news/feeds.py index b6766b49..0904d2f6 100644 --- a/pgweb/news/feeds.py +++ b/pgweb/news/feeds.py @@ -1,5 +1,6 @@ from django.contrib.syndication.views import Feed +from pgweb.util.moderation import ModerationState from .models import NewsArticle from datetime import datetime, time @@ -17,9 +18,9 @@ def get_object(self, request, tagurl=None): def items(self, obj): if obj: - return NewsArticle.objects.filter(approved=True, tags__urlname=obj)[:10] + return NewsArticle.objects.filter(modstate=ModerationState.APPROVED, tags__urlname=obj)[:10] else: - return NewsArticle.objects.filter(approved=True)[:10] + return NewsArticle.objects.filter(modstate=ModerationState.APPROVED)[:10] def item_link(self, obj): return "https://www.postgresql.org/about/news/%s/" % obj.id diff --git a/pgweb/news/forms.py b/pgweb/news/forms.py index 4fbeedbd..4cf92717 100644 --- a/pgweb/news/forms.py +++ b/pgweb/news/forms.py @@ -1,6 +1,7 @@ from django import forms from django.forms import ValidationError +from pgweb.util.moderation import ModerationState from pgweb.core.models import Organisation from .models import NewsArticle, NewsTag @@ -14,9 +15,9 @@ def filter_by_user(self, user): self.fields['org'].queryset = Organisation.objects.filter(managers=user, approved=True) def clean_date(self): - if self.instance.pk and self.instance.approved: + if self.instance.pk and self.instance.modstate != ModerationState.CREATED: if self.cleaned_data['date'] != self.instance.date: - raise ValidationError("You cannot change the date on an article that has been approved") + raise ValidationError("You cannot change the date on an article that has been submitted or approved") return self.cleaned_data['date'] @property @@ -27,7 +28,7 @@ def described_checkboxes(self): class Meta: model = NewsArticle - exclude = ('submitter', 'approved', 'tweeted') + exclude = ('submitter', 'modstate', 'tweeted') widgets = { 'tags': forms.CheckboxSelectMultiple, } diff --git a/pgweb/news/management/commands/twitter_post.py b/pgweb/news/management/commands/twitter_post.py index 655966ae..d3a1ba7d 100644 --- a/pgweb/news/management/commands/twitter_post.py +++ b/pgweb/news/management/commands/twitter_post.py @@ -11,6 +11,7 @@ from datetime import datetime, timedelta import time +from pgweb.util.moderation import ModerationState from pgweb.news.models import NewsArticle import requests_oauthlib @@ -25,7 +26,7 @@ def handle(self, *args, **options): if not curs.fetchall()[0][0]: raise CommandError("Failed to get advisory lock, existing twitter_post process stuck?") - articles = list(NewsArticle.objects.filter(tweeted=False, approved=True, date__gt=datetime.now() - timedelta(days=7)).order_by('date')) + articles = list(NewsArticle.objects.filter(tweeted=False, modstate=ModerationState.APPROVED, date__gt=datetime.now() - timedelta(days=7)).order_by('date')) if not len(articles): return diff --git a/pgweb/news/migrations/0004_modstate.py b/pgweb/news/migrations/0004_modstate.py new file mode 100644 index 00000000..128e6b0a --- /dev/null +++ b/pgweb/news/migrations/0004_modstate.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.27 on 2020-07-02 12:41 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0003_news_tags'), + ] + + operations = [ + migrations.RemoveField( + model_name='newsarticle', + name='approved', + ), + migrations.RunSQL( + "UPDATE news_newsarticle SET modstate=CASE WHEN approved THEN 2 ELSE 0 END", + "UPDATE news_newsarticle SET approved=(modstate = 2)", + ), + migrations.AddField( + model_name='newsarticle', + name='modstate', + field=models.IntegerField(choices=[(0, 'Created (submitter edits)'), (1, 'Pending moderation'), (2, 'Approved and published')], default=0, verbose_name='Moderation state'), + ), + ] diff --git a/pgweb/news/models.py b/pgweb/news/models.py index 296a11d9..06f19ae6 100644 --- a/pgweb/news/models.py +++ b/pgweb/news/models.py @@ -1,6 +1,7 @@ from django.db import models from datetime import date from pgweb.core.models import Organisation +from pgweb.util.moderation import TristateModerateModel, ModerationState class NewsTag(models.Model): @@ -15,18 +16,19 @@ class Meta: ordering = ('urlname', ) -class NewsArticle(models.Model): +class NewsArticle(TristateModerateModel): org = models.ForeignKey(Organisation, null=False, blank=False, verbose_name="Organisation", help_text="If no organisations are listed, please check the organisation list and contact the organisation manager or webmaster@postgresql.org if none are listed.", on_delete=models.CASCADE) - approved = models.BooleanField(null=False, blank=False, default=False) date = models.DateField(null=False, blank=False, default=date.today) title = models.CharField(max_length=200, null=False, blank=False) content = models.TextField(null=False, blank=False) tweeted = models.BooleanField(null=False, blank=False, default=False) tags = models.ManyToManyField(NewsTag, blank=False, help_text="Select the tags appropriate for this post") - send_notification = True - send_m2m_notification = True + account_edit_suburl = 'news' markdown_fields = ('content',) + moderation_fields = ('org', 'date', 'title', 'content', 'taglist') + preview_fields = ('title', 'content', 'taglist') + extramodnotice = "In particular, note that news articles will be sent by email to subscribers, and therefor cannot be recalled in any way once sent." def purge_urls(self): yield '/about/news/%s/' % self.pk @@ -47,9 +49,23 @@ def is_migrated(self): return True return False + @property + def taglist(self): + return ", ".join([t.name for t in self.tags.all()]) + @property def displaydate(self): return self.date.strftime("%Y-%m-%d") class Meta: ordering = ('-date',) + + @classmethod + def get_formclass(self): + from pgweb.news.forms import NewsArticleForm + return NewsArticleForm + + @property + def block_edit(self): + # Don't allow editing of news articles that have been published + return self.modstate in (ModerationState.PENDING, ModerationState.APPROVED) diff --git a/pgweb/news/struct.py b/pgweb/news/struct.py index b67c8d0f..6af63dc3 100644 --- a/pgweb/news/struct.py +++ b/pgweb/news/struct.py @@ -1,6 +1,8 @@ from datetime import date, timedelta from .models import NewsArticle +from pgweb.util.moderation import ModerationState + def get_struct(): now = date.today() @@ -10,7 +12,7 @@ def get_struct(): # since we don't care about getting it indexed. # Also, don't bother indexing anything > 4 years old - for n in NewsArticle.objects.filter(approved=True, date__gt=fouryearsago): + for n in NewsArticle.objects.filter(modstate=ModerationState.APPROVED, date__gt=fouryearsago): yearsold = (now - n.date).days / 365 if yearsold > 4: yearsold = 4 diff --git a/pgweb/news/views.py b/pgweb/news/views.py index 8258ca5d..e5a85c16 100644 --- a/pgweb/news/views.py +++ b/pgweb/news/views.py @@ -1,12 +1,10 @@ from django.shortcuts import get_object_or_404 from django.http import HttpResponse, Http404 -from pgweb.util.decorators import login_required from pgweb.util.contexts import render_pgweb -from pgweb.util.helpers import simple_form +from pgweb.util.moderation import ModerationState from .models import NewsArticle, NewsTag -from .forms import NewsArticleForm import json @@ -14,10 +12,10 @@ def archive(request, tag=None, paging=None): if tag: tag = get_object_or_404(NewsTag, urlname=tag.strip('/')) - news = NewsArticle.objects.filter(approved=True, tags=tag) + news = NewsArticle.objects.filter(modstate=ModerationState.APPROVED, tags=tag) else: tag = None - news = NewsArticle.objects.filter(approved=True) + news = NewsArticle.objects.filter(modstate=ModerationState.APPROVED) return render_pgweb(request, 'about', 'news/newsarchive.html', { 'news': news, 'tag': tag, @@ -27,7 +25,7 @@ def archive(request, tag=None, paging=None): def item(request, itemid, throwaway=None): news = get_object_or_404(NewsArticle, pk=itemid) - if not news.approved: + if news.modstate != ModerationState.APPROVED: raise Http404 return render_pgweb(request, 'about', 'news/item.html', { 'obj': news, @@ -39,9 +37,3 @@ def taglist_json(request): return HttpResponse(json.dumps({ 'tags': [{'name': t.urlname, 'description': t.description} for t in NewsTag.objects.distinct('urlname')], }), content_type='application/json') - - -@login_required -def form(request, itemid): - return simple_form(NewsArticle, itemid, request, NewsArticleForm, - redirect='/account/edit/news/') diff --git a/pgweb/profserv/forms.py b/pgweb/profserv/forms.py index 05d44a58..53b1a44f 100644 --- a/pgweb/profserv/forms.py +++ b/pgweb/profserv/forms.py @@ -5,9 +5,6 @@ class ProfessionalServiceForm(forms.ModelForm): - form_intro = """Note that in order to register a new professional service, you must first register an organisation. -If you have not done so, use this form.""" - def __init__(self, *args, **kwargs): super(ProfessionalServiceForm, self).__init__(*args, **kwargs) diff --git a/pgweb/profserv/models.py b/pgweb/profserv/models.py index ad32ddba..016b4d74 100644 --- a/pgweb/profserv/models.py +++ b/pgweb/profserv/models.py @@ -1,11 +1,10 @@ from django.db import models from pgweb.core.models import Organisation +from pgweb.util.moderation import TwostateModerateModel -class ProfessionalService(models.Model): - approved = models.BooleanField(null=False, blank=False, default=False) - +class ProfessionalService(TwostateModerateModel): org = models.OneToOneField(Organisation, null=False, blank=False, db_column="organisation_id", on_delete=models.CASCADE, verbose_name="organisation", @@ -29,15 +28,26 @@ class ProfessionalService(models.Model): provides_hosting = models.BooleanField(null=False, default=False) interfaces = models.CharField(max_length=512, null=True, blank=True, verbose_name="Interfaces (for hosting)") + account_edit_suburl = 'services' + moderation_fields = ('org', 'description', 'employees', 'locations', 'region_africa', 'region_asia', 'region_europe', + 'region_northamerica', 'region_oceania', 'region_southamerica', 'hours', 'languages', + 'customerexample', 'experience', 'contact', 'url', 'provides_support', 'provides_hosting', 'interfaces') purge_urls = ('/support/professional_', ) - send_notification = True - def verify_submitter(self, user): return (len(self.org.managers.filter(pk=user.pk)) == 1) def __str__(self): return self.org.name + @property + def title(self): + return self.org.name + class Meta: ordering = ('org__name',) + + @classmethod + def get_formclass(self): + from pgweb.profserv.forms import ProfessionalServiceForm + return ProfessionalServiceForm diff --git a/pgweb/profserv/views.py b/pgweb/profserv/views.py index ff768485..509bccc0 100644 --- a/pgweb/profserv/views.py +++ b/pgweb/profserv/views.py @@ -1,11 +1,8 @@ from django.http import Http404 -from pgweb.util.decorators import login_required from pgweb.util.contexts import render_pgweb -from pgweb.util.helpers import simple_form from .models import ProfessionalService -from .forms import ProfessionalServiceForm regions = ( ('africa', 'Africa'), @@ -52,10 +49,3 @@ def region(request, servtype, regionname): 'regionname': regname, 'services': services, }) - - -# Forms to edit -@login_required -def profservform(request, itemid): - return simple_form(ProfessionalService, itemid, request, ProfessionalServiceForm, - redirect='/account/edit/services/') diff --git a/pgweb/urls.py b/pgweb/urls.py index 3d4d0ca6..b97fe3d0 100644 --- a/pgweb/urls.py +++ b/pgweb/urls.py @@ -146,6 +146,7 @@ url(r'^admin/pending/$', pgweb.core.views.admin_pending), url(r'^admin/purge/$', pgweb.core.views.admin_purge), url(r'^admin/mergeorg/$', pgweb.core.views.admin_mergeorg), + url(r'^admin/_moderate/(\w+)/(\d+)/$', pgweb.core.views.admin_moderate), # Uncomment the next line to enable the admin: url(r'^admin/', admin.site.urls), diff --git a/pgweb/util/admin.py b/pgweb/util/admin.py index 9e02eddb..31cbaf8e 100644 --- a/pgweb/util/admin.py +++ b/pgweb/util/admin.py @@ -1,8 +1,4 @@ from django.contrib import admin -from django.conf import settings - -from pgweb.core.models import ModerationNotification -from pgweb.mailqueue.util import send_simple_mail class PgwebAdmin(admin.ModelAdmin): @@ -11,8 +7,6 @@ class PgwebAdmin(admin.ModelAdmin): * Markdown preview for markdown capable textfields (specified by including them in a class variable named markdown_capable that is a tuple of field names) - * Add an admin field for "notification", that can be sent to the submitter - of an item to inform them of moderation issues. """ change_form_template = 'admin/change_form_pgweb.html' @@ -25,15 +19,6 @@ def formfield_for_dbfield(self, db_field, **kwargs): fld.widget.attrs['class'] = fld.widget.attrs['class'] + ' markdown_preview' return fld - def change_view(self, request, object_id, form_url='', extra_context=None): - if hasattr(self.model, 'send_notification') and self.model.send_notification: - # Anything that sends notification supports manual notifications - if extra_context is None: - extra_context = dict() - extra_context['notifications'] = ModerationNotification.objects.filter(objecttype=self.model.__name__, objectid=object_id).order_by('date') - - return super(PgwebAdmin, self).change_view(request, object_id, form_url, extra_context) - # Remove the builtin delete_selected action, so it doesn't # conflict with the custom one. def get_actions(self, request): @@ -53,75 +38,6 @@ def custom_delete_selected(self, request, queryset): custom_delete_selected.short_description = "Delete selected items" actions = ['custom_delete_selected'] - def save_model(self, request, obj, form, change): - if change and hasattr(self.model, 'send_notification') and self.model.send_notification: - # We only do processing if something changed, not when adding - # a new object. - if 'new_notification' in request.POST and request.POST['new_notification']: - # Need to send off a new notification. We'll also store - # it in the database for future reference, of course. - if not obj.org.email: - # Should not happen because we remove the form field. Thus - # a hard exception is ok. - raise Exception("Organisation does not have an email, cannot send notification!") - n = ModerationNotification() - n.objecttype = obj.__class__.__name__ - n.objectid = obj.id - n.text = request.POST['new_notification'] - n.author = request.user.username - n.save() - - # Now send an email too - msgstr = _get_notification_text(obj, - request.POST['new_notification']) - - send_simple_mail(settings.NOTIFICATION_FROM, - obj.org.email, - "postgresql.org moderation notification", - msgstr, - suppress_auto_replies=False) - - # Also generate a mail to the moderators - send_simple_mail( - settings.NOTIFICATION_FROM, - settings.NOTIFICATION_EMAIL, - "Moderation comment on %s %s" % (obj.__class__._meta.verbose_name, obj.id), - _get_moderator_notification_text( - obj, - request.POST['new_notification'], - request.user.username - ) - ) - - # Either no notifications, or done with notifications - super(PgwebAdmin, self).save_model(request, obj, form, change) - def register_pgwebadmin(model): admin.site.register(model, PgwebAdmin) - - -def _get_notification_text(obj, txt): - objtype = obj.__class__._meta.verbose_name - return """You recently submitted a %s to postgresql.org. - -During moderation, this item has received comments that need to be -addressed before it can be approved. The comment given by the moderator is: - -%s - -Please go to https://www.postgresql.org/account/ and make any changes -request, and your submission will be re-moderated. -""" % (objtype, txt) - - -def _get_moderator_notification_text(obj, txt, moderator): - return """Moderator %s made a comment to a pending object: -Object type: %s -Object id: %s -Comment: %s -""" % (moderator, - obj.__class__._meta.verbose_name, - obj.id, - txt, - ) diff --git a/pgweb/util/helpers.py b/pgweb/util/helpers.py index d2d71dd0..f594ea16 100644 --- a/pgweb/util/helpers.py +++ b/pgweb/util/helpers.py @@ -7,6 +7,7 @@ from django.conf import settings from pgweb.util.contexts import render_pgweb +from pgweb.util.moderation import ModerationState import io import re @@ -34,7 +35,7 @@ def MarkdownValidator(val): return val -def simple_form(instancetype, itemid, request, formclass, formtemplate='base/form.html', redirect='/account/', navsection='account', fixedfields=None, createifempty=False): +def simple_form(instancetype, itemid, request, formclass, formtemplate='base/form.html', redirect='/account/', navsection='account', fixedfields=None, createifempty=False, extracontext={}): if itemid == 'new': instance = instancetype() is_new = True @@ -56,6 +57,9 @@ def simple_form(instancetype, itemid, request, formclass, formtemplate='base/for if not instance.verify_submitter(request.user): raise PermissionDenied("You are not the owner of this item!") + if getattr(instance, 'block_edit', False): + raise PermissionDenied("You cannot edit this item") + if request.method == 'POST': # Process this form form = formclass(data=request.POST, instance=instance) @@ -72,13 +76,18 @@ def simple_form(instancetype, itemid, request, formclass, formtemplate='base/for do_notify = getattr(instance, 'send_notification', False) instance.send_notification = False - if not getattr(instance, 'approved', True) and not is_new: - # If the object has an "approved" field and it's set to false, we don't - # bother notifying about the changes. But if it lacks this field, we notify - # about everything, as well as if the field exists and the item has already - # been approved. - # Newly added objects are always notified. - do_notify = False + # If the object has an "approved" field and it's set to false, we don't + # bother notifying about the changes. But if it lacks this field, we notify + # about everything, as well as if the field exists and the item has already + # been approved. + # Newly added objects are always notified. + if not is_new: + if hasattr(instance, 'approved'): + if not getattr(instance, 'approved', True): + do_notify = False + elif hasattr(instance, 'modstate'): + if getattr(instance, 'modstate', None) == ModerationState.CREATED: + do_notify = False notify = io.StringIO() @@ -176,14 +185,17 @@ def simple_form(instancetype, itemid, request, formclass, formtemplate='base/for 'class': 'toggle-checkbox', }) - return render_pgweb(request, navsection, formtemplate, { + ctx = { 'form': form, 'formitemtype': instance._meta.verbose_name, 'form_intro': hasattr(form, 'form_intro') and form.form_intro or None, 'described_checkboxes': getattr(form, 'described_checkboxes', {}), 'savebutton': (itemid == "new") and "Submit New" or "Save", 'operation': (itemid == "new") and "New" or "Edit", - }) + } + ctx.update(extracontext) + + return render_pgweb(request, navsection, formtemplate, ctx) def template_to_string(templatename, attrs={}): diff --git a/pgweb/util/moderation.py b/pgweb/util/moderation.py index 324a1488..4ea492d0 100644 --- a/pgweb/util/moderation.py +++ b/pgweb/util/moderation.py @@ -1,30 +1,137 @@ -# models needed to generate unapproved list -from pgweb.news.models import NewsArticle -from pgweb.events.models import Event -from pgweb.core.models import Organisation -from pgweb.downloads.models import Product -from pgweb.profserv.models import ProfessionalService -from pgweb.quotes.models import Quote +from django.db import models + +import datetime + +import markdown + + +class ModerateModel(models.Model): + def _get_field_data(self, k): + val = getattr(self, k) + yield k + + try: + yield self._meta.get_field(k).verbose_name.capitalize() + except Exception: + yield k.capitalize() + yield val + + if k in getattr(self, 'markdown_fields', []): + yield markdown.markdown(val) + else: + yield None + + if isinstance(val, datetime.date): + yield "Will be reset to today's date when this {} is approved".format(self._meta.verbose_name) + else: + yield None + + def get_preview_fields(self): + if getattr(self, 'preview_fields', []): + return [list(self._get_field_data(k)) for k in self.preview_fields] + return self.get_moderation_preview_fields() + + def get_moderation_preview_fields(self): + return [list(self._get_field_data(k)) for k in self.moderation_fields] + + class Meta: + abstract = True + + @property + def block_edit(self): + return False + + +class ModerationState(object): + CREATED = 0 + PENDING = 1 + APPROVED = 2 + REJECTED = -1 # Never stored, so not available as a choice + + CHOICES = ( + (CREATED, 'Created (submitter edits)'), + (PENDING, 'Pending moderation'), + (APPROVED, 'Approved and published'), + ) + + @classmethod + def get_string(cls, modstate): + return next(filter(lambda x: x[0] == modstate, cls.CHOICES))[1] + + +class TristateModerateModel(ModerateModel): + + modstate = models.IntegerField(null=False, blank=False, default=0, choices=ModerationState.CHOICES, + verbose_name="Moderation state") + + send_notification = True + send_m2m_notification = True + + class Meta: + abstract = True + + @property + def modstate_string(self): + return ModerationState.get_string(self.modstate) + + @property + def is_approved(self): + return self.modstate == ModerationState.APPROVED + + +class TwostateModerateModel(ModerateModel): + approved = models.BooleanField(null=False, blank=False, default=False) + + send_notification = True + send_m2m_notification = True + + class Meta: + abstract = True + + @property + def modstate_string(self): + return self.approved and 'Approved' or 'Created/Pending' + + @property + def modstate(self): + return self.approved and ModerationState.APPROVED or ModerationState.CREATED + + @property + def is_approved(self): + return self.approved # Pending moderation requests (including URLs for the admin interface)) def _get_unapproved_list(objecttype): - objects = objecttype.objects.filter(approved=False) + if hasattr(objecttype, 'approved'): + objects = objecttype.objects.filter(approved=False) + else: + objects = objecttype.objects.filter(modstate=ModerationState.PENDING) if not len(objects): return None return { 'name': objects[0]._meta.verbose_name_plural, - 'entries': [{'url': '/admin/%s/%s/%s/' % (x._meta.app_label, x._meta.model_name, x.pk), 'title': str(x)} for x in objects] + 'entries': [{'url': '/admin/_moderate/%s/%s/' % (x._meta.model_name, x.pk), 'title': str(x)} for x in objects] } +def _modclasses(): + from pgweb.news.models import NewsArticle + from pgweb.events.models import Event + from pgweb.core.models import Organisation + from pgweb.downloads.models import Product + from pgweb.profserv.models import ProfessionalService + return [NewsArticle, Event, Organisation, Product, ProfessionalService] + + def get_all_pending_moderations(): - applist = [ - _get_unapproved_list(NewsArticle), - _get_unapproved_list(Event), - _get_unapproved_list(Organisation), - _get_unapproved_list(Product), - _get_unapproved_list(ProfessionalService), - _get_unapproved_list(Quote), - ] + applist = [_get_unapproved_list(c) for c in _modclasses()] return [x for x in applist if x] + + +def get_moderation_model(modelname): + return next((c for c in _modclasses() if c._meta.model_name == modelname)) + + +def get_moderation_model_from_suburl(suburl): + return next((c for c in _modclasses() if c.account_edit_suburl == suburl)) diff --git a/pgweb/util/signals.py b/pgweb/util/signals.py index 5f076c76..895cb45e 100644 --- a/pgweb/util/signals.py +++ b/pgweb/util/signals.py @@ -6,6 +6,7 @@ from pgweb.util.middleware import get_current_user from pgweb.util.misc import varnish_purge +from pgweb.util.moderation import ModerationState from pgweb.mailqueue.util import send_simple_mail @@ -51,7 +52,7 @@ def _get_all_notification_fields(obj): else: # Include all field names except specified ones, # that are local to this model (not auto created) - return [f.name for f in obj._meta.get_fields() if f.name not in ('approved', 'submitter', 'id', ) and not f.auto_created] + return [f.name for f in obj._meta.get_fields() if f.name not in ('approved', 'modstate', 'submitter', 'id', ) and not f.auto_created] def _get_attr_value(obj, fieldname): @@ -82,21 +83,29 @@ def _get_notification_text(obj): return ('A new {0} has been added'.format(obj._meta.verbose_name), _get_full_text_representation(obj)) - if hasattr(obj, 'approved'): + if hasattr(obj, 'approved') or hasattr(obj, 'modstate'): # This object has the capability to do approving. Apply the following logic: # 1. If object was unapproved, and is still unapproved, don't send notification # 2. If object was unapproved, and is now approved, send "object approved" notification # 3. If object was approved, and is no longer approved, send "object unapproved" notification # 4. (FIXME: configurable?) If object was approved and is still approved, send changes notification - if not obj.approved: - if not oldobj.approved: + + if hasattr(obj, 'approved'): + approved = obj.approved + oldapproved = oldobj.approved + else: + approved = obj.modstate != ModerationState.CREATED + oldapproved = oldobj.modstate != ModerationState.CREATED + + if not approved: + if not oldapproved: # Was approved, still approved -> no notification return (None, None) # From approved to unapproved return ('{0} id {1} has been unapproved'.format(obj._meta.verbose_name, obj.id), _get_full_text_representation(obj)) else: - if not oldobj.approved: + if not oldapproved: # Object went from unapproved to approved return ('{0} id {1} has been approved'.format(obj._meta.verbose_name, obj.id), _get_full_text_representation(obj)) diff --git a/templates/account/index.html b/templates/account/index.html index c7c0e698..dae8262a 100644 --- a/templates/account/index.html +++ b/templates/account/index.html @@ -34,58 +34,18 @@

Migrated data

and let us know which objects to connect together.

-{%if newsarticles or events or organisations or products or profservs %} -

Submissions awaiting moderation

-

-You have submitted the following objects that are still waiting moderator -approval before they are published: -

- -{%if newsarticles%} -

News articles

+{% for cat in modobjects %} +

Items {{cat.title}}

+{%for l in cat.objects %} +{%if l %} +

{{l.title}}

+{%for o in l.objects %}
    -{%for article in newsarticles%} -
  • {{article}}
    • -{%endfor%} +
  • {{o.title}}
-{%endif%} - -{%if events%} -

Events

-
    -{%for event in events%} -
  • {{event}}
    • {%endfor%} -
{%endif%} - -{%if organisations%} -

Organisations

-
    -{%for org in organisations%} -
  • {{org}}
    • {%endfor%} -
-{%endif%} - -{%if products%} -

Products

-
    -{%for product in products%} -
  • {{product}}
    • {%endfor%} -
-{%endif%} - -{%if profservs%} -

Professional Services

- -{%endif%} - -{%endif%} {%endblock%} diff --git a/templates/account/objectlist.html b/templates/account/objectlist.html index fa617ce3..fcd9e9ac 100644 --- a/templates/account/objectlist.html +++ b/templates/account/objectlist.html @@ -1,27 +1,67 @@ {%extends "base/page.html"%} {%block title%}Your account{%endblock%} {%block contents%} -

{{title}}s

+

{{title|title}}s

-Objects that are awaiting moderator approval are listed in their own category. -Note that modifying anything that was previously approved might result in -additional moderation based upon what has changed in the content. + The following {{title}}s are associated with an organisation you are a manager for.

+{%if objects.inprogress %} +

Not submitted

+

+ You can edit these {{title}}s an unlimited number of times, but they will not + be visible to anybody. +

+{%endif%} + {% if objects.unapproved %} -

Awaiting Moderation

+

Waiting for moderator approval

+

+ These {{title}}s are pending moderator approval. As soon as a moderator has reviewed them, + they will be published. +{%if not tristate%} + You can make further changes to them while you wait for moderator approval. +{%else%} + If you withdraw a submission, it will return to Not submitted status and you can make + further changes. +{%endif%} +

    {%for o in objects.unapproved %} +{%if tristate%} +
  • {{o}} (Withdraw)
    • +{%else%}{# one-step approval allows editing in unapproved state #}
  • {{o}}
    • +{%endif%} {%endfor%}
{% endif %} {% if objects.approved %}

Approved

+{%if not editapproved%} +

+ These {{title}}s are approved and published, and can no longer be edited. If you need to make + any changes to these objects, please contact + webmaster@postgresql.org. +

+{%else%} +

+ These objects are approved and published, but you can still edit them. Any changes you make + will notify moderators, who may decide to reject the object based on the changes. +

+{%endif%}
    {%for o in objects.approved %} +{%if editapproved%}
  • {{o}}
    • +{%else%} +
  • {{o}}
    • +{%endif%} {%endfor%}
{% endif %} diff --git a/templates/account/submit_form.html b/templates/account/submit_form.html new file mode 100644 index 00000000..6cda8512 --- /dev/null +++ b/templates/account/submit_form.html @@ -0,0 +1,22 @@ +{%extends "base/form.html"%} +{%block post_form%} +{%if notices%} +

Moderation notices

+

+ This {{formitemtype}} has previously received the following moderation notices: +

+ + + + + +{%for n in notices%} + + + + +{%endfor%} +
DateNote
{{n.date}}{{n.text}}
+{%endif%} + +{%endblock%} diff --git a/templates/account/submit_preview.html b/templates/account/submit_preview.html new file mode 100644 index 00000000..eed65e4c --- /dev/null +++ b/templates/account/submit_preview.html @@ -0,0 +1,28 @@ +{%extends "base/form.html"%} +{%load markup%} +{%block title%}Confirm {{objtype}} submission{%endblock%} +{%block contents%} +

Confirm {{objtype}} submission

+ +

+ You are about to submit the following {{objtype}} for moderation. Note that once submitted, + the contents can no longer be changed. +

+{%if obj.extramodnotice %} +

+ {{obj.extramodnotice}} +

+{%endif%} + +

Your {{objtype}}

+{%for fld, title, contents, mdcontents, note in preview %} +
+
{{title}}
+
{%if mdcontents%}{{mdcontents|safe}}{%else%}{{contents}}{%endif%}
+
+{%endfor%} + +

Confirm

+{%include "base/form_contents.html" with savebutton="Submit for moderation"%} + +{%endblock%} diff --git a/templates/admin/change_form_pgweb.html b/templates/admin/change_form_pgweb.html index c891f7b4..456b275c 100644 --- a/templates/admin/change_form_pgweb.html +++ b/templates/admin/change_form_pgweb.html @@ -1,10 +1,4 @@ {% extends "admin/change_form.html" %} -{% block form_top %} -

-Note that the summary field can use -markdown markup. -

-{%endblock%} {% block extrahead %} {{ block.super }} @@ -14,25 +8,25 @@ {%endblock%} -{%if notifications%} -{%block after_field_sets%} -

Notifications sent for this item

-
    - {%for n in notifications%} -
  • {{n.text}} by {{n.author}} sent at {{n.date}}
    • - {%empty%} -
  • No notifications sent for this item
    • - {%endfor%} -
-

-{%if original.org.email%} -New notification: (Note! This comment is emailed to the organisation!)
-To send a notification on rejection, first add the notification above and hit -"Save and continue editing". Then as a separate step, delete the record. -{%else%} -Organisation has no email, so cannot send notifications to it! +{%block form_top%} +{%if original.is_approved%} +

+

This {{opts.verbose_name}} has already been approved! Be very careful with editing!

+
{%endif%} +

+ Moderate this {{opts.verbose_name}}

-
{%endblock%} + +{% block after_field_sets %} +

+ Moderate this {{opts.verbose_name}} +

+{%if original.is_approved%} + +
+

This {{opts.verbose_name}} has already been approved! Be very careful with editing!

+
{%endif%} +{% endblock %} diff --git a/templates/admin/news/newsarticle/change_form.html b/templates/admin/news/newsarticle/change_form.html deleted file mode 100644 index 69565566..00000000 --- a/templates/admin/news/newsarticle/change_form.html +++ /dev/null @@ -1,10 +0,0 @@ -{% extends "admin/change_form_pgweb.html" %} -{% block after_field_sets %} -{{block.super}} -

Previous 10 posts by this organization:

-
    -{%for p in latest %} -
  • {{p.date}}: {{p.title}}
    • -{%endfor%} -
-{% endblock %} diff --git a/templates/core/admin_moderation_form.html b/templates/core/admin_moderation_form.html new file mode 100644 index 00000000..79df11b8 --- /dev/null +++ b/templates/core/admin_moderation_form.html @@ -0,0 +1,113 @@ +{%extends "admin/base_site.html"%} +{%load pgfilters%} + +{%block breadcrumbs%} +
Home › Pending moderation
+{%endblock%} + +{% block extrahead %} +{{ block.super }} + + +{% endblock %} + +{% block coltype %}colM{% endblock %} + + +{%block content%} +

Pending moderation

+ +
+
{%csrf_token%} +
+{% if errors %} +

+ {% if errors|length == 1 %}Please correct the error below.{% else %}Please correct the errors below{% endif %} +

+ {{ form.non_field_errors }} +{% endif %} +
+ +{%if obj.is_approved%} +
+

This {{itemtype}} has already been approved!

+
+{%endif%} +
+

{{itemtype|capfirst}}

+{%for fld, title, contents, mdcontents, note in object_fields %} +
+
{{title}}
+{%if mdcontents%} +
{{contents}}
+ +
{{mdcontents|safe}}
+{%else%} +
{{contents}} +{%if note%}
{{note}}
{%endif%} +
+{%endif%} + +
+{%endfor%} +{%if user.is_staff %} + Edit {{itemtype}} in admin view +{%endif%} +
+ +{%if previous %} +
+

Previous {{itemtypeplural}}

+

These are the latest {{itemtypeplural}} from this organisation:

+ + {%for p in previous %} + + + + + + {%endfor%} +
{{p.date}}{{p.modstate_string}}{{p.title}}
+
+{%endif%} + +{%if notices%} +
+

Moderation notices

+

These moderation notices have previously been sent for this item:

+ + {%for n in notices %} + + + + + + {%endfor%} +
{{n.date}}{{n.author}}{{n.text}}
+
+{%endif%} + +
+

Moderation

+{%if obj.is_approved%} +

This {{itemtype}} has already been approved!

+

+Be careful if you unapprove it! +

+{%endif%} +{% for field in form %} +
+
+ {{ field.label_tag }} + {{ field }} + {%if field.field.help_text %} +
{{ field.help_text|safe }}
+ {%endif%} +
+
+{% endfor %} +
+ +
+
+{%endblock%} From 7b966900ec318e453971e3e931517807347728dc Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Sat, 4 Jul 2020 15:48:37 +0200 Subject: [PATCH 3/9] Remove references to website migration This migration happened 10 years ago, so if someone hasn't updated their records by now, they're not going to. We still allow and special-case the migrated data in the database in order not to delete history, but this removes the direct texts about it which take up unnecessary space and confuse some people. --- templates/account/index.html | 10 ---------- templates/account/orglist.html | 9 +-------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/templates/account/index.html b/templates/account/index.html index dae8262a..94761e00 100644 --- a/templates/account/index.html +++ b/templates/account/index.html @@ -24,16 +24,6 @@

Permissions model

and managers here.

-

Migrated data

-

-For most of the data migrated from the old website has unfortunately not -been connected to the proper organisations and accounts. If you have any -data submitted that is not properly connected to you (most likely this -will be your account not being connected to the proper organisation(s)), -please contact webmaster@postgresql.org -and let us know which objects to connect together. -

- {% for cat in modobjects %}

Items {{cat.title}}

{%for l in cat.objects %} diff --git a/templates/account/orglist.html b/templates/account/orglist.html index b7ea3670..c7302455 100644 --- a/templates/account/orglist.html +++ b/templates/account/orglist.html @@ -3,14 +3,7 @@ {%block contents%}

Organisations

-The following organisations are registered in our database. Note that any -organisations listed as Migrated Connections are organisations that -have been migrated from our old website and not been given a proper -manager in the new system. If you are the manager of one of these -organisations, please send an email to -webmaster@postgresql.org -letting us know this, and including the name of your community account. -We will then link your account to this organisation. +The following organisations are registered in our database.

From 60f75e498ecf22e439619d894260f0e39f632a5c Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Sat, 4 Jul 2020 17:58:28 +0200 Subject: [PATCH 4/9] Implement permissions on news tags This makes it possible to limit which organisations can use specific tags in news, and verify those as news is submitted. Administrators can, as always, override. In passing also add a sortkey field to newstags to make them, well, sortable. --- pgweb/news/admin.py | 1 + pgweb/news/forms.py | 15 ++++++++++ pgweb/news/migrations/0005_tag_permissions.py | 30 +++++++++++++++++++ pgweb/news/models.py | 5 +++- 4 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 pgweb/news/migrations/0005_tag_permissions.py diff --git a/pgweb/news/admin.py b/pgweb/news/admin.py index 21d9b580..b32d296c 100644 --- a/pgweb/news/admin.py +++ b/pgweb/news/admin.py @@ -14,6 +14,7 @@ class NewsArticleAdmin(PgwebAdmin): class NewsTagAdmin(PgwebAdmin): list_display = ('urlname', 'name', 'description') + filter_horizontal = ('allowed_orgs', ) admin.site.register(NewsArticle, NewsArticleAdmin) diff --git a/pgweb/news/forms.py b/pgweb/news/forms.py index 4cf92717..a8af1582 100644 --- a/pgweb/news/forms.py +++ b/pgweb/news/forms.py @@ -26,6 +26,21 @@ def described_checkboxes(self): 'tags': {t.id: t.description for t in NewsTag.objects.all()} } + def clean(self): + data = super().clean() + + for t in data['tags']: + # Check each tag for permissions. This is not very db-efficient, but people + # don't save news articles that often... + if t.allowed_orgs.exists() and not t.allowed_orgs.filter(pk=data['org'].pk).exists(): + self.add_error('tags', + 'The organisation {} is not allowed to use the tag {}.'.format( + data['org'], + t, + )) + + return data + class Meta: model = NewsArticle exclude = ('submitter', 'modstate', 'tweeted') diff --git a/pgweb/news/migrations/0005_tag_permissions.py b/pgweb/news/migrations/0005_tag_permissions.py new file mode 100644 index 00000000..7e5a9a9c --- /dev/null +++ b/pgweb/news/migrations/0005_tag_permissions.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.27 on 2020-07-04 15:47 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0002_block_oauth'), + ('news', '0004_modstate'), + ] + + operations = [ + migrations.AddField( + model_name='newstag', + name='allowed_orgs', + field=models.ManyToManyField(blank=True, help_text='Organisations allowed to use this tag', to='core.Organisation'), + ), + migrations.AddField( + model_name='newstag', + name='sortkey', + field=models.IntegerField(default=100), + ), + migrations.AlterModelOptions( + name='newstag', + options={'ordering': ('sortkey', 'urlname', )}, + ) + ] diff --git a/pgweb/news/models.py b/pgweb/news/models.py index 06f19ae6..f7978697 100644 --- a/pgweb/news/models.py +++ b/pgweb/news/models.py @@ -8,12 +8,15 @@ class NewsTag(models.Model): urlname = models.CharField(max_length=20, null=False, blank=False, unique=True) name = models.CharField(max_length=32, null=False, blank=False) description = models.CharField(max_length=200, null=False, blank=False) + allowed_orgs = models.ManyToManyField(Organisation, blank=True, + help_text="Organisations allowed to use this tag") + sortkey = models.IntegerField(null=False, blank=False, default=100) def __str__(self): return self.name class Meta: - ordering = ('urlname', ) + ordering = ('sortkey', 'urlname', ) class NewsArticle(TristateModerateModel): From 74d9a03f3fce505de50602422ff42c7856dd47d3 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Tue, 7 Jul 2020 18:38:35 +0200 Subject: [PATCH 5/9] Simplify admin preview of emails Use the python3 function to get the plaintext body of the email, instead of our own very limited one we had before. --- pgweb/mailqueue/admin.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/pgweb/mailqueue/admin.py b/pgweb/mailqueue/admin.py index 468283b6..77d07ee1 100644 --- a/pgweb/mailqueue/admin.py +++ b/pgweb/mailqueue/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin from email.parser import Parser +from email import policy from .models import QueuedMail @@ -13,18 +14,9 @@ def parsed_content(self, obj): # We only try to parse the *first* piece, because we assume # all our emails are trivial. try: - parser = Parser() + parser = Parser(policy=policy.default) msg = parser.parsestr(obj.fullmsg) - b = msg.get_payload(decode=True) - if b: - return b.decode('utf8') - - pl = msg.get_payload() - for p in pl: - b = p.get_payload(decode=True) - if b: - return b.decode('utf8') - return "Could not find body" + return msg.get_body(preferencelist=('plain', )).get_payload(decode=True).decode('utf8') except Exception as e: return "Failed to get body: %s" % e From 5ea4c3732bdeb70cb7244bd097447f144569ca1b Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Fri, 10 Jul 2020 17:14:03 +0200 Subject: [PATCH 6/9] Add templatefilter joinandor This filter takes a list of a,b,c,d and turns it into "a, b, c and d" or "a, b, c or d" depending on parameter given. --- pgweb/core/templatetags/pgfilters.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pgweb/core/templatetags/pgfilters.py b/pgweb/core/templatetags/pgfilters.py index 3ff4c4ef..03a89d3e 100644 --- a/pgweb/core/templatetags/pgfilters.py +++ b/pgweb/core/templatetags/pgfilters.py @@ -80,3 +80,16 @@ def release_notes_pg_minor_version(minor_version, major_version): if str(major_version) in ['0', '1']: return str(minor_version)[2:4] return minor_version + + +@register.filter() +def joinandor(value, andor): + # Value is a list of objects. Join them on comma, add "and" or "or" before the last. + if len(value) == 1: + return str(value) + + if not isinstance(value, list): + # Must have a list to index from the end + value = list(value) + + return ", ".join([str(x) for x in value[:-1]]) + ' ' + andor + ' ' + str(value[-1]) From 4e71d1ac96c5aad97f39f8569a9399ffb778a422 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Fri, 10 Jul 2020 17:14:51 +0200 Subject: [PATCH 7/9] Store project root directory in settings This makes it possible to reference for example templates in relation to the root directory. --- pgweb/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pgweb/settings.py b/pgweb/settings.py index f9782dc0..b37709a2 100644 --- a/pgweb/settings.py +++ b/pgweb/settings.py @@ -1,5 +1,8 @@ # Django settings for pgweb project. +import os +PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) + DEBUG = False ADMINS = ( From afe3676b5b28966c37e8316aac6e4d1c790538e6 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Fri, 10 Jul 2020 17:16:07 +0200 Subject: [PATCH 8/9] Teach send_simple_mail about sending HTML email If a HTML body is specified, the plaintext and html bodies will be sent as a multipart/alternative MIME object. Also teach it to add attachments with Content-ID and overriding the Content-Disposition, to make it possible to reference images attached using cid: type URLs. --- pgweb/mailqueue/util.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/pgweb/mailqueue/util.py b/pgweb/mailqueue/util.py index e2f3f7c6..6828cabb 100644 --- a/pgweb/mailqueue/util.py +++ b/pgweb/mailqueue/util.py @@ -3,7 +3,7 @@ from email.mime.nonmultipart import MIMENonMultipart from email.utils import formatdate, formataddr from email.utils import make_msgid -from email import encoders +from email import encoders, charset from email.header import Header from .models import QueuedMail @@ -15,7 +15,14 @@ def _encoded_email_header(name, email): return email -def send_simple_mail(sender, receiver, subject, msgtxt, attachments=None, usergenerated=False, cc=None, replyto=None, sendername=None, receivername=None, messageid=None, suppress_auto_replies=True, is_auto_reply=False): +# Default for utf-8 in python is to encode subject with "shortest" and body with "base64". For our texts, +# make it always quoted printable, for easier reading and testing. +_utf8_charset = charset.Charset('utf-8') +_utf8_charset.header_encoding = charset.QP +_utf8_charset.body_encoding = charset.QP + + +def send_simple_mail(sender, receiver, subject, msgtxt, attachments=None, usergenerated=False, cc=None, replyto=None, sendername=None, receivername=None, messageid=None, suppress_auto_replies=True, is_auto_reply=False, htmlbody=None, headers={}): # attachment format, each is a tuple of (name, mimetype,contents) # content should be *binary* and not base64 encoded, since we need to # use the base64 routines from the email library to get a properly @@ -44,14 +51,27 @@ def send_simple_mail(sender, receiver, subject, msgtxt, attachments=None, userge elif not usergenerated: msg['Auto-Submitted'] = 'auto-generated' - msg.attach(MIMEText(msgtxt, _charset='utf-8')) + for h in headers.keys(): + msg[h] = headers[h] + + if htmlbody: + mpart = MIMEMultipart("alternative") + mpart.attach(MIMEText(msgtxt, _charset=_utf8_charset)) + mpart.attach(MIMEText(htmlbody, 'html', _charset=_utf8_charset)) + msg.attach(mpart) + else: + # Just a plaintext body, so append it directly + msg.attach(MIMEText(msgtxt, _charset='utf-8')) if attachments: - for filename, contenttype, content in attachments: - main, sub = contenttype.split('/') + for a in attachments: + main, sub = a['contenttype'].split('/') part = MIMENonMultipart(main, sub) - part.set_payload(content) - part.add_header('Content-Disposition', 'attachment; filename="%s"' % filename) + part.set_payload(a['content']) + part.add_header('Content-Disposition', a.get('disposition', 'attachment; filename="%s"' % a['filename'])) + if 'id' in a: + part.add_header('Content-ID', a['id']) + encoders.encode_base64(part) msg.attach(part) From ba9138f36be9313a1e33db7132c33e2911e3495e Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Fri, 10 Jul 2020 17:20:42 +0200 Subject: [PATCH 9/9] Add support for sending out news as HTML email When a news article is approved, it gets delivered as an email to the pgsql-announce mailinglist. It will render the markdown of the news article into a HTML part of the email, and include the markdown raw as the text part (for those unable or unwilling to read html mail). For each organisation, a mail template can be specified. Initially only two templates are supported, one "default" and one "pgproject" which is for official project news. The intention is *not* to provide generic templates, but we may want to extend this to certain related projects in the future *maybe* (such as regional NPOs). These templates are stored in templates/news/mail/*.html, and for each template *all* images found in templates/news/mail/img.