Source code for

# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals

import os
import pickle
import sys
from time import sleep

import boto3
from aspen import log_dammit
from aspen.simplates.pagination import parse_specline, split_and_escape
from aspen_jinja2_renderer import SimplateLoader
from jinja2 import Environment
from markupsafe import escape as htmlescape

from gratipay.exceptions import NoEmailAddress, Throttled
from gratipay.models.participant import Participant
from gratipay.utils import find_files, i18n

[docs]class Queue(object): """Model an outbound email queue. """ def __init__(self, env, db, tell_sentry, root): if self._have_ses(env): log_dammit("AWS SES is configured! We'll send mail through SES.") self._mailer = boto3.client( service_name='ses' , region_name=env.aws_ses_default_region , aws_access_key_id=env.aws_ses_access_key_id , aws_secret_access_key=env.aws_ses_secret_access_key ) else: log_dammit("AWS SES is not configured! Mail will be dumped to the console here.") self._mailer = ConsoleMailer() self.db = db self.tell_sentry = tell_sentry self.sleep_for = env.email_queue_sleep_for self.allow_up_to = env.email_queue_allow_up_to self.log_every = env.email_queue_log_metrics_every templates = {} templates_dir = os.path.join(root, 'emails') assert os.path.isdir(templates_dir) i = len(templates_dir) + 1 for spt in find_files(templates_dir, '*.spt'): base_name = spt[i:-4] templates[base_name] = compile_email_spt(spt) self._email_templates = templates def _have_ses(self, env): return env.aws_ses_access_key_id \ and env.aws_ses_secret_access_key \ and env.aws_ses_default_region
[docs] def put(self, to, template, _user_initiated=True, email=None, **context): """Put an email message on the queue. :param Participant to: the participant to send the email message to. In cases where an email is not linked to a participant, this can be ``None``. :param unicode template: the name of the template to use when rendering the email, corresponding to a filename in ``emails/`` without the file extension :param unicode email: The email address to send this message to. If not provided, the ``to`` participant's primary email is used. :param bool _user_initiated: user-initiated emails are throttled; system-initiated messages don't count against throttling :param dict context: the values to use when rendering the template :raise Throttled: if the participant already has a few messages in the queue (that they put there); the specific number is tunable with the ``EMAIL_QUEUE_ALLOW_UP_TO`` envvar. :returns: ``None`` """ assert to or email # Either participant or email address required. with self.db.get_cursor() as cursor: participant_id = if to else None""" INSERT INTO email_messages (participant, email_address, spt_name, context, user_initiated) VALUES (%s, %s, %s, %s, %s) """, (participant_id, email, template, pickle.dumps(context), _user_initiated)) if _user_initiated: nqueued = self._get_nqueued(cursor, to, email) if nqueued > self.allow_up_to: raise Throttled()
def _get_nqueued(self, cursor, participant, email_address): """Returns the number of messages already queued for the given participant or email address. Prefers participant if provided, falls back to email_address otherwise. :param Participant participant: The participant to check queued messages for. :param unicode email_address: The email address to check queued messages for. :returns number of queued messages """ if participant: return""" SELECT COUNT(*) FROM email_messages WHERE user_initiated AND participant=%s AND result is null """, (, )) else: return""" SELECT COUNT(*) FROM email_messages WHERE user_initiated AND email_address=%s AND result is null """, (email_address, ))
[docs] def flush(self): """Load messages queued for sending, and send them. """ fetch_messages = lambda: self.db.all(""" SELECT * FROM email_messages WHERE result is null ORDER BY ctime ASC LIMIT 60 """) nsent = 0 while True: messages = fetch_messages() if not messages: break for rec in messages: try: message = self._prepare_email_message_for_ses(rec) result = self._mailer.send_email(**message) remote_message_id = result['MessageId'] # let KeyErrors go to Sentry except Exception as exc: self._store_result(, repr(exc), None) raise # we want to see this in Sentry self._store_result(, '', remote_message_id) nsent += 1 sleep(self.sleep_for) return nsent
def _store_result(self, message_id, result, remote_message_id):"UPDATE email_messages SET result=%s, remote_message_id=%s " "WHERE id=%s", (result, remote_message_id, message_id)) def _prepare_email_message_for_ses(self, rec): """Prepare an email message for delivery via Amazon SES. :param Record rec: a database record from the ``email_messages`` table :returns: ``dict`` if we can find an email address to send to :raises: ``NoEmailAddress`` if we can't find an email address to send to We look for an email address to send to in two places: #. the context stored in ``rec.context``, and then #. ``participant.email_address``. """ participant = Participant.from_id(rec.participant) spt = self._email_templates[rec.spt_name] context = pickle.loads(rec.context) email = rec.email_address or participant.email_address # Previously, email_address was stored in the 'email' key on `context` # and not in the `email_address` field. Let's handle that case so that # old emails don't suffer # # TODO: Remove this once we're sure old emails have gone out. email = context.get('email', email) if not email: raise NoEmailAddress() context['email'] = email if participant: context['participant'] = participant context['username'] = participant.username context['button_style'] = ( "color: #fff; text-decoration:none; display:inline-block; " "padding: 0 15px; background: #396; white-space: nowrap; " "font: normal 14px/40px Arial, sans-serif; border-radius: 3px" ) context.setdefault('include_unsubscribe', True) accept_lang = (participant and participant.email_lang) or 'en' langs = i18n.parse_accept_lang(accept_lang) locale = i18n.match_lang(langs) i18n.add_helpers_to_context(self.tell_sentry, context, locale) context['escape'] = lambda s: s context_html = dict(context) i18n.add_helpers_to_context(self.tell_sentry, context_html, locale) context_html['escape'] = htmlescape base_spt = self._email_templates['base'] def render(t, context): b = base_spt[t].render(context).strip() return b.replace('$body', spt[t].render(context).strip()) message = {} message['Source'] = 'Gratipay Support <>' message['Destination'] = {} if participant: # "username <>" destination = "%s <%s>" % (participant.username, email) else: destination = email message['Destination']['ToAddresses'] = [destination] message['Message'] = {} message['Message']['Subject'] = {} message['Message']['Subject']['Data'] = spt['subject'].render(context).strip() message['Message']['Body'] = { 'Text': { 'Data': render('text/plain', context) }, 'Html': { 'Data': render('text/html', context_html) } } return message def log_metrics(self, _print=print): stats =""" SELECT count(CASE WHEN result = '' THEN 1 END) AS sent , count(CASE WHEN result > '' THEN 1 END) AS failed , count(CASE WHEN result is null THEN 1 END) AS pending FROM email_messages WHERE ctime > now() - %s::interval """, ('{} seconds'.format(self.log_every),), back_as=dict) prefix = 'count#email_queue' variables = ('sent', 'failed', 'pending') _print(' '.join('{}_{}={}'.format(prefix, v, stats[v]) for v in variables))
jinja_env = Environment() jinja_env_html = Environment(autoescape=True, extensions=['jinja2.ext.autoescape'])
[docs]def compile_email_spt(fpath): """Compile an email template from a simplate. :param unicode fpath: the filesystem path of the simplate """ r = {} with open(fpath) as f: pages = list(split_and_escape( for i, page in enumerate(pages, 1): tmpl = b'\n' * page.offset + page.content content_type, renderer = parse_specline(page.header) key = 'subject' if i == 1 else content_type env = jinja_env_html if content_type == 'text/html' else jinja_env r[key] = SimplateLoader(fpath, tmpl).load(env, fpath) return r
[docs]class ConsoleMailer(object): """Dumps mail to stdout. """ def __init__(self, fp=sys.stdout): self.fp = fp def send_email(self, **email): p = lambda *a, **kw: print(*a, file=self.fp) p('-'*78, ) for i, address in enumerate(email['Destination']['ToAddresses']): if not i: p('To: ', address) else: p(' ', address) p('Subject: ', email['Message']['Subject']['Data']) p('Body:') p() for line in email['Message']['Body']['Text']['Data'].splitlines(): p(' ', line) p() p('-'*78) return {'MessageId': 'deadbeef'} # simulate a remote message id