source: asadb/groups/diffs.py

#!/usr/bin/python
import collections
import datetime
import os
import sys

if __name__ == '__main__':
    cur_file = os.path.abspath(__file__)
    django_dir = os.path.abspath(os.path.join(os.path.dirname(cur_file), '..'))
    sys.path.append(django_dir)
    os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'

from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.core.mail import EmailMessage, mail_admins
from django.db import connection
from django.db.models import Q
from django.template import Context, Template
from django.template.loader import get_template

import reversion.models

import groups.models
import util.emails
import util.mailinglist

if settings.PRODUCTION_DEPLOYMENT:
    asa_all_groups_list = util.mailinglist.MailmanList('asa-official')
    gsc_fb_list = util.mailinglist.MailmanList('gsc-fb-')
    finboard_groups_list = util.mailinglist.MoiraList('finboard-groups-only')
else:
    asa_all_groups_list = util.mailinglist.MailmanList('asa-test-mailman')
    gsc_fb_list = util.mailinglist.MailmanList('asa-test-mailman')
    finboard_groups_list = util.mailinglist.MoiraList('asa-test-moira')

class DiffCallback(object):
    def start_run(self, since, now, ):
        pass
    def end_run(self, ):
        pass
    def handle_group(self, before, after, before_fields, after_fields, ):
        pass
    def handle_signatories(self, signatories, ):
        pass
    def new_group(self, after, after_fields, ):
        pass

class StaticMailCallback(DiffCallback):
    def __init__(self, fields, address, template, signatories=[]):
        self.fields = fields
        self.address = address
        self.template = template
        self.interesting_signatories = signatories
        self.care_about_groups = True
        self.care_about_signatories = True

    def start_run(self, since, now, ):
        self.updates = []
        self.signatory_updates = []
        self.signatory_type_counts = {
            'Added': collections.defaultdict(lambda: 0),
            'Expired': collections.defaultdict(lambda: 0),
        }
        self.signatory_types_seen = set()
        self.since = since
        self.now = now

    def handle_group(self, before, after, before_fields, after_fields, ):
        after_revision = after.revision
        update = "Group: %s (ID #%d)\n" % (after_fields['name'], after_fields['id'], )
        update += "  At %s by %s (and possibly other people or times)\n" % (
            after_revision.date_created, after_revision.user, )
        for field in self.fields:
            if before_fields[field] != after_fields[field]:
                update += '  %18s: %12s -> %12s\n' % (
                    field, before_fields[field], after_fields[field], )
        self.updates.append(update)

    def handle_signatories(self, signatories, ):
        prev_group = None
        for signatory in signatories:
            if signatory.end_time > self.now:
                change_type = "Added"
            else:
                change_type = "Expired"
            counter = self.signatory_type_counts[change_type]
            counter[signatory.role.slug] += 1
            self.signatory_types_seen.add(signatory.role.slug)
            if signatory.role.slug in self.interesting_signatories:
                if signatory.group != prev_group:
                    self.signatory_updates.append("")
                self.signatory_updates.append(
                    "%s: %s: %s: %s:\n\trange %s to %s" % (
                        change_type,
                        signatory.group,
                        signatory.role,
                        signatory.person,
                        signatory.start_time.strftime(settings.DATETIME_FORMAT_PYTHON),
                        signatory.end_time.strftime(settings.DATETIME_FORMAT_PYTHON),
                    ))
                prev_group = signatory.group
            else:
                pass
                #print "Ignoring role %s (signatory %s)" % (signatory.role.slug, signatory, )

    def build_change_stats(self, ):
        lines = []
        care_about = 0

        line = "%20s" % ("", )
        change_types = self.signatory_type_counts.keys()
        for change_type in change_types:
            line += "\t%s" % (change_type, )
        lines.append(line); line = ""

        for sig_type in self.signatory_types_seen:
            anno_sig = sig_type
            if sig_type in self.interesting_signatories:
                anno_sig += " "
            else:
                anno_sig += "*"
            line += "%20s" % (anno_sig, )
            for change_type in change_types:
                if sig_type in self.signatory_type_counts[change_type]:
                    count = self.signatory_type_counts[change_type][sig_type]
                else:
                    count = 0
                if sig_type in self.interesting_signatories:
                    care_about += count
                out = "\t%4d" % (count, )
                line += out
            lines.append(line); line = ""

        line = "* Details for this signatory type not included in email."
        lines.append(line)

        return "\n".join(lines), care_about

    def end_run(self, ):
        change_stats, care_about = self.build_change_stats()
        print "\nChange stats for email to %s:" % (self.address, )
        print change_stats

        message = "\n\n".join(self.updates)
        signatories_message = "\n".join(self.signatory_updates)
        if (self.care_about_groups and self.updates) or (self.care_about_signatories and self.signatory_updates):
            pass
        else:
            return
        tmpl = get_template(self.template)
        ctx = Context({
            'num_groups': len(self.updates),
            'groups_message': message,
            'care_about': care_about,
            'change_stats': change_stats,
            'signatory_types': self.interesting_signatories,
            'signatories_message': signatories_message,
        })
        body = tmpl.render(ctx)
        email = EmailMessage(
            subject='ASA Database Updates',
            body=body,
            from_email='asa-db@mit.edu',
            to=[self.address, ],
            bcc=['asa-db-outgoing@mit.edu', ]
        )
        email.send()
        self.updates = []
        self.signatory_updates = []


class UpdateOfficerListCallback(DiffCallback):
    def __init__(self, listobj, include_pred=None):
        self.listobj = listobj
        if not include_pred:
            include_pred = lambda version, fields: True
        self.include_pred = include_pred

    def start_run(self, since, now, ):
        self.add = []
        self.delete = []
        self.notes = []

    def end_run(self, ):
        if self.add or self.delete:
            errors = self.listobj.change_members(self.add, self.delete)
            listname = self.listobj.name
            subject = "[ASA DB] %s updater" % (listname, )
            if errors:
                subject = "ERROR: " + subject
            context = {
                'listname': listname,
                'add': self.add,
                'delete': self.delete,
                'errors': errors,
                'notes': self.notes,
            }
            util.emails.email_from_template(
                tmpl='groups/diffs/list-update.txt',
                context=context, subject=subject,
                to=['asa-db-auto@mit.edu'],
            ).send()

    def update_changes(self, change_list, phase_name, name, addr, include, force_note=False, ):
        """
        Given an address and whether to process this item, update a list as appropriate and supply appropriate diagnostic notes.
        """

        note = None
        if addr and include:
            change_list.append((name, addr, ))
            if force_note:
                note = "email address is %s" % (addr, )
        elif not include:
            note = "doesn't pass predicate"
        elif not addr:
            note = "address is blank"
        else:
            note = "Something weird happened while adding (addr='%s', include=%s)" % (addr, include)
        if note:
            self.notes.append("%8s: %s: %s" % (phase_name, name, note, ))

    def handle_group(self, before, after, before_fields, after_fields, ):
        before_include = self.include_pred(before, before_fields)
        after_include = self.include_pred(after, after_fields)
        before_addr = before_fields['officer_email']
        after_addr  = after_fields['officer_email']

        # check if a change is appropriate
        effective_before_addr = before_addr if before_include else None
        effective_after_addr = after_addr if after_include else None

        if effective_before_addr != effective_after_addr:
            name = after_fields['name']
            self.update_changes(self.delete, "Delete", name, before_addr, before_include)
            self.update_changes(self.add, "Add", name, after_addr, after_include)

    def new_group(self, after, after_fields, ):
        name = after_fields['name']
        email = after_fields['officer_email']
        include = self.include_pred(after, after_fields)
        self.update_changes(self.add, "New", name, email, include, force_note=True)


def default_active_pred():
    status_objs = groups.models.GroupStatus.objects.filter(slug__in=['active', 'suspended', ])
    status_pks = [status.pk for status in status_objs]
    def pred(version, fields):
        return fields['group_status'] in status_pks
    return pred

def funded_pred(funding_slug):
    classes = groups.models.GroupClass.objects
    class_pk = classes.get(slug='mit-funded').pk
    fundings = groups.models.GroupFunding.objects
    fund_pk = fundings.get(slug=funding_slug).pk
    active_pred = default_active_pred()
    def pred(version, fields):
        return active_pred(version, fields) and fields['group_class'] == class_pk and fields['group_funding'] == fund_pk
    return pred

def build_callbacks():
    callbacks = []
    callbacks.append(StaticMailCallback(
        fields=[
            'name', 'abbreviation',
            'officer_email', 'group_email', 'athena_locker',
            'website_url', 'constitution_url',
            'activity_category', 'group_class', 'group_status', 'group_funding',
            'advisor_name',
            'num_undergrads', 'num_grads', 'num_community', 'num_other',
            'main_account_id', 'funding_account_id',
        ],
        address='asa-internal@mit.edu',     # some of these fields aren't public
        template='groups/diffs/asa-update-mail.txt',
        signatories=['president', 'treasurer', 'financial', 'group-admin', 'temp-admin', ]
    ))
    sao_callback = StaticMailCallback(
        fields=['name', 'abbreviation', 'officer_email', ],
        address='funds@mit.edu',
        template='groups/diffs/sao-update-mail.txt',
        signatories=['president', 'treasurer', 'financial', ]
    )
    sao_callback.care_about_groups = False
    callbacks.append(sao_callback)
    callbacks.append(UpdateOfficerListCallback(
        listobj=asa_all_groups_list,
        include_pred=default_active_pred(),
    ))
    callbacks.append(UpdateOfficerListCallback(
        listobj=finboard_groups_list,
        include_pred=funded_pred('undergrad'),
    ))
    callbacks.append(UpdateOfficerListCallback(
        listobj=gsc_fb_list,
        include_pred=funded_pred('grad'),
    ))
    return callbacks

def recent_groups(since):
    group_type = ContentType.objects.get_by_natural_key(app_label='groups', model='group')
    revisions = reversion.models.Revision.objects.filter(date_created__gte=since)
    versions = reversion.models.Version.objects.filter(revision__in=revisions, content_type=group_type)
    objs = versions.values("content_type", "object_id").distinct()
    return objs

def diff_objects(objs, since, callbacks, stats, ):
    revs  = reversion.models.Revision.objects.all()
    old_revs = revs.filter(date_created__lte=since)
    new_revs = revs.filter(date_created__gte=since)
    for obj in objs:
        all_versions = reversion.models.Version.objects.filter(content_type=obj['content_type'], object_id=obj['object_id']).order_by('-revision__date_created')
        before_versions = all_versions.filter(revision__in=old_revs)[:1]
        # This object being passed in means that some version changed it.
        after_versions = all_versions.filter(revision__in=new_revs).select_related('revision__user')
        after = after_versions[0]
        after_fields = after.field_dict

        if len(before_versions) > 0 or len(after_versions) > 1:
            if len(before_versions) > 0:
                before = before_versions[0]
                stats['change_old'] += 1
            else:
                # New group that's been edited since. Diff against the creation
                # (since creation sent mail, but later changes haven't)
                before = after_versions.reverse()[0]
                stats['change_new'] += 1
            stats['change_total'] += 1
            #print "Change?: before=%s (%d), after=%s (%d), type=%s, new=%s" % (
            #    before, before.pk,
            #    after, after.pk,
            #    after.type, after.field_dict,
            #)
            before_fields = before.field_dict
            for callback in callbacks:
                callback.handle_group(before, after, before_fields, after_fields)
        else:
            # New group that's only been edited once
            # Note that "normal" new groups will have their startup form
            # (which creates the Group object) and the approval (which makes
            # more changes, so this is group startups + NGEs, not actually
            # normal new groups)
            stats['new_group'] += 1
            callback.new_group(after, after_fields)

def diff_signatories(since, now, callbacks):
    # First: still around; then added recently
    qobj_added = Q(end_time__gte=now, start_time__gte=since)
    # First: already gone; then it existed for a while; finally expired recently
    qobj_expired = Q(end_time__lte=now, start_time__lte=since, end_time__gte=since)
    changed_signatories = groups.models.OfficeHolder.objects.filter(qobj_added|qobj_expired)
    changed_signatories = changed_signatories.order_by('group__name', 'role__display_name', 'person', )
    changed_signatories = changed_signatories.select_related('role', 'group')
    for callback in callbacks: callback.handle_signatories(changed_signatories)

def generate_diffs():
    now = datetime.datetime.now()
    recent = now - datetime.timedelta(hours=24, minutes=15)
    objs = recent_groups(since=recent)
    callbacks = build_callbacks()
    stats = collections.defaultdict(lambda: 0)
    for callback in callbacks: callback.start_run(since=recent, now=now, )
    diff_objects(objs, since=recent, callbacks=callbacks, stats=stats)
    diff_signatories(recent, now, callbacks=callbacks, )
    for callback in callbacks: callback.end_run()

    print "\nOverall change stats:"
    for stat_key, stat_val in stats.items():
        print "%20s:\t%6d" % (stat_key, stat_val, )
    print ""

if __name__ == '__main__':
    generate_diffs()