#!/usr/bin/python import sys import os if __name__ == '__main__': cur_file = os.path.abspath(__file__) django_dir = os.path.abspath(os.path.join(os.path.dirname(cur_file), '..')) proj_dir = os.path.abspath(os.path.join(django_dir, '..')) sys.path.append(django_dir) sys.path.append(proj_dir) os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' import groups.models import collections import datetime from django.db import transaction fields = [ # Django field, in order matching the input fields 'username', 'mit_id', 'first_name', 'last_name', 'account_class', ] def load_dcm(dcm_stream): dcm_people = {} for line in dcm_stream: line = line.strip() field_list = line.split("\t") field_dict = {} for index, field in enumerate(fields): field_dict[field] = field_list[index] dcm_people[field_dict['username']] = field_dict return dcm_people @transaction.commit_manually def load_people(dcm_people): django_people = groups.models.AthenaMoiraAccount.objects.all() stat_loops = 0 stat_django_people = len(django_people) stat_dcm_people = len(dcm_people) stat_changed = 0 stat_mut_ign = 0 stat_unchanged = 0 stat_del = 0 stat_pre_del = 0 stat_undel = 0 stat_add = 0 stat_people = collections.defaultdict(list) for django_person in django_people: stat_loops += 1 if stat_loops % 100 == 0: transaction.commit() pass mutable = django_person.mutable if django_person.username in dcm_people: # great, they're still in the dump changed = False dcm_person = dcm_people[django_person.username] del dcm_people[django_person.username] for key in fields: if django_person.__dict__[key] != dcm_person[key]: changed = True if mutable: django_person.__dict__[key] = dcm_person[key] if django_person.del_date is not None: changed = True if mutable: django_person.del_date = None stat_undel += 1 stat_people['undel'].append(django_person.username) if changed: if mutable: django_person.mod_date = datetime.date.today() django_person.save() stat_changed += 1 stat_people['changed'].append(django_person.username) else: stat_mut_ign += 1 stat_people['mut_ign'].append(django_person.username) else: stat_unchanged += 1 else: if django_person.del_date is None: if mutable: django_person.del_date = datetime.date.today() stat_del += 1 stat_people['del'].append(django_person.username) django_person.save() else: stat_mut_ign += 1 stat_people['mut_ign'].append(django_person.username) else: stat_pre_del += 1 for username, dcm_person in dcm_people.items(): stat_loops += 1 if stat_loops % 100 == 0: transaction.commit() pass django_person = groups.models.AthenaMoiraAccount() for key in fields: django_person.__dict__[key] = dcm_person[key] django_person.add_date = datetime.date.today() stat_add += 1 stat_people['add'].append(django_person.username) django_person.save() transaction.commit() stats = { 'loops': stat_loops, 'django_people': stat_django_people, 'dcm_people': stat_dcm_people, 'changed': stat_changed, 'mut_ign': stat_mut_ign, 'unchanged': stat_unchanged, 'del': stat_del, 'pre_del': stat_pre_del, 'undel': stat_undel, 'add': stat_add, } return stats, stat_people if __name__ == '__main__': print "Phase 1 (DCM parsing): starting at %s" % (datetime.datetime.now(), ) dcm_people = load_dcm(sys.stdin) print "Phase 1 (DCM parsing): complete at %s" % (datetime.datetime.now(), ) print "Phase 2 (Django updating): starting at %s" % (datetime.datetime.now(), ) stats, stat_people = load_people(dcm_people) print "Phase 2 (Django updating): complete at %s" % (datetime.datetime.now(), ) print """ Loop iterations: %(loops)6d Initial in Django: %(django_people)6d People in DCM: %(dcm_people)6d Already Deleted: %(pre_del)6d Unchanged: %(unchanged)6d Changed: %(changed)6d Change ignored: %(mut_ign)6d Deleted: %(del)6d Undeleted: %(undel)6d Added: %(add)6d """ % stats for change_type, people in stat_people.items(): for person in people: print "%12s\t%s" % (change_type, person, ) print ""