Changeset 7ef7143

Timestamp:

Jan 22, 2012, 10:48:51 AM (14 years ago)

Author:

Alex Dehnert <adehnert@…>

Branches:

master, space-access, stable, stage

Children:

fb1c047

Parents:

0c4be73

git-author:

Alex Dehnert <adehnert@…> (01/19/12 05:20:05)

git-committer:

Alex Dehnert <adehnert@…> (01/22/12 10:48:51)

Message:

Regularly download constitutions (Trac #50)

Adds groups/gather_constitutions.py and associated support code to regularly
download group constitutions and track them.

This will not treat 404ing or 500ing constitutions as an error.

Files:

• .gitignore (modified) (1 diff)
• asadb/groups/admin.py (modified) (1 diff)
• asadb/groups/gather_constitutions.py (added)
• asadb/groups/migrations/0020_constitutions.py (added)
• asadb/groups/models.py (modified) (3 diffs)

.gitignore

@@ -10 +10 @@
 asadb/util/static-data/
 asadb/util/warehouse-dump.csv
+constitutions

asadb/groups/admin.py

@@ -26 +26 @@
     search_fields = [ 'id', 'name', 'abbreviation', 'officer_email', 'athena_locker', ]
 admin.site.register(groups.models.Group, GroupAdmin)
+
+
+class Admin_GroupConstitution(VersionAdmin):
+    list_display = (
+        'pk',
+        'group',
+        'source_url',
+        'dest_file',
+        'last_update',
+        'last_download',
+        'failure_date',
+    )
+admin.site.register(groups.models.GroupConstitution, Admin_GroupConstitution)
 
 

asadb/groups/models.py

@@ -1 +1 @@
 from django.db import models
 from django.contrib.auth.models import User
+from django.template.defaultfilters import slugify
 import reversion
 
 import datetime
+import filecmp
+import mimetypes
+import os
+import re
+import shutil
+import urllib
 
 import settings
@@ -85 +92 @@
         return office_holders
 
+    def slug(self, ):
+        return slugify(self.name)
+
     def __str__(self, ):
         return self.name
@@ -111 +121 @@
     (GROUP_STARTUP_STAGE_REJECTED,      'rejected'),
 )
+
+constitution_dir = os.path.join(settings.SITE_ROOT, '..', 'constitutions')
+
+class GroupConstitution(models.Model):
+    group = models.ForeignKey(Group, unique=True, )
+    source_url = models.URLField()
+    dest_file = models.CharField(max_length=100)
+    last_update = models.DateTimeField(help_text='Last time when this constitution actually changed.')
+    last_download = models.DateTimeField(help_text='Last time we downloaded this constitution to see if it had changed.')
+    failure_date = models.DateTimeField(null=True, blank=True, default=None, help_text='Time this URL started failing to download. (Null if currently working.)')
+    status_msg = models.CharField(max_length=20)
+    failure_reason = models.CharField(max_length=100, blank=True, default="")
+
+    def update(self, ):
+        url = self.source_url
+        success = None
+        old_success = (self.failure_date is None)
+        if url:
+            url_opener = urllib.FancyURLopener()
+            now = datetime.datetime.now()
+            try:
+                tmp_path, headers = url_opener.retrieve(url)
+            except IOError:
+                self.failure_date = now
+                self.save()
+                success = False
+                self.status_msg = "retrieval failed"
+                self.failure_reason = self.status_msg
+                return (success, self.status_msg, old_success, )
+            if tmp_path == url:
+                mover = shutil.copyfile
+            else:
+                mover = shutil.move
+            save_filename = self.compute_filename(tmp_path, headers, )
+            dest_path = self.path_from_filename(self.dest_file)
+            if save_filename != self.dest_file:
+                if self.dest_file: os.remove(dest_path)
+                mover(tmp_path, self.path_from_filename(save_filename))
+                self.dest_file = save_filename
+                self.last_update = now
+                self.status_msg = "new path"
+            else:
+                if filecmp.cmp(tmp_path, dest_path, shallow=False, ):
+                    self.status_msg = "no change"
+                else:
+                    # changed
+                    mover(tmp_path, dest_path)
+                    self.last_update = now
+                    self.status_msg = "updated in place"
+            self.last_download = now
+            self.failure_date = None
+            self.failure_reason = ""
+            self.save()
+            success = True
+        else:
+            success = False
+            self.status_msg = "no url"
+            self.failure_reason = self.status_msg
+        return (success, self.status_msg, old_success, )
+
+    def compute_filename(self, tmp_path, headers, ):
+        slug = self.group.slug()
+        basename, fileext = os.path.splitext(tmp_path)
+        if fileext:
+            ext = fileext
+        else:
+            if headers.getheader('Content-Type'):
+                mimeext = mimetypes.guess_extension(headers.gettype())
+                if mimeext:
+                    ext = mimeext
+                else:
+                    ext = ''
+            else:
+                ext = ''
+        return "%04d-%s%s" % (self.group.pk, slug, ext, )
+
+    def path_from_filename(self, filename):
+        path = os.path.join(constitution_dir, filename)
+        return path
+
+reversion.register(GroupConstitution)
+
 
 class GroupStartup(models.Model):