Changeset e0632f6 for asadb/groups

Timestamp:

Sep 10, 2012, 5:02:55 AM (13 years ago)

Author:

Alex Dehnert <adehnert@…>

Branches:

master, space-access, stable, stage

Children:

ab17953

Parents:

3af5d74 (diff), 51b384a (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

git-author:

Alex Dehnert <adehnert@…> (09/10/12 05:02:55)

git-committer:

Alex Dehnert <adehnert@…> (09/10/12 05:02:55)

Message:

Merge branch 'constitutions'

Location:

asadb/groups

Files:

• gather_constitutions.py (modified) (2 diffs)
• migrations/0025_gather_constitutions_user.py (added)
• models.py (modified) (5 diffs)
• urls.py (modified) (1 diff)
• views.py (modified) (2 diffs)

asadb/groups/gather_constitutions.py

@@ -12 +12 @@
 import datetime
 import subprocess
+
+import django.contrib.auth.models
+import reversion
 
 import groups.models
@@ -71 +74 @@
         print "%4d\t%s" % (len(gs), code, )
 
+def list_constitutions():
+    constitutions = groups.models.GroupConstitution.objects.all()
+    for const in constitutions:
+        if const.dest_file: print const.dest_file
+
 if __name__ == '__main__':
     if len(sys.argv) == 1 or sys.argv[1] == "gather":
-        additions, changed = gather_constitutions()
+        with reversion.create_revision():
+            additions, changed = gather_constitutions()
+            importer = django.contrib.auth.models.User.objects.get(username='gather-constitutions@SYSTEM', )
+            reversion.set_user(importer)
+            reversion.set_comment("gather constitutions")
+
         update_repo(additions, changed)
     elif sys.argv[1] == "webstat":
         webstat()
+    elif sys.argv[1] == "list":
+        list_constitutions()
     else:
         raise NotImplementedError

asadb/groups/models.py

@@ -11 +11 @@
 import re
 import shutil
+import urlparse
 import urllib
+import urllib2
 
 import settings
+
+import mit
 
 # Create your models here.
@@ -34 +38 @@
     group_funding = models.ForeignKey('GroupFunding', null=True, blank=True, db_index=True, )
     website_url = models.URLField()
-    constitution_url = models.CharField(max_length=200, blank=True)
+    constitution_url = models.CharField(max_length=200, blank=True, validators=[mit.UrlOrAfsValidator])
     meeting_times = models.TextField(blank=True)
     advisor_name = models.CharField(max_length=100, blank=True)
@@ -137 +141 @@
     failure_reason = models.CharField(max_length=100, blank=True, default="")
 
+    def record_failure(self, msg):
+        now = datetime.datetime.now()
+        if not self.failure_date:
+            self.failure_date = now
+        self.status_msg = msg
+        self.failure_reason = self.status_msg
+        self.save()
+
+    def record_success(self, msg, updated):
+        now = datetime.datetime.now()
+        if updated:
+            self.last_update = now
+        self.status_msg = msg
+        self.last_download = now
+        self.failure_date = None
+        self.failure_reason = ""
+        self.save()
+
     def update(self, ):
         url = self.source_url
@@ -142 +164 @@
         old_success = (self.failure_date is None)
         if url:
-            url_opener = urllib.FancyURLopener()
-            now = datetime.datetime.now()
+            # Fetch the file
+            error_msg = None
             try:
-                tmp_path, headers = url_opener.retrieve(url)
+                new_mimetype = None
+                if url.startswith('/afs/') or url.startswith('/mit/'):
+                    new_fp = open(url, 'rb')
+                else:
+                    new_fp = urllib2.urlopen(url)
+                    if new_fp.info().getheader('Content-Type'):
+                        new_mimetype = new_fp.info().gettype()
+
+                new_data = new_fp.read()
+                new_fp.close()
+            except urllib2.HTTPError, e:
+                error_msg = "HTTPError: %s %s" % (e.code, e.msg)
+            except urllib2.URLError, e:
+                error_msg = "URLError: %s" % (e.reason)
             except IOError:
-                self.failure_date = now
-                self.save()
-                success = False
-                self.status_msg = "retrieval failed"
-                self.failure_reason = self.status_msg
-                return (success, self.status_msg, old_success, )
-            if tmp_path == url:
-                mover = shutil.copyfile
-            else:
-                mover = shutil.move
-            save_filename = self.compute_filename(tmp_path, headers, )
-            dest_path = self.path_from_filename(self.dest_file)
-            if save_filename != self.dest_file:
-                if self.dest_file: os.remove(dest_path)
-                mover(tmp_path, self.path_from_filename(save_filename))
-                self.dest_file = save_filename
-                self.last_update = now
-                self.status_msg = "new path"
-            else:
-                if filecmp.cmp(tmp_path, dest_path, shallow=False, ):
-                    self.status_msg = "no change"
+                error_msg = "IOError"
+            except ValueError, e:
+                if e.args[0].startswith('unknown url type'):
+                    error_msg = "unknown url type"
                 else:
-                    # changed
-                    mover(tmp_path, dest_path)
-                    self.last_update = now
-                    self.status_msg = "updated in place"
-            self.last_download = now
-            self.failure_date = None
-            self.failure_reason = ""
-            self.save()
+                    raise
+            if error_msg:
+                self.record_failure(error_msg)
+                return (False, self.status_msg, old_success, )
+
+            # At this point, failures are our fault, not the group's.
+            # We can let any errors bubble all the way up, rather than
+            # trying to catch and neatly record them
             success = True
+
+            # Find a destination, and how to put it there
+            old_path = self.path_from_filename(self.dest_file)
+            new_filename = self.compute_filename(url, new_mimetype, )
+
+            # Process the update
+            if new_filename != self.dest_file: # new filename
+                if self.dest_file:
+                    if os.path.exists(old_path):
+                        os.remove(old_path)
+                    else:
+                        print "Warning: %s doesn't exist, but is referenced by dest_file" % (old_path, )
+                self.dest_file = new_filename
+                new_path = self.path_from_filename(new_filename)
+                with open(new_path, 'wb') as fp:
+                    fp.write(new_data)
+                self.record_success("new path", updated=True)
+            else: # old filename
+                with open(old_path, 'rb') as old_fp:
+                    old_data = old_fp.read()
+                if old_data == new_data: # unchanged
+                    self.record_success("no change", updated=False)
+                else: # changed
+                    with open(old_path, 'wb') as fp:
+                        fp.write(new_data)
+                    self.record_success("updated in place", updated=True)
+
         else:
+            self.record_failure("no url")
             success = False
-            self.status_msg = "no url"
-            self.failure_reason = self.status_msg
+
         return (success, self.status_msg, old_success, )
 
-    def compute_filename(self, tmp_path, headers, ):
+    def compute_filename(self, url, mimetype):
         slug = self.group.slug()
-        basename, fileext = os.path.splitext(tmp_path)
+        known_ext = set([
+            '.pdf',
+            '.ps',
+            '.doc',
+            '.rtf',
+            '.html',
+            '.tex',
+            '.txt'
+        ])
+
+        # This probably breaks on Windows. But that's probably true of
+        # everything...
+        path = urlparse.urlparse(url).path
+        basename, fileext = os.path.splitext(path)
+
         if fileext:
             ext = fileext
         else:
-            if headers.getheader('Content-Type'):
-                mimeext = mimetypes.guess_extension(headers.gettype())
-                if mimeext:
-                    ext = mimeext
+            if mimetype:
+                extensions = mimetypes.guess_all_extensions(mimetype)
+                for extension in extensions:
+                    if extension in known_ext:
+                        ext = extension
+                        break
                 else:
-                    ext = ''
+                    if len(extensions) > 0:
+                        ext = extensions[0]
+                    else:
+                        ext = ''
             else:
                 ext = ''
+
+        extmap = {
+            '.htm': '.html',
+            '.php': '.html',
+            '.PS':  '.ps',
+            '.shtml':   '.html',
+            '.text':    '.txt',
+        }
+        # we have no real handling of no extension, .old, and .ksh
+        if ext in extmap: ext = extmap[ext]
+        if ext not in known_ext: ext = ext + '.unknown'
+
         return "%04d-%s%s" % (self.group.pk, slug, ext, )
 
@@ -210 +286 @@
                 stream = urllib.urlopen(self.source_url)
                 return stream.getcode()
-            except:
+            except IOError:
                 return "IOError"
         else:

asadb/groups/urls.py

@@ -23 +23 @@
     url(r'^recent_changes/$', groups.views.GroupHistoryView.as_view(), name='manage-history', ),
     url(r'^signatories/$', groups.views.view_signatories, name='signatories', ),
+    url(r'^constitutions/$', groups.views.downloaded_constitutions, name='constitutions', ),
     url(r'^account_lookup/$', groups.views.account_lookup, name='account-lookup', ),
 )

asadb/groups/views.py

@@ -128 +128 @@
         for field in self.force_required:
             self.fields[field].required = True
-        self.fields['constitution_url'].help_text = mark_safe("Please put your current constitution URL, if you have one.<br>If your constitution is currently an AFS path, you can either use the corresponding web.mit.edu (e.g., http://web.mit.edu/locker/path/to/const.html) or stuff.mit.edu path, or just use http://asa.mit.edu/const/afs/your-afs-path.<br>If you don't currently know where your constitution is, put http://mit.edu/asa/start/constitution-req.html.<br>(In either of these last two cases, we'll get in touch with you later about putting something better in.)")
+        self.fields['constitution_url'].help_text = mark_safe("""Please put your current constitution URL or AFS path.<br>If you don't currently know where your constitution is, put "http://mit.edu/asa/start/constitution-req.html" and draft a constitution soon.""")
 
     exec_only_fields = [
@@ -1025 +1025 @@
 
 
+def downloaded_constitutions(request, ):
+    constitutions = groups.models.GroupConstitution.objects
+    constitutions = constitutions.order_by('failure_reason', 'status_msg', 'failure_date', 'group__name', ).select_related('group', 'group__group_status', )
+    failures = collections.defaultdict(list)
+    successes = collections.defaultdict(list)
+    for const in constitutions:
+        if const.failure_reason:
+            failures[const.failure_reason].append(const)
+        else:
+            successes[const.status_msg].append(const)
+    context = {}
+    context['failures']  = sorted(failures.items(),  key=lambda x: x[0])
+    context['successes'] = sorted(successes.items(), key=lambda x: x[0])
+    context['pagename'] = 'groups'
+    return render_to_response('groups/groups_constitutions.html', context, context_instance=RequestContext(request), )
+
+
 
 #######################