Rewrite API to use the Python requests library.

(imported from commit 314cf906eb25ea46f580ca70852b9d3478531229)

common.py

@@ -1,21 +1,14 @@
 #!/usr/bin/python
-import mechanize
-import urllib
 import simplejson
-from urllib2 import HTTPError
+import requests
 import time
 import traceback
 
-# TODO: Just switch to pycurl
+# TODO: Drop verify=False once we have real certificates
-class NoHistory(object):
+# Or switch to specifying a testing cert manually
-    def add(self, *a, **k): pass
-    def clear(self): pass
 
 class HumbugAPI():
     def __init__(self, email, api_key, verbose=False, site="https://app.humbughq.com"):
-        self.browser = mechanize.Browser(history=NoHistory())
-        self.browser.set_handle_robots(False)
-        self.browser.add_password("https://app.humbughq.com/", "tabbott", "xxxxxxxxxxxxxxxxx", "wiki")
         self.api_key = api_key
         self.email = email
         self.verbose = verbose
@@ -24,41 +17,54 @@ class HumbugAPI():
     def send_message(self, submit_hash):
         submit_hash["email"] = self.email
         submit_hash["api-key"] = self.api_key
-        submit_data = urllib.urlencode([(k, v) for k,v in submit_hash.items()])
+        try:
-        res = self.browser.open(self.base_url + "/api/v1/send_message", submit_data)
+            res = requests.post(self.base_url + "/api/v1/send_message",
-        return simplejson.loads(res.read())
+                                data=submit_hash,
+                                verify=False,
+                                auth=requests.auth.HTTPDigestAuth('tabbott', 'xxxxxxxxxxxxxxxxx'))
+            # TODO: Add some sort of automated retry for certain errors
+        except requests.exceptions.ConnectionError:
+            return {'msg': "Connection error\n%s" % traceback.format_exc(),
+                    "result": "connection-error"}
+        if res.json is not None:
+            return res.json
+        return {'msg': res.text, "result": "unexpected-error",
+                "status_code": res.status_code}
 
     def get_messages(self, options = {}):
         options["email"] = self.email
         options["api-key"] = self.api_key
-        submit_data = urllib.urlencode([(k, v.encode('utf-8')) for k,v in options.items()])
+        try:
-        res = self.browser.open(self.base_url + "/api/v1/get_messages", submit_data)
+            res = requests.post(self.base_url + "/api/v1/get_messages",
-        return simplejson.loads(res.read())['messages']
+                                data=options,
+                                verify=False,
+                                auth=requests.auth.HTTPDigestAuth('tabbott', 'xxxxxxxxxxxxxxxxx'))
+        except requests.exceptions.ConnectionError:
+            return {'msg': "Connection error\n%s" % traceback.format_exc(),
+                    "result": "connection-error"}
+        if res.json is not None:
+            return res.json
+        return {'msg': res.text, "result": "unexpected-error",
+                "status_code": res.status_code}
 
     def call_on_each_message(self, callback, options = {}):
         max_message_id = None
         while True:
-            try:
+            if max_message_id is not None:
-                if max_message_id is not None:
+                options["first"] = "0"
-                    options["first"] = "0"
+                options["last"] = str(max_message_id)
-                    options["last"] = str(max_message_id)
+            res = self.get_messages(options)
-                messages = self.get_messages(options)
+            if 'error' in res.get('result'):
-            except HTTPError, e:
-                # 502/503 typically means the server was restarted; sleep
-                # a bit, then try again
                 if self.verbose:
-                    print e
+                    if res["result"] == "unexpected-error":
-                    print "Error getting messages (probably server restart); retrying..."
+                        print "Unexpected error -- probably a server restart"
+                    elif res["result"] == "connection-error":
+                        print "Connection error -- probably server is down?"
+                    else:
+                        print "Server returned error:\n%s" % res["msg"]
+                # TODO: Make this back off once it's more reliable
                 time.sleep(1)
                 continue
-            except Exception, e:
+            for message in sorted(res['messages'], key=lambda x: x["id"]):
-                # For other errors, just try again
-                if self.verbose:
-                    print e
-                    print traceback.format_exc()
-                    print "Unexpected error!  You should send Tim the above traceback.  Retrying..."
-                time.sleep(2)
-                continue
-            for message in sorted(messages, key=lambda x: x["id"]):
                 max_message_id = max(max_message_id, message["id"])
                 callback(message)

zephyr_mirror.py

@@ -75,7 +75,7 @@ def send_humbug(zeph):
         elif isinstance(zeph[key], str):
             zeph[key] = zeph[key].decode("utf-8")
 
-    humbug_client.send_message(zeph)
+    return humbug_client.send_message(zeph)
 
 def fetch_fullname(username):
     try:
@@ -166,7 +166,11 @@ def process_loop(log):
 
             print "%s: received a message on %s/%s from %s..." % \
                 (datetime.datetime.now(), notice.cls, notice.instance, notice.sender)
-            send_humbug(zeph)
+            res = send_humbug(zeph)
+            if res.get("result") != "success":
+                print >>sys.stderr, 'Error relaying zephyr'
+                print zeph
+                print res
         except:
             print >>sys.stderr, 'Error relaying zephyr'
             traceback.print_exc()