Alert system DoS prevention

This fixes two alert system vulnerabilities found by
Sergio Lerner; you could send peers unlimited numbers
of invalid alert message to try to either fill up their
debug.log with messages and/or keep their CPU busy
checking signatures.

Fixed by disconnecting/banning peers if they send 10 or more
bad (invalid/expired/cancelled) alerts.

src/main.cpp

@@ -2997,16 +2997,29 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv)
         CAlert alert;
         vRecv >> alert;
 
+        uint256 alertHash = alert.GetHash();
+        if (pfrom->setKnown.count(alertHash) == 0)
+        {
             if (alert.ProcessAlert())
             {
                 // Relay
-            pfrom->setKnown.insert(alert.GetHash());
+                pfrom->setKnown.insert(alertHash);
                 {
                     LOCK(cs_vNodes);
                     BOOST_FOREACH(CNode* pnode, vNodes)
                         alert.RelayTo(pnode);
                 }
             }
+            else {
+                // Small DoS penalty so peers that send us lots of
+                // duplicate/expired/invalid-signature/whatever alerts
+                // eventually get banned.
+                // This isn't a Misbehaving(100) (immediate ban) because the
+                // peer might be an older or different implementation with
+                // a different signature key, etc.
+                pfrom->Misbehaving(10);
+            }
+        }
     }
 
 

src/main.h

@@ -1535,7 +1535,7 @@ public:
 
     uint256 GetHash() const
     {
-        return SerializeHash(*this);
+        return Hash(this->vchMsg.begin(), this->vchMsg.end());
     }
 
     bool IsInEffect() const