Add short caching for failed URL fetches

Previously, failed fetches (blocked sites, timeouts, etc.) were never
cached, causing every request for the same URL to re-hit the target
site. This wastes resources and can trigger further rate limiting.

Failed results are now cached for 5 minutes.

Author: roryist

unfurlist.go

@@ -406,10 +406,18 @@ hasMatch:
 		result.Image, result.ImageWidth, result.ImageHeight = "", 0, 0
 	}
 
-	if mc := h.Cache; mc != nil && !result.Empty() {
+	if mc := h.Cache; mc != nil {
 		if cdata, err := json.Marshal(result); err == nil {
-			h.Log.Printf("Cache update for %q", link)
-			mc.Set(&memcache.Item{Key: mcKey(link), Value: snappy.Encode(nil, cdata)})
+			item := &memcache.Item{Key: mcKey(link), Value: snappy.Encode(nil, cdata)}
+			if result.Empty() {
+				// Cache failures briefly to avoid repeatedly hitting sites
+				// that block us. A short TTL ensures we retry eventually.
+				item.Expiration = 300 // 5 minutes
+				h.Log.Printf("Empty result cache for %q", link)
+			} else {
+				h.Log.Printf("Cache update for %q", link)
+			}
+			mc.Set(item)
 		}
 	}
 	return result