Fix overflow bug and missing return in Polygon encodeCompressed. (#271)

This change also adds compressed encodings to the fuzz test which would
have allowed it to catch this.

Author: alan-strohm

s2/polygon.go

@@ -1189,9 +1189,10 @@ func (p *Polygon) decodeCompressed(d *decoder) {
 	}
 	// Polygons with no loops are explicitly allowed here: a newly created
 	// polygon has zero loops and such polygons encode and decode properly.
-	nloops := int(d.readUvarint())
+	nloops := d.readUvarint()
 	if nloops > maxEncodedLoops {
 		d.err = fmt.Errorf("too many loops (%d; max is %d)", nloops, maxEncodedLoops)
+		return
 	}
 	p.loops = make([]*Loop, nloops)
 	for i := range p.loops {

s2/polygon_test.go

@@ -1207,14 +1207,49 @@ func TestPolygonInvert(t *testing.T) {
 //   TestCloselySpacedEdgeVerticesKept
 //   TestPolylineAssemblyBug
 
+func TestPolygonDecodeDoesNotOverflowForLargeNLoops(t *testing.T) {
+	var buf bytes.Buffer
+	e := encoder{w: &buf}
+
+	e.writeUint8(uint8(encodingPolygonCompressedVersion))
+	e.writeUint8(0) // snap level 0
+	// nloops: large value that overflows to negative when cast to int.
+	e.writeUvarint(1 << 63)
+
+	if e.err != nil {
+		t.Fatal(e.err)
+	}
+
+	var p Polygon
+	if err := p.Decode(bytes.NewReader(buf.Bytes())); err == nil {
+		t.Fatalf("Decode(overflow nloops) got nil error, want non-nil")
+	}
+}
+
 // go test -fuzz=FuzzDecodePolygon github.com/golang/geo/s2
 func FuzzDecodePolygon(f *testing.F) {
 	for _, p := range []*Polygon{near0Polygon, near01Polygon, near30Polygon, near23Polygon, far01Polygon, far21Polygon, south0abPolygon} {
-		buf := new(bytes.Buffer)
-		if err := p.Encode(buf); err != nil {
+		// Add lossless encoding.
+		var buf bytes.Buffer
+		if err := p.Encode(&buf); err != nil {
 			f.Errorf("error encoding %v: ", err)
 		}
 		f.Add(buf.Bytes())
+
+		// Add compressed encoding.
+		var bufCompressed bytes.Buffer
+		e := encoder{w: &bufCompressed}
+
+		vs := make([]xyzFaceSiTi, 0, p.numVertices)
+		for _, l := range p.loops {
+			vs = append(vs, l.xyzFaceSiTiVertices()...)
+		}
+
+		p.encodeCompressed(&e, 0, vs)
+		if e.err != nil {
+			f.Errorf("error encoding compressed polygon: %v", e.err)
+		}
+		f.Add(bufCompressed.Bytes())
 	}
 
 	f.Fuzz(func(t *testing.T, encoded []byte) {
@@ -1223,6 +1258,10 @@ func FuzzDecodePolygon(f *testing.F) {
 			// Construction failed, no need to test further.
 			return
 		}
+		if err := p.Validate(); err != nil {
+			// Decoded but invalid polygon. Skip property checks.
+			return
+		}
 		if got := p.Area(); got < 0 {
 			t.Errorf("Area() = %v, want >= 0. Polygon: %v", got, p)
 		}