Use ptrdiff_t, not int, for overlay counts.

* buffer.h (overlays_at, sort_overlays, GET_OVERLAYS_AT):
* editfns.c (overlays_around, get_pos_property):
* textprop.c (get_char_property_and_overlay):
* xdisp.c (next_overlay_change, note_mouse_highlight):
* xfaces.c (face_at_buffer_position):
* buffer.c (overlays_at, sort_overlays, Foverlays_at)
(Fnext_overlay_change, Fprevious_overlay_change):
Use ptrdiff_t, not int, for sizes.
(overlays_at): Check for size-calculation overflow.

7 files changed, 44 insertions, 34 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index 41b2fe9aab..875dcb8516 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,5 +1,16 @@
 2011-06-21  Paul Eggert  <[email protected]>
 
+	Use ptrdiff_t, not int, for overlay counts.
+	* buffer.h (overlays_at, sort_overlays, GET_OVERLAYS_AT):
+	* editfns.c (overlays_around, get_pos_property):
+	* textprop.c (get_char_property_and_overlay):
+	* xdisp.c (next_overlay_change, note_mouse_highlight):
+	* xfaces.c (face_at_buffer_position):
+	* buffer.c (overlays_at, sort_overlays, Foverlays_at)
+	(Fnext_overlay_change, Fprevious_overlay_change):
+	Use ptrdiff_t, not int, for sizes.
+	(overlays_at): Check for size-calculation overflow.
+
 	* xterm.c (xim_initialize, same_x_server): Strlen may not fit in int.
 
 	* xsmfns.c (smc_save_yourself_CB, x_session_initialize): Avoid strlen.
diff --git a/src/buffer.c b/src/buffer.c
index fb9b15e4c7..b8f85d1a1c 100644
--- a/src/buffer.c
+++ b/src/buffer.c
@@ -2518,18 +2518,21 @@ swap_out_buffer_local_variables (struct buffer *b)
    *NEXT_PTR is guaranteed to be not equal to POS, unless it is the
    default (BEGV or ZV).  */
 
-int
-overlays_at (EMACS_INT pos, int extend, Lisp_Object **vec_ptr, int *len_ptr,
+ptrdiff_t
+overlays_at (EMACS_INT pos, int extend, Lisp_Object **vec_ptr,
+	     ptrdiff_t *len_ptr,
 	     EMACS_INT *next_ptr, EMACS_INT *prev_ptr, int change_req)
 {
   Lisp_Object overlay, start, end;
   struct Lisp_Overlay *tail;
-  int idx = 0;
-  int len = *len_ptr;
+  ptrdiff_t idx = 0;
+  ptrdiff_t len = *len_ptr;
   Lisp_Object *vec = *vec_ptr;
   EMACS_INT next = ZV;
   EMACS_INT prev = BEGV;
   int inhibit_storing = 0;
+  ptrdiff_t len_lim = min (MOST_POSITIVE_FIXNUM,
+			   min (PTRDIFF_MAX, SIZE_MAX) / sizeof (Lisp_Object));
 
   for (tail = current_buffer->overlays_before; tail; tail = tail->next)
     {
@@ -2561,10 +2564,10 @@ overlays_at (EMACS_INT pos, int extend, Lisp_Object **vec_ptr, int *len_ptr,
 		 Either make it bigger, or don't store any more in it.  */
 	      if (extend)
 		{
+		  if ((len_lim - 4) / 2 < len)
+		    memory_full (SIZE_MAX);
 		  /* Make it work with an initial len == 0.  */
-		  len *= 2;
-		  if (len == 0)
-		    len = 4;
+		  len = len * 2 + 4;
 		  *len_ptr = len;
 		  vec = (Lisp_Object *) xrealloc (vec, len * sizeof (Lisp_Object));
 		  *vec_ptr = vec;
@@ -2604,10 +2607,10 @@ overlays_at (EMACS_INT pos, int extend, Lisp_Object **vec_ptr, int *len_ptr,
 	    {
 	      if (extend)
 		{
+		  if ((len_lim - 4) / 2 < len)
+		    memory_full (SIZE_MAX);
 		  /* Make it work with an initial len == 0.  */
-		  len *= 2;
-		  if (len == 0)
-		    len = 4;
+		  len = len * 2 + 4;
 		  *len_ptr = len;
 		  vec = (Lisp_Object *) xrealloc (vec, len * sizeof (Lisp_Object));
 		  *vec_ptr = vec;
@@ -2871,10 +2874,10 @@ compare_overlays (const void *v1, const void *v2)
 /* Sort an array of overlays by priority.  The array is modified in place.
    The return value is the new size; this may be smaller than the original
    size if some of the overlays were invalid or were window-specific.  */
-int
-sort_overlays (Lisp_Object *overlay_vec, int noverlays, struct window *w)
+ptrdiff_t
+sort_overlays (Lisp_Object *overlay_vec, ptrdiff_t noverlays, struct window *w)
 {
-  int i, j;
+  ptrdiff_t i, j;
   struct sortvec *sortvec;
   sortvec = (struct sortvec *) alloca (noverlays * sizeof (struct sortvec));
 
@@ -3880,9 +3883,8 @@ DEFUN ("overlays-at", Foverlays_at, Soverlays_at, 1, 1, 0,
        doc: /* Return a list of the overlays that contain the character at POS.  */)
   (Lisp_Object pos)
 {
-  int noverlays;
+  ptrdiff_t len, noverlays;
   Lisp_Object *overlay_vec;
-  int len;
   Lisp_Object result;
 
   CHECK_NUMBER_COERCE_MARKER (pos);
@@ -3942,11 +3944,9 @@ If there are no overlay boundaries from POS to (point-max),
 the value is (point-max).  */)
   (Lisp_Object pos)
 {
-  int noverlays;
+  ptrdiff_t i, len, noverlays;
   EMACS_INT endpos;
   Lisp_Object *overlay_vec;
-  int len;
-  int i;
 
   CHECK_NUMBER_COERCE_MARKER (pos);
 
@@ -3985,7 +3985,7 @@ the value is (point-min).  */)
 {
   EMACS_INT prevpos;
   Lisp_Object *overlay_vec;
-  int len;
+  ptrdiff_t len;
 
   CHECK_NUMBER_COERCE_MARKER (pos);
 
diff --git a/src/buffer.h b/src/buffer.h
index a13351b5ea..4643e0d9d0 100644
--- a/src/buffer.h
+++ b/src/buffer.h
@@ -887,10 +887,10 @@ extern struct buffer buffer_local_symbols;
 extern void delete_all_overlays (struct buffer *);
 extern void reset_buffer (struct buffer *);
 extern void evaporate_overlays (EMACS_INT);
-extern int overlays_at (EMACS_INT pos, int extend, Lisp_Object **vec_ptr,
-                        int *len_ptr, EMACS_INT *next_ptr,
-                        EMACS_INT *prev_ptr, int change_req);
-extern int sort_overlays (Lisp_Object *, int, struct window *);
+extern ptrdiff_t overlays_at (EMACS_INT pos, int extend, Lisp_Object **vec_ptr,
+			      ptrdiff_t *len_ptr, EMACS_INT *next_ptr,
+			      EMACS_INT *prev_ptr, int change_req);
+extern ptrdiff_t sort_overlays (Lisp_Object *, ptrdiff_t, struct window *);
 extern void recenter_overlay_lists (struct buffer *, EMACS_INT);
 extern EMACS_INT overlay_strings (EMACS_INT, struct window *, unsigned char **);
 extern void validate_region (Lisp_Object *, Lisp_Object *);
@@ -908,7 +908,7 @@ extern void mmap_set_vars (int);
 
 #define GET_OVERLAYS_AT(posn, overlays, noverlays, nextp, chrq)		\
   do {									\
-    int maxlen = 40;							\
+    ptrdiff_t maxlen = 40;							\
     overlays = (Lisp_Object *) alloca (maxlen * sizeof (Lisp_Object));	\
     noverlays = overlays_at (posn, 0, &overlays, &maxlen,		\
 			     nextp, NULL, chrq);				\
diff --git a/src/editfns.c b/src/editfns.c
index f0f8c9eb63..c0c0e53026 100644
--- a/src/editfns.c
+++ b/src/editfns.c
@@ -331,13 +331,13 @@ If you set the marker not to point anywhere, the buffer will have no mark.  */)
    Return the number found, and store them in a vector in VEC
    of length LEN.  */
 
-static int
-overlays_around (EMACS_INT pos, Lisp_Object *vec, int len)
+static ptrdiff_t
+overlays_around (EMACS_INT pos, Lisp_Object *vec, ptrdiff_t len)
 {
   Lisp_Object overlay, start, end;
   struct Lisp_Overlay *tail;
   EMACS_INT startpos, endpos;
-  int idx = 0;
+  ptrdiff_t idx = 0;
 
   for (tail = current_buffer->overlays_before; tail; tail = tail->next)
     {
@@ -405,7 +405,7 @@ get_pos_property (Lisp_Object position, register Lisp_Object prop, Lisp_Object o
   else
     {
       EMACS_INT posn = XINT (position);
-      int noverlays;
+      ptrdiff_t noverlays;
       Lisp_Object *overlay_vec, tem;
       struct buffer *obuf = current_buffer;
 
diff --git a/src/textprop.c b/src/textprop.c
index 350892cdad..dd8695f7af 100644
--- a/src/textprop.c
+++ b/src/textprop.c
@@ -613,7 +613,7 @@ get_char_property_and_overlay (Lisp_Object position, register Lisp_Object prop,
     }
   if (BUFFERP (object))
     {
-      int noverlays;
+      ptrdiff_t noverlays;
       Lisp_Object *overlay_vec;
       struct buffer *obuf = current_buffer;
 
diff --git a/src/xdisp.c b/src/xdisp.c
index ae5c334447..d04ceddacb 100644
--- a/src/xdisp.c
+++ b/src/xdisp.c
@@ -3062,10 +3062,9 @@ compute_stop_pos (struct it *it)
 static EMACS_INT
 next_overlay_change (EMACS_INT pos)
 {
-  int noverlays;
+  ptrdiff_t i, noverlays;
   EMACS_INT endpos;
   Lisp_Object *overlays;
-  int i;
 
   /* Get all overlays at the given position.  */
   GET_OVERLAYS_AT (pos, overlays, noverlays, &endpos, 1);
@@ -25453,13 +25452,13 @@ note_mouse_highlight (struct frame *f, int x, int y)
       && XFASTINT (w->last_modified) == BUF_MODIFF (b)
       && XFASTINT (w->last_overlay_modified) == BUF_OVERLAY_MODIFF (b))
     {
-      int hpos, vpos, i, dx, dy, area;
+      int hpos, vpos, dx, dy, area;
       EMACS_INT pos;
       struct glyph *glyph;
       Lisp_Object object;
       Lisp_Object mouse_face = Qnil, position;
       Lisp_Object *overlay_vec = NULL;
-      int noverlays;
+      ptrdiff_t i, noverlays;
       struct buffer *obuf;
       EMACS_INT obegv, ozv;
       int same_region;
diff --git a/src/xfaces.c b/src/xfaces.c
index 78ea913526..951cf69a4c 100644
--- a/src/xfaces.c
+++ b/src/xfaces.c
@@ -5934,7 +5934,7 @@ face_at_buffer_position (struct window *w, EMACS_INT pos,
   struct frame *f = XFRAME (w->frame);
   Lisp_Object attrs[LFACE_VECTOR_SIZE];
   Lisp_Object prop, position;
-  int i, noverlays;
+  ptrdiff_t i, noverlays;
   Lisp_Object *overlay_vec;
   Lisp_Object frame;
   EMACS_INT endpos;