* gtkutil.c: Integer overflow fixes.

(get_utf8_string, xg_store_widget_in_map):
Check for size-calculation overflow.
(get_utf8_string): Use ptrdiff_t, not size_t, where either will
do, as we prefer signed integers.
(id_to_widget.max_size, id_to_widget.used)
(xg_store_widget_in_map, xg_remove_widget_from_map)
(xg_get_widget_from_map, xg_get_scroll_id_for_window)
(xg_remove_scroll_bar, xg_update_scrollbar_pos):
Use and return ptrdiff_t, not int.
(xg_gtk_scroll_destroy): Don't assume ptrdiff_t fits in int.
* gtkutil.h: Change prototypes to match the above.

3 files changed, 39 insertions, 18 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index 84d7bf4cb4..b984072c6e 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,5 +1,18 @@
 2011-07-28  Paul Eggert  <[email protected]>
 
+	* gtkutil.c: Integer overflow fixes.
+	(get_utf8_string, xg_store_widget_in_map):
+	Check for size-calculation overflow.
+	(get_utf8_string): Use ptrdiff_t, not size_t, where either will
+	do, as we prefer signed integers.
+	(id_to_widget.max_size, id_to_widget.used)
+	(xg_store_widget_in_map, xg_remove_widget_from_map)
+	(xg_get_widget_from_map, xg_get_scroll_id_for_window)
+	(xg_remove_scroll_bar, xg_update_scrollbar_pos):
+	Use and return ptrdiff_t, not int.
+	(xg_gtk_scroll_destroy): Don't assume ptrdiff_t fits in int.
+	* gtkutil.h: Change prototypes to match the above.
+
 	* ftfont.c: Check for size overflow.
 	(ftfont_get_open_type_spec, setup_otf_gstring, ftfont_shape_by_flt):
 	Check for integer overflow in size calculations.
diff --git a/src/gtkutil.c b/src/gtkutil.c
index 70bc18a75f..f56e888e68 100644
--- a/src/gtkutil.c
+++ b/src/gtkutil.c
@@ -487,7 +487,8 @@ get_utf8_string (const char *str)
   if (!utf8_str)
     {
       /* Probably some control characters in str.  Escape them. */
-      size_t nr_bad = 0;
+      ptrdiff_t len;
+      ptrdiff_t nr_bad = 0;
       gsize bytes_read;
       gsize bytes_written;
       unsigned char *p = (unsigned char *)str;
@@ -511,7 +512,10 @@ get_utf8_string (const char *str)
         }
       if (cp) g_free (cp);
 
-      up = utf8_str = xmalloc (strlen (str) + nr_bad * 4 + 1);
+      len = strlen (str);
+      if ((min (PTRDIFF_MAX, SIZE_MAX) - len - 1) / 4 < nr_bad)
+	memory_full (SIZE_MAX);
+      up = utf8_str = xmalloc (len + nr_bad * 4 + 1);
       p = (unsigned char *)str;
 
       while (! (cp = g_locale_to_utf8 ((char *)p, -1, &bytes_read,
@@ -3296,8 +3300,8 @@ static int scroll_bar_width_for_theme;
 static struct
 {
   GtkWidget **widgets;
-  int max_size;
-  int used;
+  ptrdiff_t max_size;
+  ptrdiff_t used;
 } id_to_widget;
 
 /* Grow this much every time we need to allocate more  */
@@ -3306,15 +3310,20 @@ static struct
 
 /* Store the widget pointer W in id_to_widget and return the integer index.  */
 
-static int
+static ptrdiff_t
 xg_store_widget_in_map (GtkWidget *w)
 {
-  int i;
+  ptrdiff_t i;
 
   if (id_to_widget.max_size == id_to_widget.used)
     {
-      int new_size = id_to_widget.max_size + ID_TO_WIDGET_INCR;
+      ptrdiff_t new_size;
+      ptrdiff_t lim = min (TYPE_MAXIMUM (Window),
+			   min (PTRDIFF_MAX, SIZE_MAX) / sizeof (GtkWidget *));
+      if (lim - ID_TO_WIDGET_INCR < id_to_widget.max_size)
+	memory_full (SIZE_MAX);
 
+      new_size = id_to_widget.max_size + ID_TO_WIDGET_INCR;
       id_to_widget.widgets = xrealloc (id_to_widget.widgets,
                                        sizeof (GtkWidget *)*new_size);
 
@@ -3345,7 +3354,7 @@ xg_store_widget_in_map (GtkWidget *w)
    Called when scroll bar is destroyed.  */
 
 static void
-xg_remove_widget_from_map (int idx)
+xg_remove_widget_from_map (ptrdiff_t idx)
 {
   if (idx < id_to_widget.max_size && id_to_widget.widgets[idx] != 0)
     {
@@ -3357,7 +3366,7 @@ xg_remove_widget_from_map (int idx)
 /* Get the widget pointer at IDX from id_to_widget. */
 
 static GtkWidget *
-xg_get_widget_from_map (int idx)
+xg_get_widget_from_map (ptrdiff_t idx)
 {
   if (idx < id_to_widget.max_size && id_to_widget.widgets[idx] != 0)
     return id_to_widget.widgets[idx];
@@ -3396,10 +3405,10 @@ xg_get_default_scrollbar_width (void)
 /* Return the scrollbar id for X Window WID on display DPY.
    Return -1 if WID not in id_to_widget.  */
 
-int
+ptrdiff_t
 xg_get_scroll_id_for_window (Display *dpy, Window wid)
 {
-  int idx;
+  ptrdiff_t idx;
   GtkWidget *w;
 
   w = xg_win_to_widget (dpy, wid);
@@ -3421,7 +3430,7 @@ xg_get_scroll_id_for_window (Display *dpy, Window wid)
 static void
 xg_gtk_scroll_destroy (GtkWidget *widget, gpointer data)
 {
-  int id = (intptr_t) data;
+  intptr_t id = (intptr_t) data;
   xg_remove_widget_from_map (id);
 }
 
@@ -3496,7 +3505,7 @@ xg_create_scroll_bar (FRAME_PTR f,
 /* Remove the scroll bar represented by SCROLLBAR_ID from the frame F.  */
 
 void
-xg_remove_scroll_bar (FRAME_PTR f, int scrollbar_id)
+xg_remove_scroll_bar (FRAME_PTR f, ptrdiff_t scrollbar_id)
 {
   GtkWidget *w = xg_get_widget_from_map (scrollbar_id);
   if (w)
@@ -3515,7 +3524,7 @@ xg_remove_scroll_bar (FRAME_PTR f, int scrollbar_id)
 
 void
 xg_update_scrollbar_pos (FRAME_PTR f,
-                         int scrollbar_id,
+                         ptrdiff_t scrollbar_id,
                          int top,
                          int left,
                          int width,
diff --git a/src/gtkutil.h b/src/gtkutil.h
index 769e56da91..2dfb3a5ed6 100644
--- a/src/gtkutil.h
+++ b/src/gtkutil.h
@@ -114,17 +114,17 @@ extern int xg_event_is_for_menubar (FRAME_PTR f, XEvent *event);
 
 extern int xg_have_tear_offs (void);
 
-extern int xg_get_scroll_id_for_window (Display *dpy, Window wid);
+extern ptrdiff_t xg_get_scroll_id_for_window (Display *dpy, Window wid);
 
 extern void xg_create_scroll_bar (FRAME_PTR f,
                                   struct scroll_bar *bar,
                                   GCallback scroll_callback,
                                   GCallback end_callback,
                                   const char *scroll_bar_name);
-extern void xg_remove_scroll_bar (FRAME_PTR f, int scrollbar_id);
+extern void xg_remove_scroll_bar (FRAME_PTR f, ptrdiff_t scrollbar_id);
 
 extern void xg_update_scrollbar_pos (FRAME_PTR f,
-                                     int scrollbar_id,
+                                     ptrdiff_t scrollbar_id,
                                      int top,
                                      int left,
                                      int width,
@@ -185,4 +185,3 @@ extern int xg_ignore_gtk_scrollbar;
 
 #endif /* USE_GTK */
 #endif /* GTKUTIL_H */
-