xf86drm: fix integer overflow in allocations

drmAllocCpy() and many resource-fetching functions took kernel-returned
counts as int or __u32, multiplied them by sizeof(...) as ints, and
passed the product to drmMalloc(int size). On LP64 systems this
truncates any size_t result down to a signed 32-bit quantity. Products
at or above 2^31 become negative and are sign-extended when passed to
calloc(), so the allocation either fails (yielding a NULL that is
subsequently used as a write target) or, combined with an overflowing
multiply, succeeds with an undersized buffer.

drmAllocCpy() compounded this: it multiplied count and entry_size as
ints before handing the result to drmMalloc(), and iterated with an int
index. A large count could wrap the product to a small positive value,
producing an undersized allocation filled by a full-count memcpy.

Fix by replacing drmMalloc(count * sizeof(T)) with calloc(count,
sizeof(T)) at all call sites where count originates from the kernel.
calloc's two-argument form performs the multiplication in size_t and
rejects overflowing products, and drmAllocCpy now takes size_t
parameters and iterates with a size_t index so all arithmetic stays in
an unsigned 64-bit domain on LP64.

Affected functions: drmAllocCpy, drmModeGetResources,
drmModeGetConnector, drmModeGetProperty, drmModeObjectGetProperties,
drmModeAtomicDuplicate, drmModeAtomicCommit, drmMarkBufs, drmGetBufInfo,
drmMapBufs, drmDMA, drmGetReservedContextList.

Signed-off-by: Mario Limonciello <mario.limonciello@amd.com>

2 files changed, 29 insertions, 31 deletions

diff --git a/xf86drm.c b/xf86drm.c
index 16eda782..a8b39a31 100644
--- a/xf86drm.c
+++ b/xf86drm.c
@@ -1690,7 +1690,7 @@ drm_public int drmMarkBufs(int fd, double low, double high)
     if (!info.count)
         return -EINVAL;
 
-    if (!(info.list = drmMalloc(info.count * sizeof(*info.list))))
+    if (!(info.list = calloc(info.count, sizeof(*info.list))))
         return -ENOMEM;
 
     if (drmIoctl(fd, DRM_IOCTL_INFO_BUFS, &info)) {
@@ -1828,7 +1828,7 @@ drm_public drmBufInfoPtr drmGetBufInfo(int fd)
         return NULL;
 
     if (info.count) {
-        if (!(info.list = drmMalloc(info.count * sizeof(*info.list))))
+        if (!(info.list = calloc(info.count, sizeof(*info.list))))
             return NULL;
 
         if (drmIoctl(fd, DRM_IOCTL_INFO_BUFS, &info)) {
@@ -1838,7 +1838,7 @@ drm_public drmBufInfoPtr drmGetBufInfo(int fd)
 
         retval = drmMalloc(sizeof(*retval));
         retval->count = info.count;
-        if (!(retval->list = drmMalloc(info.count * sizeof(*retval->list)))) {
+        if (!(retval->list = calloc(info.count, sizeof(*retval->list)))) {
                 drmFree(retval);
                 drmFree(info.list);
                 return NULL;
@@ -1884,7 +1884,7 @@ drm_public drmBufMapPtr drmMapBufs(int fd)
     if (!bufs.count)
         return NULL;
 
-    if (!(bufs.list = drmMalloc(bufs.count * sizeof(*bufs.list))))
+    if (!(bufs.list = calloc(bufs.count, sizeof(*bufs.list))))
         return NULL;
 
     if (drmIoctl(fd, DRM_IOCTL_MAP_BUFS, &bufs)) {
@@ -1894,7 +1894,7 @@ drm_public drmBufMapPtr drmMapBufs(int fd)
 
     retval = drmMalloc(sizeof(*retval));
     retval->count = bufs.count;
-    retval->list  = drmMalloc(bufs.count * sizeof(*retval->list));
+    retval->list  = calloc(bufs.count, sizeof(*retval->list));
     for (i = 0; i < bufs.count; i++) {
         retval->list[i].idx     = bufs.list[i].idx;
         retval->list[i].total   = bufs.list[i].total;
@@ -2041,9 +2041,9 @@ drm_public drm_context_t *drmGetReservedContextList(int fd, int *count)
     if (!res.count)
         return NULL;
 
-    if (!(list   = drmMalloc(res.count * sizeof(*list))))
+    if (!(list   = calloc(res.count, sizeof(*list))))
         return NULL;
-    if (!(retval = drmMalloc(res.count * sizeof(*retval))))
+    if (!(retval = calloc(res.count, sizeof(*retval))))
         goto err_free_list;
 
     res.contexts = list;
diff --git a/xf86drmMode.c b/xf86drmMode.c
index 950b9843..a456e411 100644
--- a/xf86drmMode.c
+++ b/xf86drmMode.c
@@ -72,15 +72,15 @@ static inline int DRM_IOCTL(int fd, unsigned long cmd, void *arg)
  * Util functions
  */
 
-static void* drmAllocCpy(char *array, int count, int entry_size)
+static void* drmAllocCpy(char *array, size_t count, size_t entry_size)
 {
 	char *r;
-	int i;
+	size_t i;
 
 	if (!count || !array || !entry_size)
 		return 0;
 
-	if (!(r = drmMalloc(count*entry_size)))
+	if (!(r = calloc(count, entry_size)))
 		return 0;
 
 	for (i = 0; i < count; i++)
@@ -174,22 +174,22 @@ retry:
 	counts = res;
 
 	if (res.count_fbs) {
-		res.fb_id_ptr = VOID2U64(drmMalloc(res.count_fbs*sizeof(uint32_t)));
+		res.fb_id_ptr = VOID2U64(calloc(res.count_fbs, sizeof(uint32_t)));
 		if (!res.fb_id_ptr)
 			goto err_allocs;
 	}
 	if (res.count_crtcs) {
-		res.crtc_id_ptr = VOID2U64(drmMalloc(res.count_crtcs*sizeof(uint32_t)));
+		res.crtc_id_ptr = VOID2U64(calloc(res.count_crtcs, sizeof(uint32_t)));
 		if (!res.crtc_id_ptr)
 			goto err_allocs;
 	}
 	if (res.count_connectors) {
-		res.connector_id_ptr = VOID2U64(drmMalloc(res.count_connectors*sizeof(uint32_t)));
+		res.connector_id_ptr = VOID2U64(calloc(res.count_connectors, sizeof(uint32_t)));
 		if (!res.connector_id_ptr)
 			goto err_allocs;
 	}
 	if (res.count_encoders) {
-		res.encoder_id_ptr = VOID2U64(drmMalloc(res.count_encoders*sizeof(uint32_t)));
+		res.encoder_id_ptr = VOID2U64(calloc(res.count_encoders, sizeof(uint32_t)));
 		if (!res.encoder_id_ptr)
 			goto err_allocs;
 	}
@@ -525,16 +525,16 @@ retry:
 	counts = conn;
 
 	if (conn.count_props) {
-		conn.props_ptr = VOID2U64(drmMalloc(conn.count_props*sizeof(uint32_t)));
+		conn.props_ptr = VOID2U64(calloc(conn.count_props, sizeof(uint32_t)));
 		if (!conn.props_ptr)
 			goto err_allocs;
-		conn.prop_values_ptr = VOID2U64(drmMalloc(conn.count_props*sizeof(uint64_t)));
+		conn.prop_values_ptr = VOID2U64(calloc(conn.count_props, sizeof(uint64_t)));
 		if (!conn.prop_values_ptr)
 			goto err_allocs;
 	}
 
 	if (conn.count_modes) {
-		conn.modes_ptr = VOID2U64(drmMalloc(conn.count_modes*sizeof(struct drm_mode_modeinfo)));
+		conn.modes_ptr = VOID2U64(calloc(conn.count_modes, sizeof(struct drm_mode_modeinfo)));
 		if (!conn.modes_ptr)
 			goto err_allocs;
 	} else {
@@ -543,7 +543,7 @@ retry:
 	}
 
 	if (conn.count_encoders) {
-		conn.encoders_ptr = VOID2U64(drmMalloc(conn.count_encoders*sizeof(uint32_t)));
+		conn.encoders_ptr = VOID2U64(calloc(conn.count_encoders, sizeof(uint32_t)));
 		if (!conn.encoders_ptr)
 			goto err_allocs;
 	}
@@ -681,14 +681,14 @@ drm_public drmModePropertyPtr drmModeGetProperty(int fd, uint32_t property_id)
 		return 0;
 
 	if (prop.count_values)
-		prop.values_ptr = VOID2U64(drmMalloc(prop.count_values * sizeof(uint64_t)));
+		prop.values_ptr = VOID2U64(calloc(prop.count_values, sizeof(uint64_t)));
 
 	if (prop.count_enum_blobs && (prop.flags & (DRM_MODE_PROP_ENUM | DRM_MODE_PROP_BITMASK)))
-		prop.enum_blob_ptr = VOID2U64(drmMalloc(prop.count_enum_blobs * sizeof(struct drm_mode_property_enum)));
+		prop.enum_blob_ptr = VOID2U64(calloc(prop.count_enum_blobs, sizeof(struct drm_mode_property_enum)));
 
 	if (prop.count_enum_blobs && (prop.flags & DRM_MODE_PROP_BLOB)) {
-		prop.values_ptr = VOID2U64(drmMalloc(prop.count_enum_blobs * sizeof(uint32_t)));
-		prop.enum_blob_ptr = VOID2U64(drmMalloc(prop.count_enum_blobs * sizeof(uint32_t)));
+		prop.values_ptr = VOID2U64(calloc(prop.count_enum_blobs, sizeof(uint32_t)));
+		prop.enum_blob_ptr = VOID2U64(calloc(prop.count_enum_blobs, sizeof(uint32_t)));
 	}
 
 	if (drmIoctl(fd, DRM_IOCTL_MODE_GETPROPERTY, &prop)) {
@@ -1290,12 +1290,10 @@ retry:
 	count = properties.count_props;
 
 	if (count) {
-		properties.props_ptr = VOID2U64(drmMalloc(count *
-							  sizeof(uint32_t)));
+		properties.props_ptr = VOID2U64(calloc(count, sizeof(uint32_t)));
 		if (!properties.props_ptr)
 			goto err_allocs;
-		properties.prop_values_ptr = VOID2U64(drmMalloc(count *
-						      sizeof(uint64_t)));
+		properties.prop_values_ptr = VOID2U64(calloc(count, sizeof(uint64_t)));
 		if (!properties.prop_values_ptr)
 			goto err_allocs;
 	}
@@ -1400,7 +1398,7 @@ drm_public drmModeAtomicReqPtr drmModeAtomicDuplicate(const drmModeAtomicReqPtr
 	new->size_items = old->size_items;
 
 	if (old->size_items) {
-		new->items = drmMalloc(old->size_items * sizeof(*new->items));
+		new->items = calloc(old->size_items, sizeof(*new->items));
 		if (!new->items) {
 			free(new);
 			return NULL;
@@ -1569,25 +1567,25 @@ drm_public int drmModeAtomicCommit(int fd, const drmModeAtomicReqPtr req,
 	for (i = 0; i < sorted->cursor; i++)
 		sorted->items[i].cursor = i;
 
-	objs_ptr = drmMalloc(atomic.count_objs * sizeof objs_ptr[0]);
+	objs_ptr = calloc(atomic.count_objs, sizeof objs_ptr[0]);
 	if (!objs_ptr) {
 		errno = ENOMEM;
 		goto out;
 	}
 
-	count_props_ptr = drmMalloc(atomic.count_objs * sizeof count_props_ptr[0]);
+	count_props_ptr = calloc(atomic.count_objs, sizeof count_props_ptr[0]);
 	if (!count_props_ptr) {
 		errno = ENOMEM;
 		goto out;
 	}
 
-	props_ptr = drmMalloc(sorted->cursor * sizeof props_ptr[0]);
+	props_ptr = calloc(sorted->cursor, sizeof props_ptr[0]);
 	if (!props_ptr) {
 		errno = ENOMEM;
 		goto out;
 	}
 
-	prop_values_ptr = drmMalloc(sorted->cursor * sizeof prop_values_ptr[0]);
+	prop_values_ptr = calloc(sorted->cursor, sizeof prop_values_ptr[0]);
 	if (!prop_values_ptr) {
 		errno = ENOMEM;
 		goto out;