minigbm: add more error handling especially for oom

This change also fixes a potential prime_fd leak in mediatek backend.

BUG=b:201110412
TEST=CQ

Change-Id: Ia3e10c94b536f83ecfb6580666103fe654bbc616
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/minigbm/+/3188852
Commit-Queue: Yiwei Zhang <zzyiwei@chromium.org>
Tested-by: Yiwei Zhang <zzyiwei@chromium.org>
Reviewed-by: Chia-I Wu <olv@google.com>

cros_gralloc/gralloc0/tests/gralloctest.c

@@ -262,6 +262,8 @@ static struct gralloctest_context *test_init_gralloc()
 	int err;
 	hw_module_t const *hw_module;
 	struct gralloctest_context *ctx = calloc(1, sizeof(*ctx));
+	if (!ctx)
+		return NULL;
 
 	err = hw_get_module(GRALLOC_HARDWARE_MODULE_ID, &hw_module);
 	if (err)

drv.c

@@ -508,6 +508,12 @@ void *drv_bo_map(struct bo *bo, const struct rectangle *rect, uint32_t map_flags
 	}
 
 	mapping.vma = calloc(1, sizeof(*mapping.vma));
+	if (!mapping.vma) {
+		*map_data = NULL;
+		pthread_mutex_unlock(&drv->mappings_lock);
+		return MAP_FAILED;
+	}
+
 	memcpy(mapping.vma->map_strides, bo->meta.strides, sizeof(mapping.vma->map_strides));
 	addr = bo->drv->backend->bo_map(bo, mapping.vma, plane, map_flags);
 	if (addr == MAP_FAILED) {

helpers_array.c

@@ -23,10 +23,17 @@ struct drv_array *drv_array_init(uint32_t item_size)
 	struct drv_array *array;
 
 	array = calloc(1, sizeof(*array));
+	if (!array)
+		return NULL;
 
 	/* Start with a power of 2 number of allocations. */
 	array->allocations = 2;
 	array->items = calloc(array->allocations, sizeof(*array->items));
+	if (!array->items) {
+		free(array);
+		return NULL;
+	}
+
 	array->item_size = item_size;
 	return array;
 }

mediatek.c

@@ -172,6 +172,7 @@ static void *mediatek_bo_map(struct bo *bo, struct vma *vma, size_t plane, uint3
 	int ret, prime_fd;
 	struct drm_mtk_gem_map_off gem_map = { 0 };
 	struct mediatek_private_map_data *priv;
+	void *addr = NULL;
 
 	gem_map.handle = bo->handles[0].u32;
 
@@ -187,22 +188,38 @@ static void *mediatek_bo_map(struct bo *bo, struct vma *vma, size_t plane, uint3
 		return MAP_FAILED;
 	}
 
-	void *addr = mmap(0, bo->meta.total_size, drv_get_prot(map_flags), MAP_SHARED, bo->drv->fd,
-			  gem_map.offset);
+	addr = mmap(0, bo->meta.total_size, drv_get_prot(map_flags), MAP_SHARED, bo->drv->fd,
+		    gem_map.offset);
+	if (addr == MAP_FAILED)
+		goto out_close_prime_fd;
 
 	vma->length = bo->meta.total_size;
 
 	priv = calloc(1, sizeof(*priv));
-	priv->prime_fd = prime_fd;
-	vma->priv = priv;
+	if (!priv)
+		goto out_unmap_addr;
 
 	if (bo->meta.use_flags & BO_USE_RENDERSCRIPT) {
 		priv->cached_addr = calloc(1, bo->meta.total_size);
+		if (!priv->cached_addr)
+			goto out_free_priv;
+
 		priv->gem_addr = addr;
 		addr = priv->cached_addr;
 	}
 
+	priv->prime_fd = prime_fd;
+	vma->priv = priv;
+
 	return addr;
+
+out_free_priv:
+	free(priv);
+out_unmap_addr:
+	munmap(addr, bo->meta.total_size);
+out_close_prime_fd:
+	close(prime_fd);
+	return MAP_FAILED;
 }
 
 static int mediatek_bo_unmap(struct bo *bo, struct vma *vma)

rockchip.c

@@ -184,6 +184,7 @@ static void *rockchip_bo_map(struct bo *bo, struct vma *vma, size_t plane, uint3
 	int ret;
 	struct rockchip_private_map_data *priv;
 	struct drm_rockchip_gem_map_off gem_map = { 0 };
+	void *addr = NULL;
 
 	/* We can only map buffers created with SW access flags, which should
 	 * have no modifiers (ie, not AFBC). */
@@ -197,20 +198,34 @@ static void *rockchip_bo_map(struct bo *bo, struct vma *vma, size_t plane, uint3
 		return MAP_FAILED;
 	}
 
-	void *addr = mmap(0, bo->meta.total_size, drv_get_prot(map_flags), MAP_SHARED, bo->drv->fd,
-			  gem_map.offset);
+	addr = mmap(0, bo->meta.total_size, drv_get_prot(map_flags), MAP_SHARED, bo->drv->fd,
+		    gem_map.offset);
+	if (addr == MAP_FAILED)
+		return MAP_FAILED;
 
 	vma->length = bo->meta.total_size;
 
 	if (bo->meta.use_flags & BO_USE_RENDERSCRIPT) {
 		priv = calloc(1, sizeof(*priv));
+		if (!priv)
+			goto out_unmap_addr;
+
 		priv->cached_addr = calloc(1, bo->meta.total_size);
+		if (!priv->cached_addr)
+			goto out_free_priv;
+
 		priv->gem_addr = addr;
 		vma->priv = priv;
 		addr = priv->cached_addr;
 	}
 
 	return addr;
+
+out_free_priv:
+	free(priv);
+out_unmap_addr:
+	munmap(addr, bo->meta.total_size);
+	return MAP_FAILED;
 }
 
 static int rockchip_bo_unmap(struct bo *bo, struct vma *vma)

virtgpu_cross_domain.c

@@ -243,7 +243,15 @@ static int cross_domain_init(struct driver *drv)
 		return -ENOTSUP;
 
 	priv = calloc(1, sizeof(*priv));
+	if (!priv)
+		return -ENOMEM;
+
 	priv->metadata_cache = drv_array_init(sizeof(struct bo_metadata));
+	if (!priv->metadata_cache) {
+		ret = -ENOMEM;
+		goto free_private;
+	}
+
 	priv->ring_addr = MAP_FAILED;
 	drv->priv = priv;
 

virtgpu_virgl.c

@@ -565,6 +565,9 @@ static int virgl_init(struct driver *drv)
 	struct virgl_priv *priv;
 
 	priv = calloc(1, sizeof(*priv));
+	if (!priv)
+		return -ENOMEM;
+
 	drv->priv = priv;
 
 	virgl_init_params_and_caps(drv);