media: platform: mtk-mdp3: fix potential frame size overflow in mdp_try_fmt_mplane()
[ Upstream commit 4168720753ce6c14c5d3a35302fc2e1841383443 ]
Fix overflow risk when setting certain formats whose frame size exceeds
a RGB24 with 7723x7723 resolution.
For example, a 7723x7724 RGB24 frame:
1. bpl (byte per line) = 7723 * 3.
2. Overflow occurs when bpl * 7724 * depth.
Fixes: 61890ccaef
("media: platform: mtk-mdp3: add MediaTek MDP3 driver")
Signed-off-by: Moudy Ho <moudy.ho@mediatek.com>
Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
Signed-off-by: Sasha Levin <sashal@kernel.org>
This commit is contained in:
parent
5a72aea9ac
commit
affad9e791
@ -4,6 +4,7 @@
|
||||
* Author: Ping-Hsun Wu <ping-hsun.wu@mediatek.com>
|
||||
*/
|
||||
|
||||
#include <linux/math64.h>
|
||||
#include <media/v4l2-common.h>
|
||||
#include <media/videobuf2-v4l2.h>
|
||||
#include <media/videobuf2-dma-contig.h>
|
||||
@ -428,14 +429,15 @@ const struct mdp_format *mdp_try_fmt_mplane(struct v4l2_format *f,
|
||||
u32 bpl = pix_mp->plane_fmt[i].bytesperline;
|
||||
u32 min_si, max_si;
|
||||
u32 si = pix_mp->plane_fmt[i].sizeimage;
|
||||
u64 di;
|
||||
|
||||
bpl = clamp(bpl, min_bpl, max_bpl);
|
||||
pix_mp->plane_fmt[i].bytesperline = bpl;
|
||||
|
||||
min_si = (bpl * pix_mp->height * fmt->depth[i]) /
|
||||
fmt->row_depth[i];
|
||||
max_si = (bpl * s.max_height * fmt->depth[i]) /
|
||||
fmt->row_depth[i];
|
||||
di = (u64)bpl * pix_mp->height * fmt->depth[i];
|
||||
min_si = (u32)div_u64(di, fmt->row_depth[i]);
|
||||
di = (u64)bpl * s.max_height * fmt->depth[i];
|
||||
max_si = (u32)div_u64(di, fmt->row_depth[i]);
|
||||
|
||||
si = clamp(si, min_si, max_si);
|
||||
pix_mp->plane_fmt[i].sizeimage = si;
|
||||
|
Loading…
Reference in New Issue
Block a user