Don't needlessly convert to sal_Int32 instead of long

...(where the tools::Rectangle ctor takes arguments of type long), as that
caused CppunitTest_sc_jumbosheets_test ScFiltersTest::testTdf134392 to fail now
with UBSan with float-cast-overflow at

> svx/source/sdr/overlay/overlaymanager.cxx:294:44: runtime error: 7.22475e+09 is outside the range of representable values of type 'int'
>  #0 in sdr::overlay::OverlayManager::RangeToInvalidateRectangle(basegfx::B2DRange const&) const at svx/source/sdr/overlay/overlaymanager.cxx:294:44 (instdir/program/libsvxcorelo.so +0x5477e54)
>  #1 in sdr::overlay::OverlayManager::invalidateRange(basegfx::B2DRange const&) at svx/source/sdr/overlay/overlaymanager.cxx:314:55 (instdir/program/libsvxcorelo.so +0x5478773)
>  #2 in sdr::overlay::OverlayManager::impApplyRemoveActions(sdr::overlay::OverlayObject&) at svx/source/sdr/overlay/overlaymanager.cxx:186:13 (instdir/program/libsvxcorelo.so +0x5475566)
>  #3 in sdr::overlay::OverlayManager::~OverlayManager() at svx/source/sdr/overlay/overlaymanager.cxx:224:21 (instdir/program/libsvxcorelo.so +0x547627c)
>  #4 in sdr::overlay::OverlayManagerBuffered::~OverlayManagerBuffered() at svx/source/sdr/overlay/overlaymanagerbuffered.cxx:377:9 (instdir/program/libsvxcorelo.so +0x5423e27)
>  #5 in sdr::overlay::OverlayManagerBuffered::~OverlayManagerBuffered() at svx/source/sdr/overlay/overlaymanagerbuffered.cxx:368:9 (instdir/program/libsvxcorelo.so +0x5423ebb)
>  #6 in salhelper::SimpleReferenceObject::release() at include/salhelper/simplereferenceobject.hxx:72:49 (instdir/program/libsvxcorelo.so +0x4b748a8)
>  #7 in rtl::Reference<sdr::overlay::OverlayManager>::clear() at include/rtl/ref.hxx:180:19 (instdir/program/libsvxcorelo.so +0x56c643e)
>  #8 in SdrPaintWindow::~SdrPaintWindow() at svx/source/svdraw/sdrpaintwindow.cxx:251:22 (instdir/program/libsvxcorelo.so +0x56c3419)
>  #9 in std::default_delete<SdrPaintWindow>::operator()(SdrPaintWindow*) const at ~/gcc/trunk/inst/lib/gcc/x86_64-pc-linux-gnu/11.0.0/../../../../include/c++/11.0.0/bits/unique_ptr.h:85:2 (instdir/program/libsvxcorelo.so +0x64932ff)
>  #10 in std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> >::~unique_ptr() at ~/gcc/trunk/inst/lib/gcc/x86_64-pc-linux-gnu/11.0.0/../../../../include/c++/11.0.0/bits/unique_ptr.h:361:4 (instdir/program/libsvxcorelo.so +0x647f81f)
>  #11 in void std::destroy_at<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> > >(std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> >*) at ~/gcc/trunk/inst/lib/gcc/x86_64-pc-linux-gnu/11.0.0/../../../../include/c++/11.0.0/bits/stl_construct.h:88:15 (instdir/program/libsvxcorelo.so +0x6487c73)
>  #12 in void std::allocator_traits<std::allocator<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> > > >::destroy<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> > >(std::allocator<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> > >&, std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> >*) at ~/gcc/trunk/inst/lib/gcc/x86_64-pc-linux-gnu/11.0.0/../../../../include/c++/11.0.0/bits/alloc_traits.h:533:4 (instdir/program/libsvxcorelo.so +0x648fb80)
>  #13 in std::__cxx1998::vector<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> >, std::allocator<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> > > >::_M_erase(__gnu_cxx::__normal_iterator<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> >*, std::__cxx1998::vector<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> >, std::allocator<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> > > > >) at ~/gcc/trunk/inst/lib/gcc/x86_64-pc-linux-gnu/11.0.0/../../../../include/c++/11.0.0/bits/vector.tcc:177:7 (instdir/program/libsvxcorelo.so +0x648ea5f)
>  #14 in std::__cxx1998::vector<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> >, std::allocator<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> > > >::erase(__gnu_cxx::__normal_iterator<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> > const*, std::__cxx1998::vector<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> >, std::allocator<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> > > > >) at ~/gcc/trunk/inst/lib/gcc/x86_64-pc-linux-gnu/11.0.0/../../../../include/c++/11.0.0/bits/stl_vector.h:1431:16 (instdir/program/libsvxcorelo.so +0x648dea0)
>  #15 in std::__debug::vector<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> >, std::allocator<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> > > >::erase(__gnu_debug::_Safe_iterator<__gnu_cxx::__normal_iterator<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> > const*, std::__cxx1998::vector<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> >, std::allocator<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> > > > >, std::__debug::vector<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> >, std::allocator<std::unique_ptr<SdrPaintWindow, std::default_delete<SdrPaintWindow> > > >, std::random_access_iterator_tag>) at ~/gcc/trunk/inst/lib/gcc/x86_64-pc-linux-gnu/11.0.0/../../../../include/c++/11.0.0/debug/vector:675:32 (instdir/program/libsvxcorelo.so +0x647dc34)
>  #16 in SdrPaintView::DeletePaintWindow(SdrPaintWindow&) at svx/source/svdraw/svdpntv.cxx:83:24 (instdir/program/libsvxcorelo.so +0x645cf57)
>  #17 in SdrPaintView::DeleteWindowFromPaintView(OutputDevice*) at svx/source/svdraw/svdpntv.cxx:404:9 (instdir/program/libsvxcorelo.so +0x64682ca)
>  #18 in SdrObjEditView::DeleteWindowFromPaintView(OutputDevice*) at svx/source/svdraw/svdedxv.cxx:2324:22 (instdir/program/libsvxcorelo.so +0x59aac14)
>  #19 in FmFormView::DeleteWindowFromPaintView(OutputDevice*) at svx/source/form/fmview.cxx:196:14 (instdir/program/libsvxcorelo.so +0x7a55634)
>  #20 in ScTabView::~ScTabView() at sc/source/ui/view/tabview5.cxx:187:28 (instdir/program/libsclo.so +0x1429430f)
>  #21 in ScViewFunc::~ScViewFunc() at sc/source/ui/view/viewfunc.cxx:100:1 (instdir/program/libsclo.so +0x14657aeb)
>  #22 in ScDBFunc::~ScDBFunc() at sc/source/ui/view/dbfunc.cxx:49:1 (instdir/program/libsclo.so +0x13a72262)
>  #23 in ScTabViewShell::~ScTabViewShell() at sc/source/ui/view/tabvwsh4.cxx:1794:1 (instdir/program/libsclo.so +0x14314d5b)
>  #24 in ScTabViewShell::~ScTabViewShell() at sc/source/ui/view/tabvwsh4.cxx:1743:1 (instdir/program/libsclo.so +0x143150fb)
>  #25 in SfxViewFrame::ReleaseObjectShell_Impl() at sfx2/source/view/viewfrm.cxx:1113:9 (instdir/program/libsfxlo.so +0x5b8ee8c)
>  #26 in SfxViewFrame::~SfxViewFrame() at sfx2/source/view/viewfrm.cxx:1647:5 (instdir/program/libsfxlo.so +0x5ba8d5a)
>  #27 in SfxViewFrame::~SfxViewFrame() at sfx2/source/view/viewfrm.cxx:1641:1 (instdir/program/libsfxlo.so +0x5baa88b)
>  #28 in SfxViewFrame::Close() at sfx2/source/view/viewfrm.cxx:1165:5 (instdir/program/libsfxlo.so +0x5b9256d)
>  #29 in SfxFrame::DoClose_Impl() at sfx2/source/view/frame.cxx:142:35 (instdir/program/libsfxlo.so +0x5a190e1)
>  #30 in SfxBaseController::dispose() at sfx2/source/view/sfxbasecontroller.cxx:982:28 (instdir/program/libsfxlo.so +0x5b181ce)
>  #31 in (anonymous namespace)::XFrameImpl::setComponent(com::sun::star::uno::Reference<com::sun::star::awt::XWindow> const&, com::sun::star::uno::Reference<com::sun::star::frame::XController> const&) at framework/source/services/frame.cxx:1492:33 (instdir/program/libfwklo.so +0x2c4a8de)
>  #32 in (anonymous namespace)::XFrameImpl::close(unsigned char) at framework/source/services/frame.cxx:1699:12 (instdir/program/libfwklo.so +0x2c55b57)
>  #33 in SfxFrame::DoClose() at sfx2/source/view/frame.cxx:108:29 (instdir/program/libsfxlo.so +0x5a163b8)
>  #34 in SfxViewFrame::Notify(SfxBroadcaster&, SfxHint const&) at sfx2/source/view/viewfrm.cxx:1519:28 (instdir/program/libsfxlo.so +0x5ba0b74)
>  #35 in SfxBroadcaster::Broadcast(SfxHint const&) at svl/source/notify/SfxBroadcaster.cxx:49:24 (instdir/program/libsvllo.so +0x17ef9c4)
>  #36 in (anonymous namespace)::SfxModelListener_Impl::notifyClosing(com::sun::star::lang::EventObject const&) at sfx2/source/doc/objxtor.cxx:146:12 (instdir/program/libsfxlo.so +0x515538b)
>  #37 in SfxBaseModel::close(unsigned char) at sfx2/source/doc/sfxbasemodel.cxx:1439:76 (instdir/program/libsfxlo.so +0x5285e3c)
>  #38 in SfxBaseModel::dispose() at sfx2/source/doc/sfxbasemodel.cxx:716:13 (instdir/program/libsfxlo.so +0x5266b5f)
>  #39 in ScFiltersTest::tearDown() at sc/qa/unit/jumbosheets-test.cxx:141:74 (workdir/LinkTarget/CppunitTest/libtest_sc_jumbosheets_test.so +0x7bc6b)

This appears to date back to 88879a0d12fba95825b7f99f348094039ba3e029
"INTEGRATION: CWS aw051 (1.3.188); FILE MERGED: 2007/05/29 14:12:36 aw
1.3.188.1: #i77674#"

> --- a/svx/source/sdr/overlay/overlaymanager.cxx
> +++ b/svx/source/sdr/overlay/overlaymanager.cxx
> @@ -264,12 +264,11 @@ namespace sdr
>          {
>              if(OUTDEV_WINDOW == getOutputDevice().GetOutDevType())
>              {
> -                // transform to rectangle
> -                const basegfx::B2DPoint aMinimum(rRange.getMinimum());
> -                const basegfx::B2DPoint aMaximum(rRange.getMaximum());
> +                // #i77674# transform to rectangle. Use floor/ceil to get all covered
> +                // discrete pixels, see #i75163# and OverlayManagerBuffered::invalidateRange
>                  const Rectangle aInvalidateRectangle(
> -                    FRound(aMinimum.getX()), FRound(aMinimum.getY()),
> -                    FRound(aMaximum.getX()), FRound(aMaximum.getY()));
> +                    (sal_Int32)floor(rRange.getMinX()), (sal_Int32)floor(rRange.getMinY()),
> +                    (sal_Int32)ceil(rRange.getMaxX()), (sal_Int32)ceil(rRange.getMaxY()));
>
>                  // simply invalidate
>                  ((Window&)getOutputDevice()).Invalidate(aInvalidateRectangle, INVALIDATE_NOERASE);

but which probably used sal_Int32 instead of long for no particular reason.

Change-Id: I29459ab6c5cd06cc7d786b239221343f8de5170c
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/104096
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>

1 files changed, 4 insertions, 4 deletions

diff --git a/svx/source/sdr/overlay/overlaymanager.cxx b/svx/source/sdr/overlay/overlaymanager.cxx
index 73e6814f8195..f99e30ab18a6 100644
--- a/svx/source/sdr/overlay/overlaymanager.cxx
+++ b/svx/source/sdr/overlay/overlaymanager.cxx
@@ -290,10 +290,10 @@ namespace sdr::overlay
                 // assume AA needs one pixel more and invalidate one pixel more
                 const double fDiscreteOne(getDiscreteOne());
                 const tools::Rectangle aInvalidateRectangle(
-                    static_cast<sal_Int32>(floor(rRange.getMinX() - fDiscreteOne)),
-                    static_cast<sal_Int32>(floor(rRange.getMinY() - fDiscreteOne)),
-                    static_cast<sal_Int32>(ceil(rRange.getMaxX() + fDiscreteOne)),
-                    static_cast<sal_Int32>(ceil(rRange.getMaxY() + fDiscreteOne)));
+                    static_cast<long>(floor(rRange.getMinX() - fDiscreteOne)),
+                    static_cast<long>(floor(rRange.getMinY() - fDiscreteOne)),
+                    static_cast<long>(ceil(rRange.getMaxX() + fDiscreteOne)),
+                    static_cast<long>(ceil(rRange.getMaxY() + fDiscreteOne)));
                 return aInvalidateRectangle;
             }
             else