path: root/src/atimach64.c

/* $XFree86: xc/programs/Xserver/hw/xfree86/drivers/ati/atimach64.c,v 1.51 2003/02/24 20:46:54 tsi Exp $ */
/*
 * Copyright 1997 through 2003 by Marc Aurele La France (TSI @ UQV), tsi@xfree86.org
 *
 * Permission to use, copy, modify, distribute, and sell this software and its
 * documentation for any purpose is hereby granted without fee, provided that
 * the above copyright notice appear in all copies and that both that copyright
 * notice and this permission notice appear in supporting documentation, and
 * that the name of Marc Aurele La France not be used in advertising or
 * publicity pertaining to distribution of the software without specific,
 * written prior permission.  Marc Aurele La France makes no representations
 * about the suitability of this software for any purpose.  It is provided
 * "as-is" without express or implied warranty.
 *
 * MARC AURELE LA FRANCE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
 * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.  IN NO
 * EVENT SHALL MARC AURELE LA FRANCE BE LIABLE FOR ANY SPECIAL, INDIRECT OR
 * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
 * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 * PERFORMANCE OF THIS SOFTWARE.
 */
/*
 * Copyright 1999-2000 Precision Insight, Inc., Cedar Park, Texas.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.  IN NO EVENT SHALL
 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include "ati.h"
#include "atibus.h"
#include "atichip.h"
#include "atidac.h"
#include "atimach64.h"
#include "atimach64io.h"
#include "atirgb514.h"

#include "miline.h"

#ifndef DPMS_SERVER
# define DPMS_SERVER
#endif
#include "extensions/dpms.h"

/*
 * X-to-Mach64 mix translation table.
 */
static CARD8 ATIMach64ALU[16] =
{
    MIX_0,
    MIX_AND,
    MIX_SRC_AND_NOT_DST,
    MIX_SRC,
    MIX_NOT_SRC_AND_DST,
    MIX_DST,
    MIX_XOR,
    MIX_OR,
    MIX_NOR,
    MIX_XNOR,
    MIX_NOT_DST,
    MIX_SRC_OR_NOT_DST,
    MIX_NOT_SRC,
    MIX_NOT_SRC_OR_DST,
    MIX_NAND,
    MIX_1
};

/*
 * ATIMach64PreInit --
 *
 * This function fills in the Mach64 portion of an ATIHWRec that is common to
 * all video modes generated by the driver.
 */
void
ATIMach64PreInit
(
    ScrnInfoPtr pScreenInfo,
    ATIPtr      pATI,
    ATIHWPtr    pATIHW
)
{
    CARD32 bus_cntl, config_cntl;
    int    tmp;

#ifndef AVOID_CPIO

    if (pATI->depth <= 4)
        pATIHW->crtc_off_pitch = SetBits(pATI->displayWidth >> 4, CRTC_PITCH);
    else

#endif /* AVOID_CPIO */

    {
        pATIHW->crtc_off_pitch = SetBits(pATI->displayWidth >> 3, CRTC_PITCH);
    }

    if ((pATI->LockData.crtc_gen_cntl & CRTC_CSYNC_EN) && !pATI->OptionCSync)
    {
        xf86DrvMsg(pScreenInfo->scrnIndex, X_NOTICE,
            "Using composite sync to match input timing.\n");
        pATI->OptionCSync = TRUE;
    }

    pATIHW->bus_cntl = bus_cntl = inr(BUS_CNTL);
    if (pATI->Chip < ATI_CHIP_264VT4)
        pATIHW->bus_cntl = (pATIHW->bus_cntl & ~BUS_HOST_ERR_INT_EN) |
            BUS_HOST_ERR_INT;
    if (pATI->Chip < ATI_CHIP_264VTB)
    {
        pATIHW->bus_cntl &= ~(BUS_FIFO_ERR_INT_EN | BUS_ROM_DIS);
        pATIHW->bus_cntl |= SetBits(15, BUS_FIFO_WS) | BUS_FIFO_ERR_INT;
    }
    else if (pATI->MMIOInLinear)
    {
        pATIHW->bus_cntl &= ~BUS_APER_REG_DIS;
    }
    else
    {
        pATIHW->bus_cntl |= BUS_APER_REG_DIS;
    }
    if (pATI->Chip >= ATI_CHIP_264VT)
        pATIHW->bus_cntl |= BUS_EXT_REG_EN;     /* Enable Block 1 */

#ifdef AVOID_CPIO

    pATIHW->mem_vga_wp_sel = SetBits(0, MEM_VGA_WPS0) |
        SetBits(1, MEM_VGA_WPS1);
    pATIHW->mem_vga_rp_sel = SetBits(0, MEM_VGA_RPS0) |
        SetBits(1, MEM_VGA_RPS1);

#else /* AVOID_CPIO */

    pATIHW->mem_vga_wp_sel = SetBits(0, MEM_VGA_WPS0) |
        SetBits(pATIHW->nPlane, MEM_VGA_WPS1);
    pATIHW->mem_vga_rp_sel = SetBits(0, MEM_VGA_RPS0) |
        SetBits(pATIHW->nPlane, MEM_VGA_RPS1);

#endif /* AVOID_CPIO */

    pATIHW->dac_cntl = inr(DAC_CNTL) &
        ~(DAC1_CLK_SEL | DAC_PALETTE_ACCESS_CNTL | DAC_8BIT_EN);
    if (pATI->Chip >= ATI_CHIP_264CT)
        pATIHW->dac_cntl &= ~DAC_FEA_CON_EN;
    if (pATI->rgbBits == 8)
        pATIHW->dac_cntl |= DAC_8BIT_EN;

    pATIHW->gen_test_cntl = pATI->LockData.gen_test_cntl & ~GEN_CUR_EN;
    if (pATI->DAC == ATI_DAC_IBMRGB514)
        pATIHW->gen_test_cntl |= GEN_OVR_OUTPUT_EN;

    pATIHW->config_cntl = config_cntl = inr(CONFIG_CNTL);

#ifndef AVOID_CPIO

    if (pATI->UseSmallApertures)
        pATIHW->config_cntl |= CFG_MEM_VGA_AP_EN;
    else

#endif /* AVOID_CPIO */

    {
        pATIHW->config_cntl &= ~CFG_MEM_VGA_AP_EN;
    }

    if (pATI->LinearBase && (pATI->Chip < ATI_CHIP_264CT))
    {
        /* Replace linear aperture size and address */
        pATIHW->config_cntl &= ~(CFG_MEM_AP_LOC | CFG_MEM_AP_SIZE);
        pATIHW->config_cntl |= SetBits(pATI->LinearBase >> 22, CFG_MEM_AP_LOC);
        if ((pATI->Chip < ATI_CHIP_264CT) && (pATI->VideoRAM < 4096))
            pATIHW->config_cntl |= SetBits(1, CFG_MEM_AP_SIZE);
        else
            pATIHW->config_cntl |= SetBits(2, CFG_MEM_AP_SIZE);
    }

    if (pATI->Chip >= ATI_CHIP_264VTB)
    {
        pATIHW->mem_cntl = (pATI->LockData.mem_cntl &
            ~(CTL_MEM_LOWER_APER_ENDIAN | CTL_MEM_UPPER_APER_ENDIAN)) |
            SetBits(CTL_MEM_APER_BYTE_ENDIAN, CTL_MEM_LOWER_APER_ENDIAN);

        switch (pATI->bitsPerPixel)
        {
            default:
                pATIHW->mem_cntl |= SetBits(CTL_MEM_APER_BYTE_ENDIAN,
                    CTL_MEM_UPPER_APER_ENDIAN);
                break;

            case 16:
                pATIHW->mem_cntl |= SetBits(CTL_MEM_APER_WORD_ENDIAN,
                    CTL_MEM_UPPER_APER_ENDIAN);
                break;

            case 32:
                pATIHW->mem_cntl |= SetBits(CTL_MEM_APER_LONG_ENDIAN,
                    CTL_MEM_UPPER_APER_ENDIAN);
                break;
        }

        pATIHW->mpp_config = inr(MPP_CONFIG);
        pATIHW->mpp_config &=
            ~(MPP_PRESCALE | MPP_NSTATES | MPP_FORMAT | MPP_WAIT_STATE |
              MPP_INSERT_WAIT | MPP_TRISTATE_ADDR | MPP_AUTO_INC_EN |
              MPP_CHKREQ_EN | MPP_BUFFER_SIZE | MPP_BUFFER_MODE | MPP_BUSY);
        pATIHW->mpp_config |=
            (MPP_NSTATES_8 | MPP_FORMAT_DA8 | SetBits(4, MPP_WAIT_STATE) |
             MPP_CHKRDY_EN | MPP_READ_EARLY | MPP_RW_MODE | MPP_EN);
        pATIHW->mpp_strobe_seq = inr(MPP_STROBE_SEQ);
        pATIHW->mpp_strobe_seq &= ~(MPP_STB0_SEQ | MPP_STB1_SEQ);
        pATIHW->mpp_strobe_seq |=
            SetBits(0x0087U, MPP_STB0_SEQ) | SetBits(0x0083U, MPP_STB1_SEQ);
        pATIHW->tvo_cntl = 0;
    }

    /* Draw engine setup */
    if (pATI->OptionAccel)
    {
        /* Ensure apertures are enabled */
        outr(BUS_CNTL, pATIHW->bus_cntl);
        outr(CONFIG_CNTL, pATIHW->config_cntl);

        /*
         * When possible, max out command FIFO size.
         */
        if (pATI->Chip >= ATI_CHIP_264VT4)
            pATIHW->gui_cntl = inm(GUI_CNTL) & ~CMDFIFO_SIZE_MODE;

        /* Initialise destination registers */
        pATIHW->dst_off_pitch =
            SetBits((pATI->displayWidth * pATI->XModifier) >> 3, DST_PITCH);
        pATIHW->dst_cntl = DST_X_DIR | DST_Y_DIR | DST_LAST_PEL;

        /* Initialise source registers */
        pATIHW->src_off_pitch = pATIHW->dst_off_pitch;
        pATIHW->src_width1 = pATIHW->src_height1 =
            pATIHW->src_width2 = pATIHW->src_height2 = 1;
        pATIHW->src_cntl = SRC_LINE_X_DIR;

        /* Initialise scissor, allowing for offscreen areas */
        pATIHW->sc_right = (pATI->displayWidth * pATI->XModifier) - 1;
        tmp = (pScreenInfo->videoRam * (1024 * 8) /
            pATI->displayWidth / pATI->bitsPerPixel) - 1;
        if (tmp > ATIMach64MaxY)
            tmp = ATIMach64MaxY;
        pATIHW->sc_bottom = tmp;
        pATI->sc_left_right = SetWord(pATI->NewHW.sc_right, 1) |
            SetWord(pATI->NewHW.sc_left, 0);
        pATI->sc_top_bottom =  SetWord(pATI->NewHW.sc_bottom, 1) |
            SetWord(pATI->NewHW.sc_top, 0);

        /* Initialise data path */
        pATIHW->dp_frgd_clr = (CARD32)(-1);
        pATIHW->dp_write_mask = (CARD32)(-1);

        switch (pATI->depth)
        {
            case 8:
                pATIHW->dp_chain_mask = DP_CHAIN_8BPP;
                pATIHW->dp_pix_width =
                    SetBits(PIX_WIDTH_8BPP, DP_DST_PIX_WIDTH) |
                    SetBits(PIX_WIDTH_8BPP, DP_SRC_PIX_WIDTH) |
                    SetBits(PIX_WIDTH_1BPP, DP_HOST_PIX_WIDTH);
                break;

            case 15:
                pATIHW->dp_chain_mask = DP_CHAIN_15BPP_1555;
                pATIHW->dp_pix_width =
                    SetBits(PIX_WIDTH_15BPP, DP_DST_PIX_WIDTH) |
                    SetBits(PIX_WIDTH_15BPP, DP_SRC_PIX_WIDTH) |
                    SetBits(PIX_WIDTH_1BPP, DP_HOST_PIX_WIDTH);
                break;

            case 16:
                pATIHW->dp_chain_mask = DP_CHAIN_16BPP_565;
                pATIHW->dp_pix_width =
                    SetBits(PIX_WIDTH_16BPP, DP_DST_PIX_WIDTH) |
                    SetBits(PIX_WIDTH_16BPP, DP_SRC_PIX_WIDTH) |
                    SetBits(PIX_WIDTH_1BPP, DP_HOST_PIX_WIDTH);
                break;

            case 24:
                if (pATI->bitsPerPixel == 24)
                {
                    pATIHW->dp_chain_mask = DP_CHAIN_24BPP_888;
                    pATIHW->dp_pix_width =
                        SetBits(PIX_WIDTH_8BPP, DP_DST_PIX_WIDTH) |
                        SetBits(PIX_WIDTH_8BPP, DP_SRC_PIX_WIDTH) |
                        SetBits(PIX_WIDTH_1BPP, DP_HOST_PIX_WIDTH);
                }
                else
                {
                    pATIHW->dp_chain_mask = DP_CHAIN_32BPP_8888;
                    pATIHW->dp_pix_width =
                        SetBits(PIX_WIDTH_32BPP, DP_DST_PIX_WIDTH) |
                        SetBits(PIX_WIDTH_32BPP, DP_SRC_PIX_WIDTH) |
                        SetBits(PIX_WIDTH_1BPP, DP_HOST_PIX_WIDTH);
                }
                break;

            default:
                break;
        }

#if X_BYTE_ORDER == X_LITTLE_ENDIAN

        pATIHW->dp_pix_width |= DP_BYTE_PIX_ORDER;

#endif /* X_BYTE_ORDER */

        pATIHW->dp_mix = SetBits(MIX_SRC, DP_FRGD_MIX) |
            SetBits(MIX_DST, DP_BKGD_MIX);
        pATIHW->dp_src = DP_MONO_SRC_ALLONES |
            SetBits(SRC_FRGD, DP_FRGD_SRC) | SetBits(SRC_BKGD, DP_BKGD_SRC);

        /* Initialise colour compare */
        pATIHW->clr_cmp_msk = (1 << pATI->depth) - 1;

        /* Restore aperture enablement */
        outr(BUS_CNTL, bus_cntl);
        outr(CONFIG_CNTL, config_cntl);
    }
}

/*
 * ATIMach64Save --
 *
 * This function is called to save the Mach64 portion of the current video
 * state.
 */
void
ATIMach64Save
(
    ATIPtr      pATI,
    ATIHWPtr    pATIHW
)
{
    pATIHW->crtc_h_total_disp = inr(CRTC_H_TOTAL_DISP);
    pATIHW->crtc_h_sync_strt_wid = inr(CRTC_H_SYNC_STRT_WID);
    pATIHW->crtc_v_total_disp = inr(CRTC_V_TOTAL_DISP);
    pATIHW->crtc_v_sync_strt_wid = inr(CRTC_V_SYNC_STRT_WID);

    pATIHW->crtc_off_pitch = inr(CRTC_OFF_PITCH);

    pATIHW->crtc_gen_cntl = inr(CRTC_GEN_CNTL);

    pATIHW->ovr_clr = inr(OVR_CLR);
    pATIHW->ovr_wid_left_right = inr(OVR_WID_LEFT_RIGHT);
    pATIHW->ovr_wid_top_bottom = inr(OVR_WID_TOP_BOTTOM);

    pATIHW->cur_clr0 = inr(CUR_CLR0);
    pATIHW->cur_clr1 = inr(CUR_CLR1);
    pATIHW->cur_offset = inr(CUR_OFFSET);
    pATIHW->cur_horz_vert_posn = inr(CUR_HORZ_VERT_POSN);
    pATIHW->cur_horz_vert_off = inr(CUR_HORZ_VERT_OFF);

    pATIHW->clock_cntl = inr(CLOCK_CNTL);

    pATIHW->bus_cntl = inr(BUS_CNTL);

    pATIHW->mem_vga_wp_sel = inr(MEM_VGA_WP_SEL);
    pATIHW->mem_vga_rp_sel = inr(MEM_VGA_RP_SEL);

    pATIHW->dac_cntl = inr(DAC_CNTL);

    pATIHW->config_cntl = inr(CONFIG_CNTL);

    pATIHW->gen_test_cntl = inr(GEN_TEST_CNTL) & ~GEN_CUR_EN;

    if (pATI->Chip >= ATI_CHIP_264VTB)
    {
        pATIHW->mem_cntl = inr(MEM_CNTL);
        pATIHW->mpp_config = inr(MPP_CONFIG);
        pATIHW->mpp_strobe_seq = inr(MPP_STROBE_SEQ);
        pATIHW->tvo_cntl = inr(TVO_CNTL);
    }

    /* Save draw engine state */
    if (pATI->OptionAccel && (pATIHW == &pATI->OldHW))
    {
        /* Ensure apertures are enabled */
        outr(BUS_CNTL, pATI->NewHW.bus_cntl);
        outr(CONFIG_CNTL, pATI->NewHW.config_cntl);

        ATIMach64WaitForIdle(pATI);

        /* Save FIFO size */
        if (pATI->Chip >= ATI_CHIP_264VT4)
            pATIHW->gui_cntl = inm(GUI_CNTL);

        /* Save destination registers */
        pATIHW->dst_off_pitch = inm(DST_OFF_PITCH);
        pATIHW->dst_x = inm(DST_X);
        pATIHW->dst_y = inm(DST_Y);
        pATIHW->dst_height = inm(DST_HEIGHT);
        pATIHW->dst_bres_err = inm(DST_BRES_ERR);
        pATIHW->dst_bres_inc = inm(DST_BRES_INC);
        pATIHW->dst_bres_dec = inm(DST_BRES_DEC);
        pATIHW->dst_cntl = inm(DST_CNTL);

        /* Save source registers */
        pATIHW->src_off_pitch = inm(SRC_OFF_PITCH);
        pATIHW->src_x = inm(SRC_X);
        pATIHW->src_y = inm(SRC_Y);
        pATIHW->src_width1 = inm(SRC_WIDTH1);
        pATIHW->src_height1 = inm(SRC_HEIGHT1);
        pATIHW->src_x_start = inm(SRC_X_START);
        pATIHW->src_y_start = inm(SRC_Y_START);
        pATIHW->src_width2 = inm(SRC_WIDTH2);
        pATIHW->src_height2 = inm(SRC_HEIGHT2);
        pATIHW->src_cntl = inm(SRC_CNTL);

        /* Save host data register */
        pATIHW->host_cntl = inm(HOST_CNTL);

        /* Save pattern registers */
        pATIHW->pat_reg0 = inm(PAT_REG0);
        pATIHW->pat_reg1 = inm(PAT_REG1);
        pATIHW->pat_cntl = inm(PAT_CNTL);

        /* Save scissor registers */
        pATIHW->sc_left = pATI->sc_left = inm(SC_LEFT);
        pATIHW->sc_right = pATI->sc_right = inm(SC_RIGHT);
        pATIHW->sc_top = pATI->sc_top = inm(SC_TOP);
        pATIHW->sc_bottom = pATI->sc_bottom = inm(SC_BOTTOM);

        /* Save data path registers */
        pATIHW->dp_bkgd_clr = inm(DP_BKGD_CLR);
        pATIHW->dp_frgd_clr = inm(DP_FRGD_CLR);
        pATIHW->dp_write_mask = inm(DP_WRITE_MASK);
        pATIHW->dp_chain_mask = inm(DP_CHAIN_MASK);
        pATIHW->dp_pix_width = inm(DP_PIX_WIDTH);
        pATIHW->dp_mix = inm(DP_MIX);
        pATIHW->dp_src = inm(DP_SRC);

        /* Save colour compare registers */
        pATIHW->clr_cmp_clr = inm(CLR_CMP_CLR);
        pATIHW->clr_cmp_msk = inm(CLR_CMP_MSK);
        pATIHW->clr_cmp_cntl = inm(CLR_CMP_CNTL);

        /* Save context */
        pATIHW->context_mask = inm(CONTEXT_MASK);

        /* Restore aperture enablement */
        outr(BUS_CNTL, pATIHW->bus_cntl);
        outr(CONFIG_CNTL, pATIHW->config_cntl);
    }
}

/*
 * ATIMach64Calculate --
 *
 * This function is called to fill in the Mach64 portion of an ATIHWRec.
 */
void
ATIMach64Calculate
(
    ATIPtr         pATI,
    ATIHWPtr       pATIHW,
    DisplayModePtr pMode
)
{
    int VDisplay;

    /* If not already done adjust horizontal timings */
    if (!pMode->CrtcHAdjusted)
    {
        pMode->CrtcHAdjusted = TRUE;
        /* XXX Deal with Blank Start/End and overscan later */
        pMode->CrtcHDisplay = (pMode->HDisplay >> 3) - 1;
        pMode->CrtcHSyncStart = (pMode->HSyncStart >> 3) - 1;
        pMode->CrtcHSyncEnd = (pMode->HSyncEnd >> 3) - 1;
        pMode->CrtcHTotal = (pMode->HTotal >> 3) - 1;

        /* Make adjustments if sync pulse width is out-of-bounds */
        if ((pMode->CrtcHSyncEnd - pMode->CrtcHSyncStart) >
            (int)MaxBits(CRTC_H_SYNC_WID))
            pMode->CrtcHSyncEnd =
                pMode->CrtcHSyncStart + MaxBits(CRTC_H_SYNC_WID);
        else if (pMode->CrtcHSyncStart == pMode->CrtcHSyncEnd)
        {
            if (pMode->CrtcHDisplay < pMode->CrtcHSyncStart)
                pMode->CrtcHSyncStart--;
            else if (pMode->CrtcHSyncEnd < pMode->CrtcHTotal)
                pMode->CrtcHSyncEnd++;
        }
    }

    /*
     * Always re-do vertical adjustments.
     */
    pMode->CrtcVDisplay = pMode->VDisplay;
    pMode->CrtcVSyncStart = pMode->VSyncStart;
    pMode->CrtcVSyncEnd = pMode->VSyncEnd;
    pMode->CrtcVTotal = pMode->VTotal;

    if ((pATI->Chip >= ATI_CHIP_264CT) &&
        ((pMode->Flags & V_DBLSCAN) || (pMode->VScan > 1)))
    {
        pMode->CrtcVDisplay <<= 1;
        pMode->CrtcVSyncStart <<= 1;
        pMode->CrtcVSyncEnd <<= 1;
        pMode->CrtcVTotal <<= 1;
    }

    /*
     * Might as well default to the same as VGA with respect to sync
     * polarities.
     */
    if ((!(pMode->Flags & (V_PHSYNC | V_NHSYNC))) ||
        (!(pMode->Flags & (V_PVSYNC | V_NVSYNC))))
    {
        pMode->Flags &= ~(V_PHSYNC | V_NHSYNC | V_PVSYNC | V_NVSYNC);

        if (pATI->OptionPanelDisplay && (pATI->LCDPanelID >= 0))
            VDisplay = pATI->LCDVertical;
        else
            VDisplay = pMode->CrtcVDisplay;

        if (VDisplay < 400)
            pMode->Flags |= V_PHSYNC | V_NVSYNC;
        else if (VDisplay < 480)
            pMode->Flags |= V_NHSYNC | V_PVSYNC;
        else if (VDisplay < 768)
            pMode->Flags |= V_NHSYNC | V_NVSYNC;
        else
            pMode->Flags |= V_PHSYNC | V_PVSYNC;
    }

    pMode->CrtcVDisplay--;
    pMode->CrtcVSyncStart--;
    pMode->CrtcVSyncEnd--;
    pMode->CrtcVTotal--;
    /* Make sure sync pulse is not too wide */
    if ((pMode->CrtcVSyncEnd - pMode->CrtcVSyncStart) >
         (int)MaxBits(CRTC_V_SYNC_WID))
        pMode->CrtcVSyncEnd = pMode->CrtcVSyncStart + MaxBits(CRTC_V_SYNC_WID);
    pMode->CrtcVAdjusted = TRUE;                /* Redundant */

    /* Build register contents */
    pATIHW->crtc_h_total_disp =
        SetBits(pMode->CrtcHTotal, CRTC_H_TOTAL) |
            SetBits(pMode->CrtcHDisplay, CRTC_H_DISP);
    pATIHW->crtc_h_sync_strt_wid =
        SetBits(pMode->CrtcHSyncStart, CRTC_H_SYNC_STRT) |
            SetBits(pMode->CrtcHSkew, CRTC_H_SYNC_DLY) |         /* ? */
            SetBits(GetBits(pMode->CrtcHSyncStart, 0x0100U),
                CRTC_H_SYNC_STRT_HI) |
            SetBits(pMode->CrtcHSyncEnd - pMode->CrtcHSyncStart,
                CRTC_H_SYNC_WID);
    if (pMode->Flags & V_NHSYNC)
        pATIHW->crtc_h_sync_strt_wid |= CRTC_H_SYNC_POL;

    pATIHW->crtc_v_total_disp =
        SetBits(pMode->CrtcVTotal, CRTC_V_TOTAL) |
            SetBits(pMode->CrtcVDisplay, CRTC_V_DISP);
    pATIHW->crtc_v_sync_strt_wid =
        SetBits(pMode->CrtcVSyncStart, CRTC_V_SYNC_STRT) |
            SetBits(pMode->CrtcVSyncEnd - pMode->CrtcVSyncStart,
                CRTC_V_SYNC_WID);
    if (pMode->Flags & V_NVSYNC)
        pATIHW->crtc_v_sync_strt_wid |= CRTC_V_SYNC_POL;

    pATIHW->crtc_gen_cntl = inr(CRTC_GEN_CNTL) &
        ~(CRTC_DBL_SCAN_EN | CRTC_INTERLACE_EN |
          CRTC_HSYNC_DIS | CRTC_VSYNC_DIS | CRTC_CSYNC_EN |
          CRTC_PIX_BY_2_EN | CRTC_DISPLAY_DIS | CRTC_VGA_XOVERSCAN |
          CRTC_PIX_WIDTH | CRTC_BYTE_PIX_ORDER |
          CRTC_VGA_128KAP_PAGING | CRTC_VFC_SYNC_TRISTATE |
          CRTC_LOCK_REGS |              /* Already off, but ... */
          CRTC_SYNC_TRISTATE | CRTC_DISP_REQ_EN |
          CRTC_VGA_TEXT_132 | CRTC_CUR_B_TEST);
    pATIHW->crtc_gen_cntl |=
        CRTC_EXT_DISP_EN | CRTC_EN | CRTC_VGA_LINEAR | CRTC_CNT_EN;
    switch (pATI->depth)
    {

#ifndef AVOID_CPIO

        case 1:
            pATIHW->crtc_gen_cntl |= SetBits(PIX_WIDTH_1BPP, CRTC_PIX_WIDTH);
            break;

        case 4:
            pATIHW->crtc_gen_cntl |= SetBits(PIX_WIDTH_4BPP, CRTC_PIX_WIDTH);
            break;

#endif /* AVOID_CPIO */

        case 8:
            pATIHW->crtc_gen_cntl |= SetBits(PIX_WIDTH_8BPP, CRTC_PIX_WIDTH);
            break;

        case 15:
            pATIHW->crtc_gen_cntl |= SetBits(PIX_WIDTH_15BPP, CRTC_PIX_WIDTH);
            break;

        case 16:
            pATIHW->crtc_gen_cntl |= SetBits(PIX_WIDTH_16BPP, CRTC_PIX_WIDTH);
            break;

        case 24:
            if (pATI->bitsPerPixel == 24)
            {
                pATIHW->crtc_gen_cntl |=
                    SetBits(PIX_WIDTH_24BPP, CRTC_PIX_WIDTH);
                break;
            }
            if (pATI->bitsPerPixel != 32)
                break;
            /* Fall through */

        case 32:
            pATIHW->crtc_gen_cntl |= SetBits(PIX_WIDTH_32BPP, CRTC_PIX_WIDTH);
            break;

        default:
            break;
    }
    if ((pMode->Flags & V_DBLSCAN) || (pMode->VScan > 1))
        pATIHW->crtc_gen_cntl |= CRTC_DBL_SCAN_EN;
    if (pMode->Flags & V_INTERLACE)
        pATIHW->crtc_gen_cntl |= CRTC_INTERLACE_EN;
    if (pATI->OptionCSync || (pMode->Flags & (V_CSYNC | V_PCSYNC)))
        pATIHW->crtc_gen_cntl |= CRTC_CSYNC_EN;
    /* For now, set display FIFO low water mark as high as possible */
    if (pATI->Chip < ATI_CHIP_264VTB)
        pATIHW->crtc_gen_cntl |= CRTC_FIFO_LWM;
}

/*
 * ATIMach64Set --
 *
 * This function is called to load a Mach64's accelerator CRTC and draw engine.
 */
void
ATIMach64Set
(
    ATIPtr      pATI,
    ATIHWPtr    pATIHW
)
{
    /* First, turn off the display */
    outr(CRTC_GEN_CNTL, pATIHW->crtc_gen_cntl & ~CRTC_EN);

    if ((pATIHW->FeedbackDivider > 0) &&
        (pATI->ProgrammableClock != ATI_CLOCK_NONE))
        ATIClockSet(pATI, pATIHW);              /* Programme clock */

    if (pATI->DAC == ATI_DAC_IBMRGB514)
        ATIRGB514Set(pATI, pATIHW);

    /* Load Mach64 CRTC registers */
    outr(CRTC_H_TOTAL_DISP, pATIHW->crtc_h_total_disp);
    outr(CRTC_H_SYNC_STRT_WID, pATIHW->crtc_h_sync_strt_wid);
    outr(CRTC_V_TOTAL_DISP, pATIHW->crtc_v_total_disp);
    outr(CRTC_V_SYNC_STRT_WID, pATIHW->crtc_v_sync_strt_wid);

    outr(CRTC_OFF_PITCH, pATIHW->crtc_off_pitch);

    /* Load overscan registers */
    outr(OVR_CLR, pATIHW->ovr_clr);
    outr(OVR_WID_LEFT_RIGHT, pATIHW->ovr_wid_left_right);
    outr(OVR_WID_TOP_BOTTOM, pATIHW->ovr_wid_top_bottom);

    /* Load hardware cursor registers */
    outr(CUR_CLR0, pATIHW->cur_clr0);
    outr(CUR_CLR1, pATIHW->cur_clr1);
    outr(CUR_OFFSET, pATIHW->cur_offset);
    outr(CUR_HORZ_VERT_POSN, pATIHW->cur_horz_vert_posn);
    outr(CUR_HORZ_VERT_OFF, pATIHW->cur_horz_vert_off);

    /* Set pixel clock */
    outr(CLOCK_CNTL, pATIHW->clock_cntl | CLOCK_STROBE);

    outr(GEN_TEST_CNTL, pATIHW->gen_test_cntl | GEN_GUI_EN);
    outr(GEN_TEST_CNTL, pATIHW->gen_test_cntl);
    outr(GEN_TEST_CNTL, pATIHW->gen_test_cntl | GEN_GUI_EN);

    /* Finalise CRTC setup and turn on the screen */
    outr(CRTC_GEN_CNTL, pATIHW->crtc_gen_cntl);

    /* Load draw engine */
    if (pATI->OptionAccel)
    {
        /* Clobber MMIO cache */
        (void)memset(pATI->MMIOCached, 0, SizeOf(pATI->MMIOCached));

        /* Ensure apertures are enabled */
        outr(BUS_CNTL, pATI->NewHW.bus_cntl);
        outr(CONFIG_CNTL, pATI->NewHW.config_cntl);

        pATI->EngineIsBusy = TRUE;      /* Force engine poll */
        ATIMach64WaitForIdle(pATI);

        /* Load FIFO size */
        if (pATI->Chip >= ATI_CHIP_264VT4)
        {
            outm(GUI_CNTL, pATIHW->gui_cntl);
            pATI->nAvailableFIFOEntries = 0;
            ATIMach64PollEngineStatus(pATI);
        }

        /* Set FIFO depth */
        pATI->nFIFOEntries = pATI->nAvailableFIFOEntries;

        /* Load destination registers */
        ATIMach64WaitForFIFO(pATI, 7);
        outf(DST_OFF_PITCH, pATIHW->dst_off_pitch);
        outf(DST_Y_X, SetWord(pATIHW->dst_x, 1) | SetWord(pATIHW->dst_y, 0));
        outf(DST_HEIGHT, pATIHW->dst_height);
        outf(DST_BRES_ERR, pATIHW->dst_bres_err);
        outf(DST_BRES_INC, pATIHW->dst_bres_inc);
        outf(DST_BRES_DEC, pATIHW->dst_bres_dec);
        outf(DST_CNTL, pATIHW->dst_cntl);

        /* Load source registers */
        ATIMach64WaitForFIFO(pATI, 6);
        outf(SRC_OFF_PITCH, pATIHW->src_off_pitch);
        outf(SRC_Y_X, SetWord(pATIHW->src_x, 1) | SetWord(pATIHW->src_y, 0));
        outf(SRC_HEIGHT1_WIDTH1,
            SetWord(pATIHW->src_width1, 1) | SetWord(pATIHW->src_height1, 0));
        outf(SRC_Y_X_START,
            SetWord(pATIHW->src_x_start, 1) | SetWord(pATIHW->src_y_start, 0));
        outf(SRC_HEIGHT2_WIDTH2,
            SetWord(pATIHW->src_width2, 1) | SetWord(pATIHW->src_height2, 0));
        outf(SRC_CNTL, pATIHW->src_cntl);

        /* Load host data register */
        ATIMach64WaitForFIFO(pATI, 1);
        outf(HOST_CNTL, pATIHW->host_cntl);

        /* Set host transfer window address and size clamp */
        pATI->pHOST_DATA =
            (CARD8 *)pATI->pBlock[GetBits(HOST_DATA_0, BLOCK_SELECT)] +
            (HOST_DATA_0 & MM_IO_SELECT);
        pATI->nHostFIFOEntries = pATI->nFIFOEntries >> 1;
        if (pATI->nHostFIFOEntries > 16)
            pATI->nHostFIFOEntries = 16;

        /* Load pattern registers */
        ATIMach64WaitForFIFO(pATI, 3);
        outf(PAT_REG0, pATIHW->pat_reg0);
        outf(PAT_REG1, pATIHW->pat_reg1);
        outf(PAT_CNTL, pATIHW->pat_cntl);

        /* Load scissor registers */
        ATIMach64WaitForFIFO(pATI, 2);
        outf(SC_LEFT_RIGHT,
            SetWord(pATIHW->sc_right, 1) | SetWord(pATIHW->sc_left, 0));
        outf(SC_TOP_BOTTOM,
            SetWord(pATIHW->sc_bottom, 1) | SetWord(pATIHW->sc_top, 0));
        pATI->sc_left = pATIHW->sc_left;
        pATI->sc_right = pATIHW->sc_right;
        pATI->sc_top = pATIHW->sc_top;
        pATI->sc_bottom = pATIHW->sc_bottom;

        /* Load data path registers */
        ATIMach64WaitForFIFO(pATI, 7);
        outf(DP_BKGD_CLR, pATIHW->dp_bkgd_clr);
        outf(DP_FRGD_CLR, pATIHW->dp_frgd_clr);
        outf(DP_WRITE_MASK, pATIHW->dp_write_mask);
        outf(DP_CHAIN_MASK, pATIHW->dp_chain_mask);
        outf(DP_PIX_WIDTH, pATIHW->dp_pix_width);
        outf(DP_MIX, pATIHW->dp_mix);
        outf(DP_SRC, pATIHW->dp_src);

        /* Load colour compare registers */
        ATIMach64WaitForFIFO(pATI, 3);
        outf(CLR_CMP_CLR, pATIHW->clr_cmp_clr);
        outf(CLR_CMP_MSK, pATIHW->clr_cmp_msk);
        outf(CLR_CMP_CNTL, pATIHW->clr_cmp_cntl);

        /* Load context mask */
        ATIMach64WaitForFIFO(pATI, 1);
        outf(CONTEXT_MASK, pATIHW->context_mask);

        ATIMach64WaitForIdle(pATI);

        if (pATI->OptionMMIOCache)
        {
            /*
             * Enable write caching for selected MMIO registers.  This can only
             * be done for those registers whose value does not change without
             * driver intervention.
             */

            CacheRegister(SRC_CNTL);

            CacheRegister(HOST_CNTL);

            CacheRegister(PAT_REG0);
            CacheRegister(PAT_REG1);
            CacheRegister(PAT_CNTL);

            CacheRegister(SC_LEFT_RIGHT);
            CacheRegister(SC_TOP_BOTTOM);

            CacheRegister(DP_BKGD_CLR);
            CacheRegister(DP_FRGD_CLR);
            CacheRegister(DP_WRITE_MASK);
            CacheRegister(DP_MIX);

            CacheRegister(CLR_CMP_CLR);
            CacheRegister(CLR_CMP_MSK);
            CacheRegister(CLR_CMP_CNTL);
        }
    }

    /* Aperture setup */
    outr(MEM_VGA_WP_SEL, pATIHW->mem_vga_wp_sel);
    outr(MEM_VGA_RP_SEL, pATIHW->mem_vga_rp_sel);

    outr(DAC_CNTL, pATIHW->dac_cntl);

    outr(CONFIG_CNTL, pATIHW->config_cntl);
    outr(BUS_CNTL, pATIHW->bus_cntl);

    if (pATI->Chip >= ATI_CHIP_264VTB)
    {
        outr(MEM_CNTL, pATIHW->mem_cntl);
        outr(MPP_CONFIG, pATIHW->mpp_config);
        outr(MPP_STROBE_SEQ, pATIHW->mpp_strobe_seq);
        outr(TVO_CNTL, pATIHW->tvo_cntl);
    }
}

/*
 * ATIMach64SaveScreen --
 *
 * This function blanks or unblanks a Mach64 screen.
 */
void
ATIMach64SaveScreen
(
    ATIPtr pATI,
    int    Mode
)
{
    CARD32 crtc_gen_cntl = inr(CRTC_GEN_CNTL);

    switch (Mode)
    {
        case SCREEN_SAVER_OFF:
        case SCREEN_SAVER_FORCER:
            outr(CRTC_GEN_CNTL, crtc_gen_cntl & ~CRTC_DISPLAY_DIS);
            break;

        case SCREEN_SAVER_ON:
        case SCREEN_SAVER_CYCLE:
            outr(CRTC_GEN_CNTL, crtc_gen_cntl | CRTC_DISPLAY_DIS);
            break;

        default:
            break;
    }
}

/*
 * ATIMach64SetDPMSMode --
 *
 * This function sets a Mach64's VESA Display Power Management Signaling mode.
 */
void
ATIMach64SetDPMSMode
(
    ScrnInfoPtr pScreenInfo,
    ATIPtr      pATI,
    int         DPMSMode
)
{
    CARD32 crtc_gen_cntl =
        inr(CRTC_GEN_CNTL) & ~(CRTC_HSYNC_DIS | CRTC_VSYNC_DIS);

    switch (DPMSMode)
    {
        case DPMSModeOn:        /* HSync on, VSync on */
            break;

        case DPMSModeStandby:   /* HSync off, VSync on */
            crtc_gen_cntl |= CRTC_HSYNC_DIS;
            break;

        case DPMSModeSuspend:   /* HSync on, VSync off */
            crtc_gen_cntl |= CRTC_VSYNC_DIS;
            break;

        case DPMSModeOff:       /* HSync off, VSync off */
            crtc_gen_cntl |= CRTC_HSYNC_DIS | CRTC_VSYNC_DIS;
            break;

        default:                /* Muffle compiler */
            return;
    }

    if (pATI->pXAAInfo && pATI->pXAAInfo->NeedToSync)
        (*pATI->pXAAInfo->Sync)(pScreenInfo);

    outr(CRTC_GEN_CNTL, crtc_gen_cntl);

    if (pATI->OptionPanelDisplay && (pATI->LCDPanelID >= 0))
    {
        CARD32 lcd_index = 0;

        /*
         * ATI's BIOS simply turns the panel on and off, so do the same by
         * default, but keep the previous behaviour around for reference.
         */
        if (pATI->OptionDevel)
        {
            CARD32 power_management;

            if (pATI->Chip == ATI_CHIP_264LT)
                power_management = inr(POWER_MANAGEMENT);
            else /* if ((pATI->Chip == ATI_CHIP_264LTPRO) ||
                        (pATI->Chip == ATI_CHIP_264XL) ||
                        (pATI->Chip == ATI_CHIP_MOBILITY)) */
            {
                lcd_index = inr(LCD_INDEX);
                power_management = ATIGetMach64LCDReg(LCD_POWER_MANAGEMENT);
            }

            power_management &= ~(STANDBY_NOW | SUSPEND_NOW);

            switch (DPMSMode)
            {
                case DPMSModeOn:
                    break;

                case DPMSModeStandby:
                    power_management |= STANDBY_NOW;
                    break;

                case DPMSModeSuspend:
                    power_management |= SUSPEND_NOW;
                    break;

                case DPMSModeOff:
                    power_management |= STANDBY_NOW | SUSPEND_NOW;  /* ? */
                    break;

                default:        /* Muffle compiler */
                    return;
            }

            if (pATI->Chip == ATI_CHIP_264LT)
                outr(POWER_MANAGEMENT, power_management);
            else /* if ((pATI->Chip == ATI_CHIP_264LTPRO) ||
                        (pATI->Chip == ATI_CHIP_264XL) ||
                        (pATI->Chip == ATI_CHIP_MOBILITY)) */
            {
                ATIPutMach64LCDReg(LCD_POWER_MANAGEMENT, power_management);
                outr(LCD_INDEX, lcd_index);
            }
        }
        else
        {
            CARD32 lcd_gen_ctrl;

            if (pATI->Chip == ATI_CHIP_264LT)
                lcd_gen_ctrl = inr(LCD_GEN_CTRL);
            else /* if ((pATI->Chip == ATI_CHIP_264LTPRO) ||
                        (pATI->Chip == ATI_CHIP_264XL) ||
                        (pATI->Chip == ATI_CHIP_MOBILITY)) */
            {
                lcd_index = inr(LCD_INDEX);
                lcd_gen_ctrl = ATIGetMach64LCDReg(LCD_GEN_CNTL);
            }

            if (DPMSMode == DPMSModeOn)
                lcd_gen_ctrl |= LCD_ON;
            else
                lcd_gen_ctrl &= ~LCD_ON;

            if (pATI->Chip == ATI_CHIP_264LT)
                outr(LCD_GEN_CTRL, lcd_gen_ctrl);
            else /* if ((pATI->Chip == ATI_CHIP_264LTPRO) ||
                        (pATI->Chip == ATI_CHIP_264XL) ||
                        (pATI->Chip == ATI_CHIP_MOBILITY)) */
            {
                ATIPutMach64LCDReg(LCD_GEN_CNTL, lcd_gen_ctrl);
                outr(LCD_INDEX, lcd_index);
            }
        }
    }
}

/*
 * ATIMach64ValidateClip --
 *
 * This function ensures the current scissor settings do not interfere with
 * the current draw request.
 */
static void
ATIMach64ValidateClip
(
    ATIPtr pATI,
    int    sc_left,
    int    sc_right,
    int    sc_top,
    int    sc_bottom
)
{
    if ((sc_left < (int)pATI->sc_left) || (sc_right > (int)pATI->sc_right))
    {
        outf(SC_LEFT_RIGHT, pATI->sc_left_right);
        pATI->sc_left = pATI->NewHW.sc_left;
        pATI->sc_right = pATI->NewHW.sc_right;
    }

    if ((sc_top < (int)pATI->sc_top) || (sc_bottom > (int)pATI->sc_bottom))
    {
        outf(SC_TOP_BOTTOM, pATI->sc_top_bottom);
        pATI->sc_top = pATI->NewHW.sc_top;
        pATI->sc_bottom = pATI->NewHW.sc_bottom;
    }
}

/*
 * ATIMach64Sync --
 *
 * This is called to wait for the draw engine to become idle.
 */
static void
ATIMach64Sync
(
    ScrnInfoPtr pScreenInfo
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);

    ATIMach64WaitForIdle(pATI);

    if (pATI->OptionMMIOCache)
    {
        /*
         * For debugging purposes, attempt to verify that each cached register
         * should actually be cached.
         */
        if (RegisterIsCached(SRC_CNTL) &&
            (CacheSlot(SRC_CNTL) != inm(SRC_CNTL)))
        {
            UncacheRegister(SRC_CNTL);
            xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
                "SRC_CNTL write cache disabled!\n");
        }

        if (RegisterIsCached(HOST_CNTL) &&
            (CacheSlot(HOST_CNTL) != inm(HOST_CNTL)))
        {
            UncacheRegister(HOST_CNTL);
            xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
                "HOST_CNTL write cache disabled!\n");
        }

        if (RegisterIsCached(PAT_REG0) &&
            (CacheSlot(PAT_REG0) != inm(PAT_REG0)))
        {
            UncacheRegister(PAT_REG0);
            xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
                "PAT_REG0 write cache disabled!\n");
        }

        if (RegisterIsCached(PAT_REG1) &&
            (CacheSlot(PAT_REG1) != inm(PAT_REG1)))
        {
            UncacheRegister(PAT_REG1);
            xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
                "PAT_REG1 write cache disabled!\n");
        }

        if (RegisterIsCached(PAT_CNTL) &&
            (CacheSlot(PAT_CNTL) != inm(PAT_CNTL)))
        {
            UncacheRegister(PAT_CNTL);
            xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
                "PAT_CNTL write cache disabled!\n");
        }

        if (RegisterIsCached(SC_LEFT_RIGHT) &&
            (CacheSlot(SC_LEFT_RIGHT) !=
             (SetWord(inm(SC_RIGHT), 1) | SetWord(inm(SC_LEFT), 0))))
        {
            UncacheRegister(SC_LEFT_RIGHT);
            xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
                "SC_LEFT_RIGHT write cache disabled!\n");
        }

        if (RegisterIsCached(SC_TOP_BOTTOM) &&
            (CacheSlot(SC_TOP_BOTTOM) !=
             (SetWord(inm(SC_BOTTOM), 1) | SetWord(inm(SC_TOP), 0))))
        {
            UncacheRegister(SC_TOP_BOTTOM);
            xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
                "SC_TOP_BOTTOM write cache disabled!\n");
        }

        if (RegisterIsCached(DP_BKGD_CLR) &&
            (CacheSlot(DP_BKGD_CLR) != inm(DP_BKGD_CLR)))
        {
            UncacheRegister(DP_BKGD_CLR);
            xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
                "DP_BKGD_CLR write cache disabled!\n");
        }

        if (RegisterIsCached(DP_FRGD_CLR) &&
            (CacheSlot(DP_FRGD_CLR) != inm(DP_FRGD_CLR)))
        {
            UncacheRegister(DP_FRGD_CLR);
            xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
                "DP_FRGD_CLR write cache disabled!\n");
        }

        if (RegisterIsCached(DP_WRITE_MASK) &&
            (CacheSlot(DP_WRITE_MASK) != inm(DP_WRITE_MASK)))
        {
            UncacheRegister(DP_WRITE_MASK);
            xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
                "DP_WRITE_MASK write cache disabled!\n");
        }

        if (RegisterIsCached(DP_MIX) &&
            (CacheSlot(DP_MIX) != inm(DP_MIX)))
        {
            UncacheRegister(DP_MIX);
            xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
                "DP_MIX write cache disabled!\n");
        }

        if (RegisterIsCached(CLR_CMP_CLR) &&
            (CacheSlot(CLR_CMP_CLR) != inm(CLR_CMP_CLR)))
        {
            UncacheRegister(CLR_CMP_CLR);
            xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
                "CLR_CMP_CLR write cache disabled!\n");
        }

        if (RegisterIsCached(CLR_CMP_MSK) &&
            (CacheSlot(CLR_CMP_MSK) != inm(CLR_CMP_MSK)))
        {
            UncacheRegister(CLR_CMP_MSK);
            xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
                "CLR_CMP_MSK write cache disabled!\n");
        }

        if (RegisterIsCached(CLR_CMP_CNTL) &&
            (CacheSlot(CLR_CMP_CNTL) != inm(CLR_CMP_CNTL)))
        {
            UncacheRegister(CLR_CMP_CNTL);
            xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
                "CLR_CMP_CNTL write cache disabled!\n");
        }
    }

    /*
     * For VTB's and later, the first CPU read of the framebuffer will return
     * zeroes, so do it here.  This appears to be due to some kind of engine
     * caching of framebuffer data I haven't found any way of disabling, or
     * otherwise circumventing.  Thanks to Mark Vojkovich for the suggestion.
     */
    pATI->pXAAInfo->NeedToSync = FALSE;
    pATI = *(volatile ATIPtr *)pATI->pMemory;
}

/*
 * ATIMach64SetupForScreenToScreenCopy --
 *
 * This function sets up the draw engine for a series of screen-to-screen copy
 * operations.
 */
static void
ATIMach64SetupForScreenToScreenCopy
(
    ScrnInfoPtr  pScreenInfo,
    int          xdir,
    int          ydir,
    int          rop,
    unsigned int planemask,
    int          TransparencyColour
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);

    ATIMach64WaitForFIFO(pATI, 3);
    outf(DP_WRITE_MASK, planemask);
    outf(DP_SRC, DP_MONO_SRC_ALLONES |
        SetBits(SRC_BLIT, DP_FRGD_SRC) | SetBits(SRC_BKGD, DP_BKGD_SRC));
    outf(DP_MIX, SetBits(ATIMach64ALU[rop], DP_FRGD_MIX));

#ifdef AVOID_DGA

    if (TransparencyColour == -1)

#else /* AVOID_DGA */

    if (!pATI->XAAForceTransBlit && (TransparencyColour == -1))

#endif /* AVOID_DGA */

        outf(CLR_CMP_CNTL, CLR_CMP_FN_FALSE);
    else
    {
        ATIMach64WaitForFIFO(pATI, 2);
        outf(CLR_CMP_CLR, TransparencyColour);
        outf(CLR_CMP_CNTL, CLR_CMP_FN_EQUAL | CLR_CMP_SRC_2D);
    }

    pATI->dst_cntl = 0;

    if (ydir > 0)
        pATI->dst_cntl |= DST_Y_DIR;
    if (xdir > 0)
        pATI->dst_cntl |= DST_X_DIR;

    if (pATI->XModifier == 1)
        outf(DST_CNTL, pATI->dst_cntl);
    else
        pATI->dst_cntl |= DST_24_ROT_EN;
}

/*
 * ATIMach64SubsequentScreenToScreenCopy --
 *
 * This function performs a screen-to-screen copy operation.
 */
static void
ATIMach64SubsequentScreenToScreenCopy
(
    ScrnInfoPtr pScreenInfo,
    int         xSrc,
    int         ySrc,
    int         xDst,
    int         yDst,
    int         w,
    int         h
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);

    xSrc *= pATI->XModifier;
    xDst *= pATI->XModifier;
    w    *= pATI->XModifier;

    /* Disable clipping if it gets in the way */
    ATIMach64ValidateClip(pATI, xDst, xDst + w - 1, yDst, yDst + h - 1);

    if (!(pATI->dst_cntl & DST_X_DIR))
    {
        xSrc += w - 1;
        xDst += w - 1;
    }

    if (!(pATI->dst_cntl & DST_Y_DIR))
    {
        ySrc += h - 1;
        yDst += h - 1;
    }

    if (pATI->XModifier != 1)
        outf(DST_CNTL, pATI->dst_cntl | SetBits((xDst / 4) % 6, DST_24_ROT));

    ATIMach64WaitForFIFO(pATI, 4);
    outf(SRC_Y_X, SetWord(xSrc, 1) | SetWord(ySrc, 0));
    outf(SRC_WIDTH1, w);
    outf(DST_Y_X, SetWord(xDst, 1) | SetWord(yDst, 0));
    outf(DST_HEIGHT_WIDTH, SetWord(w, 1) | SetWord(h, 0));
}

/*
 * ATIMach64SetupForSolidFill --
 *
 * This function sets up the draw engine for a series of solid fills.
 */
static void
ATIMach64SetupForSolidFill
(
    ScrnInfoPtr  pScreenInfo,
    int          colour,
    int          rop,
    unsigned int planemask
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);

    ATIMach64WaitForFIFO(pATI, 5);
    outf(DP_WRITE_MASK, planemask);
    outf(DP_SRC, DP_MONO_SRC_ALLONES |
        SetBits(SRC_FRGD, DP_FRGD_SRC) | SetBits(SRC_BKGD, DP_BKGD_SRC));
    outf(DP_FRGD_CLR, colour);
    outf(DP_MIX, SetBits(ATIMach64ALU[rop], DP_FRGD_MIX));

    outf(CLR_CMP_CNTL, CLR_CMP_FN_FALSE);

    if (pATI->XModifier == 1)
        outf(DST_CNTL, DST_X_DIR | DST_Y_DIR);
}

/*
 * ATIMach64SubsequentSolidFillRect --
 *
 * This function performs a solid rectangle fill.
 */
static void
ATIMach64SubsequentSolidFillRect
(
    ScrnInfoPtr pScreenInfo,
    int         x,
    int         y,
    int         w,
    int         h
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);

    if (pATI->XModifier != 1)
    {
        x *= pATI->XModifier;
        w *= pATI->XModifier;

        outf(DST_CNTL, SetBits((x / 4) % 6, DST_24_ROT) |
            (DST_X_DIR | DST_Y_DIR | DST_24_ROT_EN));
    }

    /* Disable clipping if it gets in the way */
    ATIMach64ValidateClip(pATI, x, x + w - 1, y, y + h - 1);

    ATIMach64WaitForFIFO(pATI, 2);
    outf(DST_Y_X, SetWord(x, 1) | SetWord(y, 0));
    outf(DST_HEIGHT_WIDTH, SetWord(w, 1) | SetWord(h, 0));
}

/*
 * ATIMach64SetupForSolidLine --
 *
 * This function sets up the draw engine for a series of solid lines.  It is
 * not used for 24bpp because the engine doesn't support it.
 */
static void
ATIMach64SetupForSolidLine
(
    ScrnInfoPtr  pScreenInfo,
    int          colour,
    int          rop,
    unsigned int planemask
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);

    ATIMach64WaitForFIFO(pATI, 5);
    outf(DP_WRITE_MASK, planemask);
    outf(DP_SRC, DP_MONO_SRC_ALLONES |
        SetBits(SRC_FRGD, DP_FRGD_SRC) | SetBits(SRC_BKGD, DP_BKGD_SRC));
    outf(DP_FRGD_CLR, colour);
    outf(DP_MIX, SetBits(ATIMach64ALU[rop], DP_FRGD_MIX));

    outf(CLR_CMP_CNTL, CLR_CMP_FN_FALSE);

    ATIMach64ValidateClip(pATI, pATI->NewHW.sc_left, pATI->NewHW.sc_right,
        pATI->NewHW.sc_top, pATI->NewHW.sc_bottom);
}

/*
 * ATIMach64SubsequentSolidHorVertLine --
 *
 * This is called to draw a solid horizontal or vertical line.  This does a
 * one-pixel wide solid fill.
 */
static void
ATIMach64SubsequentSolidHorVertLine
(
    ScrnInfoPtr pScreenInfo,
    int         x,
    int         y,
    int         len,
    int         dir
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);

    ATIMach64WaitForFIFO(pATI, 3);
    outf(DST_CNTL, DST_X_DIR | DST_Y_DIR);
    outf(DST_Y_X, SetWord(x, 1) | SetWord(y, 0));

    if (dir == DEGREES_0)
        outf(DST_HEIGHT_WIDTH, SetWord(len, 1) | SetWord(1, 0));
    else /* if (dir == DEGREES_270) */
        outf(DST_HEIGHT_WIDTH, SetWord(1, 1) | SetWord(len, 0));
}

/*
 * ATIMach64SubsequentSolidBresenhamLine --
 *
 * This function draws a line using the Bresenham line engine.
 */
static void
ATIMach64SubsequentSolidBresenhamLine
(
    ScrnInfoPtr pScreenInfo,
    int         x,
    int         y,
    int         major,
    int         minor,
    int         err,
    int         len,
    int         octant
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);
    CARD32 dst_cntl = DST_LAST_PEL;

    if (octant & YMAJOR)
        dst_cntl |= DST_Y_MAJOR;

    if (!(octant & XDECREASING))
        dst_cntl |= DST_X_DIR;

    if (!(octant & YDECREASING))
        dst_cntl |= DST_Y_DIR;

    ATIMach64WaitForFIFO(pATI, 6);
    outf(DST_CNTL, dst_cntl);
    outf(DST_Y_X, SetWord(x, 1) | SetWord(y, 0));
    outf(DST_BRES_ERR, minor + err);
    outf(DST_BRES_INC, minor);
    outf(DST_BRES_DEC, minor - major);
    outf(DST_BRES_LNTH, len);
}

/*
 * ATIMach64SetupForMono8x8PatternFill --
 *
 * This function sets up the draw engine for a series of 8x8 1bpp pattern
 * fills.
 */
static void
ATIMach64SetupForMono8x8PatternFill
(
    ScrnInfoPtr  pScreenInfo,
    int          patx,
    int          paty,
    int          fg,
    int          bg,
    int          rop,
    unsigned int planemask
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);

    ATIMach64WaitForFIFO(pATI, 3);
    outf(DP_WRITE_MASK, planemask);
    outf(DP_SRC, DP_MONO_SRC_PATTERN |
        SetBits(SRC_FRGD, DP_FRGD_SRC) | SetBits(SRC_BKGD, DP_BKGD_SRC));
    outf(DP_FRGD_CLR, fg);

    if (bg == -1)
        outf(DP_MIX, SetBits(ATIMach64ALU[rop], DP_FRGD_MIX) |
            SetBits(MIX_DST, DP_BKGD_MIX));
    else
    {
        ATIMach64WaitForFIFO(pATI, 2);
        outf(DP_BKGD_CLR, bg);
        outf(DP_MIX, SetBits(ATIMach64ALU[rop], DP_FRGD_MIX) |
            SetBits(ATIMach64ALU[rop], DP_BKGD_MIX));
    }

    ATIMach64WaitForFIFO(pATI, 4);
    outf(PAT_REG0, patx);
    outf(PAT_REG1, paty);
    outf(PAT_CNTL, PAT_MONO_EN);

    outf(CLR_CMP_CNTL, CLR_CMP_FN_FALSE);

    if (pATI->XModifier == 1)
        outf(DST_CNTL, DST_X_DIR | DST_Y_DIR);
}

/*
 * ATIMach64SubsequentMono8x8PatternFillRect --
 *
 * This function performs an 8x8 1bpp pattern fill.
 */
static void
ATIMach64SubsequentMono8x8PatternFillRect
(
    ScrnInfoPtr pScreenInfo,
    int         patx,
    int         paty,
    int         x,
    int         y,
    int         w,
    int         h
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);

    if (pATI->XModifier != 1)
    {
        x *= pATI->XModifier;
        w *= pATI->XModifier;

        outf(DST_CNTL, SetBits((x / 4) % 6, DST_24_ROT) |
            (DST_X_DIR | DST_Y_DIR | DST_24_ROT_EN));
    }

    /* Disable clipping if it gets in the way */
    ATIMach64ValidateClip(pATI, x, x + w - 1, y, y + h - 1);

    ATIMach64WaitForFIFO(pATI, 2);
    outf(DST_Y_X, SetWord(x, 1) | SetWord(y, 0));
    outf(DST_HEIGHT_WIDTH, SetWord(w, 1) | SetWord(h, 0));
}

/*
 * ATIMach64SetupForScanlineCPUToScreenColorExpandFill --
 *
 * This function sets up the engine for a series of colour expansion fills.
 */
static void
ATIMach64SetupForScanlineCPUToScreenColorExpandFill
(
    ScrnInfoPtr  pScreenInfo,
    int          fg,
    int          bg,
    int          rop,
    unsigned int planemask
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);

    ATIMach64WaitForFIFO(pATI, 3);
    outf(DP_WRITE_MASK, planemask);
    outf(DP_SRC, DP_MONO_SRC_HOST |
        SetBits(SRC_FRGD, DP_FRGD_SRC) | SetBits(SRC_BKGD, DP_BKGD_SRC));
    outf(DP_FRGD_CLR, fg);

    if (bg == -1)
        outf(DP_MIX, SetBits(ATIMach64ALU[rop], DP_FRGD_MIX) |
            SetBits(MIX_DST, DP_BKGD_MIX));
    else
    {
        ATIMach64WaitForFIFO(pATI, 2);
        outf(DP_BKGD_CLR, bg);
        outf(DP_MIX, SetBits(ATIMach64ALU[rop], DP_FRGD_MIX) |
            SetBits(ATIMach64ALU[rop], DP_BKGD_MIX));
    }

    outf(CLR_CMP_CNTL, CLR_CMP_FN_FALSE);

    if (pATI->XModifier == 1)
        outf(DST_CNTL, DST_X_DIR | DST_Y_DIR);
}

/*
 * ATIMach64SubsequentScanlineCPUToScreenColorExpandFill --
 *
 * This function sets up the engine for a single colour expansion fill.
 */
static void
ATIMach64SubsequentScanlineCPUToScreenColorExpandFill
(
    ScrnInfoPtr pScreenInfo,
    int         x,
    int         y,
    int         w,
    int         h,
    int         skipleft
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);

    if (pATI->XModifier != 1)
    {
        x *= pATI->XModifier;
        w *= pATI->XModifier;
        skipleft *= pATI->XModifier;

        outf(DST_CNTL, SetBits((x / 4) % 6, DST_24_ROT) |
            (DST_X_DIR | DST_Y_DIR | DST_24_ROT_EN));
    }

    pATI->ExpansionBitmapWidth = (w + 31) / 32;

    ATIMach64WaitForFIFO(pATI, 3);
    pATI->sc_left = x + skipleft;
    pATI->sc_right = x + w - 1;
    outf(SC_LEFT_RIGHT,
        SetWord(pATI->sc_right, 1) | SetWord(pATI->sc_left, 0));
    outf(DST_Y_X, SetWord(x, 1) | SetWord(y, 0));
    outf(DST_HEIGHT_WIDTH,
        SetWord(pATI->ExpansionBitmapWidth * 32, 1) | SetWord(h, 0));
}

/*
 * ATIMach64SubsequentColorExpandScanline --
 *
 * This function feeds a bitmap scanline to the engine for a colour expansion
 * fill.  This is written to do burst transfers for those platforms that can do
 * them, and to improve CPU/engine concurrency.
 */
static void
ATIMach64SubsequentColorExpandScanline
(
    ScrnInfoPtr pScreenInfo,
    int         iBuffer
)
{
    ATIPtr          pATI         = ATIPTR(pScreenInfo);
    CARD32          *pBitmapData = pATI->ExpansionBitmapScanlinePtr[iBuffer];
    int             w            = pATI->ExpansionBitmapWidth;
    int             nDWord;

    while (w > 0)
    {
        /*
         * Transfers are done in chunks of up to 64 bytes in length (32 on
         * earlier controllers).
         */
        nDWord = w;
        if (nDWord > pATI->nHostFIFOEntries)
            nDWord = pATI->nHostFIFOEntries;

        /* Make enough FIFO slots available */
        ATIMach64WaitForFIFO(pATI, nDWord);

        /*
         * Always start transfers on a chuck-sized boundary.  Note that
         * HOST_DATA_0 is actually on a 512-byte boundary, but *pBitmapData can
         * only be guaranteed to be on a chunk-sized boundary.
         *
         * Transfer current chunk.  With any luck, the compiler won't mangle
         * this too badly...
         */

#       if defined(ATIMove32)

            ATIMove32(pATI->pHOST_DATA, pBitmapData, nDWord);

#       else

        {
            volatile CARD32 *pDst;
            CARD32          *pSrc;
            unsigned int    iDWord;

            iDWord = 16 - nDWord;
            pDst = (volatile CARD32 *)pATI->pHOST_DATA - iDWord;
            pSrc = pBitmapData - iDWord;

            switch (iDWord)
            {
                case  0:  MMIO_MOVE32(pDst +  0, 0, *(pSrc +  0));
                case  1:  MMIO_MOVE32(pDst +  1, 0, *(pSrc +  1));
                case  2:  MMIO_MOVE32(pDst +  2, 0, *(pSrc +  2));
                case  3:  MMIO_MOVE32(pDst +  3, 0, *(pSrc +  3));
                case  4:  MMIO_MOVE32(pDst +  4, 0, *(pSrc +  4));
                case  5:  MMIO_MOVE32(pDst +  5, 0, *(pSrc +  5));
                case  6:  MMIO_MOVE32(pDst +  6, 0, *(pSrc +  6));
                case  7:  MMIO_MOVE32(pDst +  7, 0, *(pSrc +  7));
                case  8:  MMIO_MOVE32(pDst +  8, 0, *(pSrc +  8));
                case  9:  MMIO_MOVE32(pDst +  9, 0, *(pSrc +  9));
                case 10:  MMIO_MOVE32(pDst + 10, 0, *(pSrc + 10));
                case 11:  MMIO_MOVE32(pDst + 11, 0, *(pSrc + 11));
                case 12:  MMIO_MOVE32(pDst + 12, 0, *(pSrc + 12));
                case 13:  MMIO_MOVE32(pDst + 13, 0, *(pSrc + 13));
                case 14:  MMIO_MOVE32(pDst + 14, 0, *(pSrc + 14));
                case 15:  MMIO_MOVE32(pDst + 15, 0, *(pSrc + 15));

                default:    /* Muffle compiler */
                    break;
            }
        }

#       endif

        /* Step to next chunk */
        pBitmapData += nDWord;
        w -= nDWord;
        pATI->nAvailableFIFOEntries -= nDWord;
    }

    pATI->EngineIsBusy = TRUE;
}

/*
 * ATIMach64AccelInit --
 *
 * This function fills in structure fields needed for acceleration on Mach64
 * variants.
 */
int
ATIMach64AccelInit
(
    ATIPtr        pATI,
    XAAInfoRecPtr pXAAInfo
)
{
    /* This doesn't seem quite right... */
    if (pATI->XModifier == 1)
    {
        pXAAInfo->Flags = PIXMAP_CACHE | OFFSCREEN_PIXMAPS;

#ifndef AVOID_CPIO

        if (!pATI->BankInfo.BankSize)

#endif /* AVOID_CPIO */

        {
            pXAAInfo->Flags |= LINEAR_FRAMEBUFFER;
        }
    }

    /* Sync */
    pXAAInfo->Sync = ATIMach64Sync;

    /* Screen-to-screen copy */
    pXAAInfo->SetupForScreenToScreenCopy = ATIMach64SetupForScreenToScreenCopy;
    pXAAInfo->SubsequentScreenToScreenCopy =
        ATIMach64SubsequentScreenToScreenCopy;

    /* Solid fills */
    pXAAInfo->SetupForSolidFill = ATIMach64SetupForSolidFill;
    pXAAInfo->SubsequentSolidFillRect = ATIMach64SubsequentSolidFillRect;

    /* 8x8 mono pattern fills */
    pXAAInfo->Mono8x8PatternFillFlags =

#if X_BYTE_ORDER != X_LITTLE_ENDIAN

        BIT_ORDER_IN_BYTE_MSBFIRST |

#endif /* X_BYTE_ORDER */

        HARDWARE_PATTERN_PROGRAMMED_BITS | HARDWARE_PATTERN_SCREEN_ORIGIN;
    pXAAInfo->SetupForMono8x8PatternFill = ATIMach64SetupForMono8x8PatternFill;
    pXAAInfo->SubsequentMono8x8PatternFillRect =
        ATIMach64SubsequentMono8x8PatternFillRect;

    /*
     * Use scanline version of colour expansion, not only for the non-ix86
     * case, but also to avoid PCI retries.
     */
    pXAAInfo->ScanlineCPUToScreenColorExpandFillFlags =
        LEFT_EDGE_CLIPPING | LEFT_EDGE_CLIPPING_NEGATIVE_X |
        CPU_TRANSFER_PAD_DWORD | SCANLINE_PAD_DWORD;
    if (pATI->XModifier != 1)
        pXAAInfo->ScanlineCPUToScreenColorExpandFillFlags |= TRIPLE_BITS_24BPP;
    pXAAInfo->NumScanlineColorExpandBuffers = 1;

    /* Align bitmap data on a 64-byte boundary */
    pATI->ExpansionBitmapWidth =        /* DWord size in bits */
        ((pATI->displayWidth * pATI->XModifier) + 31) & ~31U;
    pATI->ExpansionBitmapScanlinePtr[1] =
        (CARD32 *)xnfalloc((pATI->ExpansionBitmapWidth >> 3) + 63);
    pATI->ExpansionBitmapScanlinePtr[0] =
        (pointer)(((unsigned long)pATI->ExpansionBitmapScanlinePtr[1] + 63) &
                  ~63UL);
    pXAAInfo->ScanlineColorExpandBuffers =
        (CARD8 **)pATI->ExpansionBitmapScanlinePtr;
    pXAAInfo->SetupForScanlineCPUToScreenColorExpandFill =
        ATIMach64SetupForScanlineCPUToScreenColorExpandFill;
    pXAAInfo->SubsequentScanlineCPUToScreenColorExpandFill =
        ATIMach64SubsequentScanlineCPUToScreenColorExpandFill;
    pXAAInfo->SubsequentColorExpandScanline =
        ATIMach64SubsequentColorExpandScanline;

    /* The engine does not support the following primitives for 24bpp */
    if (pATI->XModifier != 1)
        return ATIMach64MaxY;

    /* Solid lines */
    pXAAInfo->SetupForSolidLine = ATIMach64SetupForSolidLine;
    pXAAInfo->SubsequentSolidHorVertLine = ATIMach64SubsequentSolidHorVertLine;
    pXAAInfo->SubsequentSolidBresenhamLine =
        ATIMach64SubsequentSolidBresenhamLine;

    return ATIMach64MaxY;
}

/*
 * ATIMach64SetCursorColours --
 *
 * Set hardware cursor foreground and background colours.
 */
static void
ATIMach64SetCursorColours
(
    ScrnInfoPtr pScreenInfo,
    int         fg,
    int         bg
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);

    outr(CUR_CLR0, SetBits(fg, CUR_CLR));
    outr(CUR_CLR1, SetBits(bg, CUR_CLR));
}

/*
 * ATIMach64SetCursorPosition --
 *
 * Set position of hardware cursor.
 */
static void
ATIMach64SetCursorPosition
(
    ScrnInfoPtr pScreenInfo,
    int         x,
    int         y
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);
    CARD16 CursorXOffset, CursorYOffset;

    /* Adjust x & y when the cursor is partially obscured */
    if (x < 0)
    {
        if ((CursorXOffset = -x) > 63)
            CursorXOffset = 63;
        x = 0;
    }
    else
    {
        CursorXOffset = pScreenInfo->frameX1 - pScreenInfo->frameX0;
        if (x > CursorXOffset)
            x = CursorXOffset;
        CursorXOffset = 0;
    }

    if (y < 0)
    {
        if ((CursorYOffset = -y) > 63)
            CursorYOffset = 63;
        y = 0;
    }
    else
    {
        CursorYOffset = pScreenInfo->frameY1 - pScreenInfo->frameY0;
        if (y > CursorYOffset)
            y = CursorYOffset;
        CursorYOffset = 0;
    }

    /* Adjust for multiscanned modes */
    if (pScreenInfo->currentMode->Flags & V_DBLSCAN)
        y *= 2;
    if (pScreenInfo->currentMode->VScan > 1)
        y *= pScreenInfo->currentMode->VScan;

    do
    {
        if (CursorYOffset != pATI->CursorYOffset)
        {
            pATI->CursorYOffset = CursorYOffset;
            outr(CUR_OFFSET, ((CursorYOffset << 4) + pATI->CursorOffset) >> 3);
        }
        else if (CursorXOffset == pATI->CursorXOffset)
            break;

        pATI->CursorXOffset = CursorXOffset;
        outr(CUR_HORZ_VERT_OFF, SetBits(CursorXOffset, CUR_HORZ_OFF) |
            SetBits(CursorYOffset, CUR_VERT_OFF));
    } while (0);

    outr(CUR_HORZ_VERT_POSN,
        SetBits(x, CUR_HORZ_POSN) | SetBits(y, CUR_VERT_POSN));
}

/*
 * ATIMach64LoadCursorImage --
 *
 * Copy hardware cursor image into offscreen video memory.
 */
static void
ATIMach64LoadCursorImage
(
    ScrnInfoPtr pScreenInfo,
    CARD8       *pImage
)
{
    ATIPtr           pATI     = ATIPTR(pScreenInfo);
    XAAInfoRecPtr    pXAAInfo = pATI->pXAAInfo;
    CARD32          *pSrc     = (pointer)pImage;
    volatile CARD32 *pDst     = pATI->pCursorImage;

    /* Synchronise video memory accesses */
    if (pXAAInfo && pXAAInfo->NeedToSync)
        (*pXAAInfo->Sync)(pScreenInfo);

#   if defined(ATIMove32)

        ATIMove32(pDst, pSrc, 256);

#   else

        /* This is lengthy, but it does maximise burst modes */
        pDst[  0] = pSrc[  0];  pDst[  1] = pSrc[  1];
        pDst[  2] = pSrc[  2];  pDst[  3] = pSrc[  3];
        pDst[  4] = pSrc[  4];  pDst[  5] = pSrc[  5];
        pDst[  6] = pSrc[  6];  pDst[  7] = pSrc[  7];
        pDst[  8] = pSrc[  8];  pDst[  9] = pSrc[  9];
        pDst[ 10] = pSrc[ 10];  pDst[ 11] = pSrc[ 11];
        pDst[ 12] = pSrc[ 12];  pDst[ 13] = pSrc[ 13];
        pDst[ 14] = pSrc[ 14];  pDst[ 15] = pSrc[ 15];
        pDst[ 16] = pSrc[ 16];  pDst[ 17] = pSrc[ 17];
        pDst[ 18] = pSrc[ 18];  pDst[ 19] = pSrc[ 19];
        pDst[ 20] = pSrc[ 20];  pDst[ 21] = pSrc[ 21];
        pDst[ 22] = pSrc[ 22];  pDst[ 23] = pSrc[ 23];
        pDst[ 24] = pSrc[ 24];  pDst[ 25] = pSrc[ 25];
        pDst[ 26] = pSrc[ 26];  pDst[ 27] = pSrc[ 27];
        pDst[ 28] = pSrc[ 28];  pDst[ 29] = pSrc[ 29];
        pDst[ 30] = pSrc[ 30];  pDst[ 31] = pSrc[ 31];
        pDst[ 32] = pSrc[ 32];  pDst[ 33] = pSrc[ 33];
        pDst[ 34] = pSrc[ 34];  pDst[ 35] = pSrc[ 35];
        pDst[ 36] = pSrc[ 36];  pDst[ 37] = pSrc[ 37];
        pDst[ 38] = pSrc[ 38];  pDst[ 39] = pSrc[ 39];
        pDst[ 40] = pSrc[ 40];  pDst[ 41] = pSrc[ 41];
        pDst[ 42] = pSrc[ 42];  pDst[ 43] = pSrc[ 43];
        pDst[ 44] = pSrc[ 44];  pDst[ 45] = pSrc[ 45];
        pDst[ 46] = pSrc[ 46];  pDst[ 47] = pSrc[ 47];
        pDst[ 48] = pSrc[ 48];  pDst[ 49] = pSrc[ 49];
        pDst[ 50] = pSrc[ 50];  pDst[ 51] = pSrc[ 51];
        pDst[ 52] = pSrc[ 52];  pDst[ 53] = pSrc[ 53];
        pDst[ 54] = pSrc[ 54];  pDst[ 55] = pSrc[ 55];
        pDst[ 56] = pSrc[ 56];  pDst[ 57] = pSrc[ 57];
        pDst[ 58] = pSrc[ 58];  pDst[ 59] = pSrc[ 59];
        pDst[ 60] = pSrc[ 60];  pDst[ 61] = pSrc[ 61];
        pDst[ 62] = pSrc[ 62];  pDst[ 63] = pSrc[ 63];
        pDst[ 64] = pSrc[ 64];  pDst[ 65] = pSrc[ 65];
        pDst[ 66] = pSrc[ 66];  pDst[ 67] = pSrc[ 67];
        pDst[ 68] = pSrc[ 68];  pDst[ 69] = pSrc[ 69];
        pDst[ 70] = pSrc[ 70];  pDst[ 71] = pSrc[ 71];
        pDst[ 72] = pSrc[ 72];  pDst[ 73] = pSrc[ 73];
        pDst[ 74] = pSrc[ 74];  pDst[ 75] = pSrc[ 75];
        pDst[ 76] = pSrc[ 76];  pDst[ 77] = pSrc[ 77];
        pDst[ 78] = pSrc[ 78];  pDst[ 79] = pSrc[ 79];
        pDst[ 80] = pSrc[ 80];  pDst[ 81] = pSrc[ 81];
        pDst[ 82] = pSrc[ 82];  pDst[ 83] = pSrc[ 83];
        pDst[ 84] = pSrc[ 84];  pDst[ 85] = pSrc[ 85];
        pDst[ 86] = pSrc[ 86];  pDst[ 87] = pSrc[ 87];
        pDst[ 88] = pSrc[ 88];  pDst[ 89] = pSrc[ 89];
        pDst[ 90] = pSrc[ 90];  pDst[ 91] = pSrc[ 91];
        pDst[ 92] = pSrc[ 92];  pDst[ 93] = pSrc[ 93];
        pDst[ 94] = pSrc[ 94];  pDst[ 95] = pSrc[ 95];
        pDst[ 96] = pSrc[ 96];  pDst[ 97] = pSrc[ 97];
        pDst[ 98] = pSrc[ 98];  pDst[ 99] = pSrc[ 99];
        pDst[100] = pSrc[100];  pDst[101] = pSrc[101];
        pDst[102] = pSrc[102];  pDst[103] = pSrc[103];
        pDst[104] = pSrc[104];  pDst[105] = pSrc[105];
        pDst[106] = pSrc[106];  pDst[107] = pSrc[107];
        pDst[108] = pSrc[108];  pDst[109] = pSrc[109];
        pDst[110] = pSrc[110];  pDst[111] = pSrc[111];
        pDst[112] = pSrc[112];  pDst[113] = pSrc[113];
        pDst[114] = pSrc[114];  pDst[115] = pSrc[115];
        pDst[116] = pSrc[116];  pDst[117] = pSrc[117];
        pDst[118] = pSrc[118];  pDst[119] = pSrc[119];
        pDst[120] = pSrc[120];  pDst[121] = pSrc[121];
        pDst[122] = pSrc[122];  pDst[123] = pSrc[123];
        pDst[124] = pSrc[124];  pDst[125] = pSrc[125];
        pDst[126] = pSrc[126];  pDst[127] = pSrc[127];
        pDst[128] = pSrc[128];  pDst[129] = pSrc[129];
        pDst[130] = pSrc[130];  pDst[131] = pSrc[131];
        pDst[132] = pSrc[132];  pDst[133] = pSrc[133];
        pDst[134] = pSrc[134];  pDst[135] = pSrc[135];
        pDst[136] = pSrc[136];  pDst[137] = pSrc[137];
        pDst[138] = pSrc[138];  pDst[139] = pSrc[139];
        pDst[140] = pSrc[140];  pDst[141] = pSrc[141];
        pDst[142] = pSrc[142];  pDst[143] = pSrc[143];
        pDst[144] = pSrc[144];  pDst[145] = pSrc[145];
        pDst[146] = pSrc[146];  pDst[147] = pSrc[147];
        pDst[148] = pSrc[148];  pDst[149] = pSrc[149];
        pDst[150] = pSrc[150];  pDst[151] = pSrc[151];
        pDst[152] = pSrc[152];  pDst[153] = pSrc[153];
        pDst[154] = pSrc[154];  pDst[155] = pSrc[155];
        pDst[156] = pSrc[156];  pDst[157] = pSrc[157];
        pDst[158] = pSrc[158];  pDst[159] = pSrc[159];
        pDst[160] = pSrc[160];  pDst[161] = pSrc[161];
        pDst[162] = pSrc[162];  pDst[163] = pSrc[163];
        pDst[164] = pSrc[164];  pDst[165] = pSrc[165];
        pDst[166] = pSrc[166];  pDst[167] = pSrc[167];
        pDst[168] = pSrc[168];  pDst[169] = pSrc[169];
        pDst[170] = pSrc[170];  pDst[171] = pSrc[171];
        pDst[172] = pSrc[172];  pDst[173] = pSrc[173];
        pDst[174] = pSrc[174];  pDst[175] = pSrc[175];
        pDst[176] = pSrc[176];  pDst[177] = pSrc[177];
        pDst[178] = pSrc[178];  pDst[179] = pSrc[179];
        pDst[180] = pSrc[180];  pDst[181] = pSrc[181];
        pDst[182] = pSrc[182];  pDst[183] = pSrc[183];
        pDst[184] = pSrc[184];  pDst[185] = pSrc[185];
        pDst[186] = pSrc[186];  pDst[187] = pSrc[187];
        pDst[188] = pSrc[188];  pDst[189] = pSrc[189];
        pDst[190] = pSrc[190];  pDst[191] = pSrc[191];
        pDst[192] = pSrc[192];  pDst[193] = pSrc[193];
        pDst[194] = pSrc[194];  pDst[195] = pSrc[195];
        pDst[196] = pSrc[196];  pDst[197] = pSrc[197];
        pDst[198] = pSrc[198];  pDst[199] = pSrc[199];
        pDst[200] = pSrc[200];  pDst[201] = pSrc[201];
        pDst[202] = pSrc[202];  pDst[203] = pSrc[203];
        pDst[204] = pSrc[204];  pDst[205] = pSrc[205];
        pDst[206] = pSrc[206];  pDst[207] = pSrc[207];
        pDst[208] = pSrc[208];  pDst[209] = pSrc[209];
        pDst[210] = pSrc[210];  pDst[211] = pSrc[211];
        pDst[212] = pSrc[212];  pDst[213] = pSrc[213];
        pDst[214] = pSrc[214];  pDst[215] = pSrc[215];
        pDst[216] = pSrc[216];  pDst[217] = pSrc[217];
        pDst[218] = pSrc[218];  pDst[219] = pSrc[219];
        pDst[220] = pSrc[220];  pDst[221] = pSrc[221];
        pDst[222] = pSrc[222];  pDst[223] = pSrc[223];
        pDst[224] = pSrc[224];  pDst[225] = pSrc[225];
        pDst[226] = pSrc[226];  pDst[227] = pSrc[227];
        pDst[228] = pSrc[228];  pDst[229] = pSrc[229];
        pDst[230] = pSrc[230];  pDst[231] = pSrc[231];
        pDst[232] = pSrc[232];  pDst[233] = pSrc[233];
        pDst[234] = pSrc[234];  pDst[235] = pSrc[235];
        pDst[236] = pSrc[236];  pDst[237] = pSrc[237];
        pDst[238] = pSrc[238];  pDst[239] = pSrc[239];
        pDst[240] = pSrc[240];  pDst[241] = pSrc[241];
        pDst[242] = pSrc[242];  pDst[243] = pSrc[243];
        pDst[244] = pSrc[244];  pDst[245] = pSrc[245];
        pDst[246] = pSrc[246];  pDst[247] = pSrc[247];
        pDst[248] = pSrc[248];  pDst[249] = pSrc[249];
        pDst[250] = pSrc[250];  pDst[251] = pSrc[251];
        pDst[252] = pSrc[252];  pDst[253] = pSrc[253];
        pDst[254] = pSrc[254];  pDst[255] = pSrc[255];

#endif

}

/*
 * ATIMach64HideCursor --
 *
 * Turn off hardware cursor.
 */
static void
ATIMach64HideCursor
(
    ScrnInfoPtr pScreenInfo
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);

    if (!(pATI->NewHW.gen_test_cntl & GEN_CUR_EN))
        return;

    pATI->NewHW.gen_test_cntl &= ~GEN_CUR_EN;
    out8(GEN_TEST_CNTL, GetByte(pATI->NewHW.gen_test_cntl, 0));
}

/*
 * ATIMach64ShowCursor --
 *
 * Turn on hardware cursor.
 */
static void
ATIMach64ShowCursor
(
    ScrnInfoPtr pScreenInfo
)
{
    ATIPtr pATI = ATIPTR(pScreenInfo);

    if (pATI->NewHW.gen_test_cntl & GEN_CUR_EN)
        return;

    pATI->NewHW.gen_test_cntl |= GEN_CUR_EN;
    out8(GEN_TEST_CNTL, GetByte(pATI->NewHW.gen_test_cntl, 0));
}

/*
 * ATIMach64UseHWCursor --
 *
 * Notify cursor layer whether a hardware cursor is configured.
 */
static Bool
ATIMach64UseHWCursor
(
    ScreenPtr pScreen,
    CursorPtr pCursor
)
{
    ScrnInfoPtr pScreenInfo = xf86Screens[pScreen->myNum];
    ATIPtr      pATI        = ATIPTR(pScreenInfo);

    if (!pATI->CursorBase)
        return FALSE;

#ifndef AVOID_CPIO

    /*
     * For some reason, the hardware cursor isn't vertically scaled when a VGA
     * doublescanned or multiscanned mode is in effect.
     */
    if (pATI->NewHW.crtc == ATI_CRTC_MACH64)
        return TRUE;
    if ((pScreenInfo->currentMode->Flags & V_DBLSCAN) ||
        (pScreenInfo->currentMode->VScan > 1))
        return FALSE;

#endif /* AVOID_CPIO */

    return TRUE;
}

/*
 * ATIMach64CursorInit --
 *
 * Initialise xf86CursorInfoRec fields with information specific to Mach64
 * variants.
 */
Bool
ATIMach64CursorInit
(
    xf86CursorInfoPtr pCursorInfo
)
{
    /*
     * For Mach64 variants, toggling hardware cursors on and off causes
     * display artifacts.  Ask the cursor support layers to always paint the
     * cursor (whether or not it is entirely transparent) and to not hide the
     * cursor when reloading its image.  The three reasons behind turning off
     * the hardware cursor that remain are when it moves to a different screen,
     * on a switch to a software cursor or to a different virtual console.
     */
    pCursorInfo->Flags = HARDWARE_CURSOR_TRUECOLOR_AT_8BPP |
        HARDWARE_CURSOR_INVERT_MASK |
        HARDWARE_CURSOR_SHOW_TRANSPARENT |
        HARDWARE_CURSOR_UPDATE_UNHIDDEN |
        HARDWARE_CURSOR_AND_SOURCE_WITH_MASK |

#if X_BYTE_ORDER != X_LITTLE_ENDIAN

        HARDWARE_CURSOR_BIT_ORDER_MSBFIRST |

#endif /* X_BYTE_ORDER */

        HARDWARE_CURSOR_SOURCE_MASK_INTERLEAVE_1;
    pCursorInfo->MaxWidth = pCursorInfo->MaxHeight = 64;

    pCursorInfo->SetCursorColors = ATIMach64SetCursorColours;
    pCursorInfo->SetCursorPosition = ATIMach64SetCursorPosition;
    pCursorInfo->LoadCursorImage = ATIMach64LoadCursorImage;
    pCursorInfo->HideCursor = ATIMach64HideCursor;
    pCursorInfo->ShowCursor = ATIMach64ShowCursor;
    pCursorInfo->UseHWCursor = ATIMach64UseHWCursor;

    return TRUE;
}