diff --git a/arch/arm/mach-at91rm9200/devices.c b/arch/arm/mach-at91rm9200/devices.c
index 5a3919f09d4d..059824376629 100644
--- a/arch/arm/mach-at91rm9200/devices.c
+++ b/arch/arm/mach-at91rm9200/devices.c
@@ -561,6 +561,7 @@ static struct resource dbgu_resources[] = {
 static struct atmel_uart_data dbgu_data = {
 	.use_dma_tx	= 0,
 	.use_dma_rx	= 0,		/* DBGU not capable of receive DMA */
+	.regs		= (void __iomem *)(AT91_VA_BASE_SYS + AT91_DBGU),
 };
 
 static struct platform_device at91rm9200_dbgu_device = {
diff --git a/arch/avr32/mach-at32ap/at32ap7000.c b/arch/avr32/mach-at32ap/at32ap7000.c
index 05d1296bd7b2..3dd305875087 100644
--- a/arch/avr32/mach-at32ap/at32ap7000.c
+++ b/arch/avr32/mach-at32ap/at32ap7000.c
@@ -523,32 +523,48 @@ void __init at32_add_system_devices(void)
  *  USART
  * -------------------------------------------------------------------- */
 
+static struct atmel_uart_data atmel_usart0_data = {
+	.use_dma_tx	= 1,
+	.use_dma_rx	= 1,
+};
 static struct resource atmel_usart0_resource[] = {
 	PBMEM(0xffe00c00),
 	IRQ(7),
 };
-DEFINE_DEV(atmel_usart, 0);
+DEFINE_DEV_DATA(atmel_usart, 0);
 DEV_CLK(usart, atmel_usart0, pba, 4);
 
+static struct atmel_uart_data atmel_usart1_data = {
+	.use_dma_tx	= 1,
+	.use_dma_rx	= 1,
+};
 static struct resource atmel_usart1_resource[] = {
 	PBMEM(0xffe01000),
 	IRQ(7),
 };
-DEFINE_DEV(atmel_usart, 1);
+DEFINE_DEV_DATA(atmel_usart, 1);
 DEV_CLK(usart, atmel_usart1, pba, 4);
 
+static struct atmel_uart_data atmel_usart2_data = {
+	.use_dma_tx	= 1,
+	.use_dma_rx	= 1,
+};
 static struct resource atmel_usart2_resource[] = {
 	PBMEM(0xffe01400),
 	IRQ(8),
 };
-DEFINE_DEV(atmel_usart, 2);
+DEFINE_DEV_DATA(atmel_usart, 2);
 DEV_CLK(usart, atmel_usart2, pba, 5);
 
+static struct atmel_uart_data atmel_usart3_data = {
+	.use_dma_tx	= 1,
+	.use_dma_rx	= 1,
+};
 static struct resource atmel_usart3_resource[] = {
 	PBMEM(0xffe01800),
 	IRQ(9),
 };
-DEFINE_DEV(atmel_usart, 3);
+DEFINE_DEV_DATA(atmel_usart, 3);
 DEV_CLK(usart, atmel_usart3, pba, 6);
 
 static inline void configure_usart0_pins(void)
@@ -597,8 +613,13 @@ static struct platform_device *setup_usart(unsigned int id)
 		configure_usart3_pins();
 		break;
 	default:
-		pdev = NULL;
-		break;
+		return NULL;
+	}
+
+	if (PXSEG(pdev->resource[0].start) == P4SEG) {
+		/* Addresses in the P4 segment are permanently mapped 1:1 */
+		struct atmel_uart_data *data = pdev->dev.platform_data;
+		data->regs = (void __iomem *)pdev->resource[0].start;
 	}
 
 	return pdev;
diff --git a/drivers/serial/atmel_serial.c b/drivers/serial/atmel_serial.c
index 2f9d99bb9587..34212df39800 100644
--- a/drivers/serial/atmel_serial.c
+++ b/drivers/serial/atmel_serial.c
@@ -694,8 +694,9 @@ static void __devinit atmel_init_port(struct atmel_uart_port *atmel_port, struct
 	port->mapbase	= pdev->resource[0].start;
 	port->irq	= pdev->resource[1].start;
 
-	if (port->mapbase == AT91_VA_BASE_SYS + AT91_DBGU)		/* Part of system perpherals - already mapped */
-		port->membase = (void __iomem *) port->mapbase;
+	if (data->regs)
+		/* Already mapped by setup code */
+		port->membase = data->regs;
 	else {
 		port->flags	|= UPF_IOREMAP;
 		port->membase	= NULL;
diff --git a/include/asm-arm/arch-at91rm9200/board.h b/include/asm-arm/arch-at91rm9200/board.h
index d56527055c9d..3cc9aec80f9d 100644
--- a/include/asm-arm/arch-at91rm9200/board.h
+++ b/include/asm-arm/arch-at91rm9200/board.h
@@ -103,6 +103,7 @@ extern void __init at91_init_serial(struct at91_uart_config *config);
 struct atmel_uart_data {
 	short		use_dma_tx;	/* use transmit DMA? */
 	short		use_dma_rx;	/* use receive DMA? */
+	void __iomem	*regs;		/* virtual base address, if any */
 };
 extern void __init at91_add_device_serial(void);
 
diff --git a/include/asm-avr32/arch-at32ap/board.h b/include/asm-avr32/arch-at32ap/board.h
index 82e5404d2f48..435507281f89 100644
--- a/include/asm-avr32/arch-at32ap/board.h
+++ b/include/asm-avr32/arch-at32ap/board.h
@@ -12,6 +12,11 @@ void at32_add_system_devices(void);
 #define ATMEL_MAX_UART	4
 extern struct platform_device *atmel_default_console_device;
 
+struct atmel_uart_data {
+	short		use_dma_tx;	/* use transmit DMA? */
+	short		use_dma_rx;	/* use receive DMA? */
+	void __iomem	*regs;		/* virtual base address, if any */
+};
 struct platform_device *at32_add_device_usart(unsigned int id);
 
 struct eth_platform_data {