diff --git a/docs/development/targets/timer-dma-conflicts.md b/docs/development/targets/timer-dma-conflicts.md index c8910d63403..c9356fbaf5b 100644 --- a/docs/development/targets/timer-dma-conflicts.md +++ b/docs/development/targets/timer-dma-conflicts.md @@ -150,6 +150,120 @@ STM32H7 uses DMAMUX which eliminates most DMA conflicts: - Fewer headaches for target developers - Conflicts still possible but rare, and some channels (e.g. `TIM15_CH2`, see above) have no DMA request line at all regardless of DMAMUX +## UART Receive Through DMA + +A UART can receive through a DMA stream instead of taking an interrupt for every byte. On +F4, F7, H7 and AT32F43x every UART does so where a stream is free: the firmware picks one +when the port opens. A target can name the stream instead, or keep a port on the interrupt: + +```c +#define UART2_RX_DMA DMA_TAG(2, 5, 0) // DMA2 stream 5 +#define UART3_RX_DMA DMA_NONE // always the byte interrupt +``` + +**What it changes.** The stream writes into the port's receive ring by itself, so a busy +port no longer costs an interrupt per byte, and bytes that arrive while interrupts are +held off (an internal flash write, for instance) are not lost. The ring and its size do +not change. On an H7 at 480 MHz the saving measured around 240 cycles per byte, a fraction +of a percent of the CPU; relatively it is larger on a slower MCU with a busy port. + +**Which ports use it.** Those whose owner reads through the ring: GPS, MSP, telemetry and +the like, and CRSF and SBUS receivers that are not half duplex: they find their frames by +timing, so the stream hands each frame to their callback in one go when the line goes idle, +every byte timed back from the end of the burst. The other serial receivers (IBUS, GHST, +SRXL2, ...), and CRSF or SBUS in half duplex, take each byte as it lands, so their port stays +on the interrupt even when the target names a stream for it. Half-duplex ports (IBUS telemetry and +an SRXL2 ESC always; SmartPort, HoTT, VTX control and others when set to half-duplex) get no +stream unless the target names one. + +### Choosing the stream + +**F4 and F7.** Each UART's receiver is wired to fixed streams on a fixed channel, and the +build fails if the tag names any other: + +| UART | Stream and channel | +|------|--------------------| +| UART1 | `DMA_TAG(2, 2, 4)` or `DMA_TAG(2, 5, 4)` | +| UART2 | `DMA_TAG(1, 5, 4)` | +| UART3 | `DMA_TAG(1, 1, 4)` | +| UART4 | `DMA_TAG(1, 2, 4)` | +| UART5 | `DMA_TAG(1, 0, 4)` | +| UART6 | `DMA_TAG(2, 1, 5)` or `DMA_TAG(2, 2, 5)` | +| UART7 | `DMA_TAG(1, 3, 5)` | +| UART8 | `DMA_TAG(1, 6, 5)` | + +**H7 and AT32F43x.** The DMAMUX connects any UART to any stream (channel, on the AT32), +so only the controller and the stream matter; leave the last field 0. + +**Picked at runtime.** A port with no stream named takes the first free one among its +candidates: on F4 and F7 the streams in the table, in order; on H7 every stream except +DMA2 streams 0 to 2 (the ADCs); on AT32F43x every channel except the ADC's. It skips the +streams a target names for any port, the SD card's, and its own other direction's. A named +stream is used if it is free, and otherwise the port stays on the interrupt. + +**The stream has to be free on that board.** The streams are shared, so this is a +property of the target rather than of the MCU: + +- *Timer outputs.* A stream a timer output will use is refused, and the port stays on the + interrupt, so a motor never loses DSHOT to a serial port: + - with the motors on DSHOT, every output's stream until the motors start, and afterwards the + streams the motors took. Ports opened before the motors (MSP, DJI HD OSD, SmartPort + master, LOG) therefore get no output's stream on a DSHOT board; + - the LED strip's pad, when that feature is on. + + Outputs that end up as servos or unused, and every output except the LED strip's when the + motors are not on DSHOT, leave their streams to the serial ports. This is decided at each + boot: after changing the motor protocol, the mixer, the output modes or the LED strip + feature, a port gets or gives back its stream at the next reboot. On an H7 the `dmavar` of + `DEF_TIM()` is the stream: 0 to 7 are DMA1 streams 0 to 7, 8 to 15 are DMA2 streams 0 to 7. +- *The ADC* is not checked against a named stream, so avoid its stream (the runtime pick + does): on F4 and AT32 `ADC1_DMA_STREAM` where the + target sets it, otherwise DMA2 stream 0 and DMA2 channel 1; DMA2 stream 0 on F7; DMA2 + stream 0, 1 or 2 on H7, for `ADC_INSTANCE` ADC1, ADC2 or ADC3. +- *The SD card* on F4 and F7 SDIO uses DMA2 stream 3 or 6 (`SDCARD_SDIO_DMA`). The runtime + pick avoids it; a named stream is not checked against it, and if a UART takes it first the + card is left unused. +- *Two UARTs* cannot share one; the second keeps its interrupt. + +On F7 and H7 the ring of every port that may receive through DMA (all but those set to +`DMA_NONE`) goes in `DMA_RAM`: on an H7 that is D2 SRAM, where the data cache does not +reach, and on an F7 ordinary RAM, or the uncached region set aside for DMA once the data +cache is on. + +## UART Transmit Through DMA + +The same for sending: every UART sends through a free stream, picked as above, unless +`UARTx_TX_DMA` names one or is set to `DMA_NONE`. + +```c +#define UART2_TX_DMA DMA_TAG(2, 7, 0) // DMA2 stream 7 +``` + +**What it changes.** The port takes one interrupt per transfer instead of one per byte. A +message written whole with `serialWriteBuf()`, as MSP and MAVLink do, goes out in one +transfer; bytes written one at a time, as the serial blackbox does, go out a few transfers +at a time, since whatever is queued while one runs goes in the next. On an H7 at 480 MHz, +MAVLink at 921600 baud sending 12.6 kB/s took 12 959 interrupts a second, 0.66% of the CPU, +and 144 through DMA, 0.006%. It is worth a stream on a port that sends a lot: serial +blackbox, MAVLink, MSP DisplayPort. + +**F4 and F7.** Each UART's transmitter is wired to fixed streams too: + +| UART | Stream and channel | +|------|--------------------| +| UART1 | `DMA_TAG(2, 7, 4)` | +| UART2 | `DMA_TAG(1, 6, 4)` | +| UART3 | `DMA_TAG(1, 3, 4)` or `DMA_TAG(1, 4, 7)` | +| UART4 | `DMA_TAG(1, 4, 4)` | +| UART5 | `DMA_TAG(1, 7, 4)` | +| UART6 | `DMA_TAG(2, 6, 5)` or `DMA_TAG(2, 7, 5)` | +| UART7 | `DMA_TAG(1, 1, 5)` | +| UART8 | `DMA_TAG(1, 0, 5)` | + +On H7 and AT32F43x any free stream (channel) will do, and the rules for a free one are the +ones above. A port's transmit stream has to be a different one from its receive stream, +and on an H7 its transmit ring moves to D2 SRAM as well, unless set to `DMA_NONE`. + ## Related Documentation - **overview.md** - Target system basics diff --git a/src/main/drivers/dma.h b/src/main/drivers/dma.h index 10ece999253..49a449eacb4 100644 --- a/src/main/drivers/dma.h +++ b/src/main/drivers/dma.h @@ -30,6 +30,8 @@ typedef uint32_t DMA_TypeDef; // DMA Tag,contains DMA id\DMA stream\DMA channel #define DMA_TAG(dma, stream, channel) ( (((dma) & 0x03) << 12) | (((stream) & 0x0F) << 8) | (((channel) & 0xFF) << 0) ) #define DMA_NONE (0) +// No DMA_TAG() reaches these bits: a stream picked at runtime, where one is free +#define DMA_TAG_AUTO (0xFFFF0000U) #define DMATAG_GET_DMA(x) ( ((x) >> 12) & 0x03 ) #define DMATAG_GET_STREAM(x) ( ((x) >> 8) & 0x0F ) diff --git a/src/main/drivers/pwm_mapping.c b/src/main/drivers/pwm_mapping.c index 06cdf5ea279..897284587ad 100644 --- a/src/main/drivers/pwm_mapping.c +++ b/src/main/drivers/pwm_mapping.c @@ -526,12 +526,14 @@ static void pwmInitServos(timMotorServoHardware_t * timOutputs) static timMotorServoHardware_t timOutputsStatic; +static bool motorsInitialised = false; bool pwmMotorAndServoInit(void) { pwmBuildTimerOutputList(&timOutputsStatic); pwmInitMotors(&timOutputsStatic); pwmInitServos(&timOutputsStatic); + motorsInitialised = true; return (pwmInitError == PWM_INIT_ERROR_NONE); } @@ -540,6 +542,27 @@ const timMotorServoHardware_t *pwmGetOutputAssignment(void) return &timOutputsStatic; } +bool pwmIsDmaStreamReserved(DMA_t dma) +{ + // Started DSHOT motors own their streams; before they start nothing can have changed the protocol + const bool motorsWillClaim = !motorsInitialised && getMotorProtocolProperties(motorConfig()->motorPwmProtocol)->isDSHOT; + + for (int i = 0; i < timerHardwareCount; i++) { + const timerHardware_t *timHw = &timerHardware[i]; + if (dmaGetByTag(timHw->dmaTag) != dma) { + continue; + } + if (motorsWillClaim) { + return true; + } + // The LED strip claims its stream only once it is enabled, late in init + if (feature(FEATURE_LED_STRIP) && (TIM_IS_LED(timHw->usageFlags) || timerOverrides(timer2id(timHw->tim))->outputMode == OUTPUT_MODE_LED)) { + return true; + } + } + return false; +} + // Upper bound for timerHardware[] size across all supported targets. // timerHardwareCount is a runtime value; this constant prevents a VLA on the MSP // task stack. Targets with 22+ entries (e.g. OMNIBUSF4) need more than MAX_PWM_OUTPUTS. diff --git a/src/main/drivers/pwm_mapping.h b/src/main/drivers/pwm_mapping.h index 158ab8ce6b5..4e1754ca264 100644 --- a/src/main/drivers/pwm_mapping.h +++ b/src/main/drivers/pwm_mapping.h @@ -103,5 +103,7 @@ pwmInitError_e getPwmInitError(void); const char * getPwmInitErrorMessage(void); #ifndef SITL_BUILD const timMotorServoHardware_t *pwmGetOutputAssignment(void); +// A stream a timer output will claim later in init; one it already owns is not counted +bool pwmIsDmaStreamReserved(DMA_t dma); void pwmCalculateAssignment(timMotorServoHardware_t *out, const uint8_t *proposedModes); #endif // SITL_BUILD diff --git a/src/main/drivers/serial.c b/src/main/drivers/serial.c index 1cba2609914..9cad0aec7b9 100644 --- a/src/main/drivers/serial.c +++ b/src/main/drivers/serial.c @@ -20,6 +20,11 @@ #include "platform.h" +#include "build/atomic.h" + +#include "drivers/nvic.h" +#include "drivers/time.h" + #include "serial.h" void serialPrint(serialPort_t *instance, const char *str) @@ -133,3 +138,37 @@ bool serialIsIdle(serialPort_t *instance) else return false; } + +// For an rxCallback; the port can still be NULL to a receiver whose first burst lands before openSerialPort() returns +timeUs_t serialRxByteTimeUs(const serialPort_t *instance) +{ + return (instance && instance->rxBursts) ? instance->rxByteTimeUs : microsISR(); +} + +// Swaps in a larger receive buffer; what the old one held is dropped +void serialSetRxBuffer(serialPort_t *instance, volatile uint8_t *buffer, uint32_t size) +{ + if (size <= instance->rxBufferSize) { + return; + } + + if (instance->vTable->setRxBuffer) { + instance->vTable->setRxBuffer(instance, buffer, size); + return; + } + + // The receive interrupt must never see the new buffer with the old size or indices + ATOMIC_BLOCK(NVIC_PRIO_MAX) { + instance->rxBuffer = buffer; + instance->rxBufferSize = size; + instance->rxBufferHead = 0; + instance->rxBufferTail = 0; + } +} + +void serialRelease(serialPort_t *instance) +{ + if (instance->vTable->release) { + instance->vTable->release(instance); + } +} diff --git a/src/main/drivers/serial.h b/src/main/drivers/serial.h index 6ca0a7d1ec2..e65724c3f43 100644 --- a/src/main/drivers/serial.h +++ b/src/main/drivers/serial.h @@ -16,6 +16,7 @@ */ #pragma once +#include "common/time.h" #include "drivers/io.h" typedef struct { @@ -53,6 +54,9 @@ typedef enum portOptions_t { SERIAL_LONGSTOP = 0 << 6, SERIAL_SHORTSTOP = 1 << 6, + + // The rxCallback takes bytes in bursts at the end of a frame and times them with serialRxByteTimeUs() + SERIAL_RX_BURSTS = 1 << 7, } portOptions_t; typedef void (*serialReceiveCallbackPtr)(uint16_t data, void *rxCallbackData); // used by serial drivers to return frames to app @@ -78,6 +82,9 @@ typedef struct serialPort_s { serialReceiveCallbackPtr rxCallback; void *rxCallbackData; + + bool rxBursts; // rxCallback gets the bytes in bursts, rxByteTimeUs timing each + timeUs_t rxByteTimeUs; // when the byte being handed to rxCallback ended on the wire } serialPort_t; struct serialPortVTable { @@ -106,6 +113,12 @@ struct serialPortVTable { // Optional functions used to buffer large writes. void (*beginWrite)(serialPort_t *instance); void (*endWrite)(serialPort_t *instance); + + // Optional: for a port whose reception is more than the ring, such as a DMA stream + void (*setRxBuffer)(serialPort_t *instance, volatile uint8_t *buffer, uint32_t size); + + // Optional: gives back what the port holds besides its pins, such as DMA streams, once no function uses it + void (*release)(serialPort_t *instance); }; void serialWrite(serialPort_t *instance, uint8_t ch); @@ -122,6 +135,9 @@ void serialPrint(serialPort_t *instance, const char *str); uint32_t serialGetBaudRate(serialPort_t *instance); bool serialIsConnected(const serialPort_t *instance); bool serialIsIdle(serialPort_t *instance); +timeUs_t serialRxByteTimeUs(const serialPort_t *instance); +void serialSetRxBuffer(serialPort_t *instance, volatile uint8_t *buffer, uint32_t size); +void serialRelease(serialPort_t *instance); // A shim that adapts the bufWriter API to the serialWriteBuf() API. void serialWriteBufShim(void *instance, const uint8_t *data, int count); diff --git a/src/main/drivers/serial_uart.c b/src/main/drivers/serial_uart.c index e2009331c6b..878d2fb1fb7 100644 --- a/src/main/drivers/serial_uart.c +++ b/src/main/drivers/serial_uart.c @@ -135,6 +135,10 @@ serialPort_t *uartOpen(USART_TypeDef *USARTx, serialReceiveCallbackPtr rxCallbac return (serialPort_t *)s; } + // Streams still running would go on with the rings about to be reset, and a receiver's with the old callback + uartTxDmaStop(s); + uartRxDmaStop(s); + // common serial initialisation code should move to serialPort::init() s->port.rxBufferHead = s->port.rxBufferTail = 0; s->port.txBufferHead = s->port.txBufferTail = 0; @@ -147,12 +151,12 @@ serialPort_t *uartOpen(USART_TypeDef *USARTx, serialReceiveCallbackPtr rxCallbac uartReconfigure(s); - if (mode & MODE_RX) { + if ((mode & MODE_RX) && !uartRxDmaStart(s)) { USART_ClearITPendingBit(s->USARTx, USART_IT_RXNE); USART_ITConfig(s->USARTx, USART_IT_RXNE, ENABLE); } - if (mode & MODE_TX) { + if ((mode & MODE_TX) && !uartTxDmaStart(s)) { USART_ITConfig(s->USARTx, USART_IT_TXE, ENABLE); } @@ -186,10 +190,12 @@ uint32_t uartTotalRxBytesWaiting(const serialPort_t *instance) { const uartPort_t *s = (const uartPort_t*)instance; - if (s->port.rxBufferHead >= s->port.rxBufferTail) { - return s->port.rxBufferHead - s->port.rxBufferTail; + const uint32_t head = uartRxBufferHead(s); + + if (head >= s->port.rxBufferTail) { + return head - s->port.rxBufferTail; } else { - return s->port.rxBufferSize + s->port.rxBufferHead - s->port.rxBufferTail; + return s->port.rxBufferSize + head - s->port.rxBufferTail; } } @@ -229,17 +235,52 @@ uint8_t uartRead(serialPort_t *instance) return ch; } -void uartWrite(serialPort_t *instance, uint8_t ch) +static void uartQueue(uartPort_t *s, uint8_t ch) { - uartPort_t *s = (uartPort_t *)instance; s->port.txBuffer[s->port.txBufferHead] = ch; if (s->port.txBufferHead + 1 >= s->port.txBufferSize) { s->port.txBufferHead = 0; } else { s->port.txBufferHead++; } +} - USART_ITConfig(s->USARTx, USART_IT_TXE, ENABLE); +void uartWrite(serialPort_t *instance, uint8_t ch) +{ + uartPort_t *s = (uartPort_t *)instance; + uartQueue(s, ch); + + if (uartTxDmaRunning(s)) { + uartStartTxDMA(s); + } else { + USART_ITConfig(s->USARTx, USART_IT_TXE, ENABLE); + } +} + +// Queued whole before the stream starts, so a message goes out in one transfer +static void uartWriteBuf(serialPort_t *instance, const void *data, int count) +{ + uartPort_t *s = (uartPort_t *)instance; + const uint8_t *p = data; + + for (; count > 0; count--, p++) { + while (!uartTotalTxBytesFree(instance)) { + // The ring is full: sending is what makes room + if (uartTxDmaRunning(s)) { + uartStartTxDMA(s); + } + } + + if (uartTxDmaRunning(s)) { + uartQueue(s, *p); + } else { + uartWrite(instance, *p); + } + } + + if (uartTxDmaRunning(s)) { + uartStartTxDMA(s); + } } bool isUartIdle(serialPort_t *instance) @@ -264,9 +305,11 @@ const struct serialPortVTable uartVTable[] = { .setMode = uartSetMode, .setOptions = uartSetOptions, .isConnected = NULL, - .writeBuf = NULL, + .writeBuf = uartWriteBuf, .beginWrite = NULL, .endWrite = NULL, .isIdle = isUartIdle, + .setRxBuffer = uartSetRxBuffer, + .release = uartRelease, } }; diff --git a/src/main/drivers/serial_uart.h b/src/main/drivers/serial_uart.h index 4c1e6d57508..ee6fdd8fd71 100644 --- a/src/main/drivers/serial_uart.h +++ b/src/main/drivers/serial_uart.h @@ -20,11 +20,10 @@ #define UART_AF(uart, af) CONCAT3(GPIO_AF, af, _ ## uart) // Since serial ports can be used for any function these buffer sizes should be equal -// The two largest things that need to be sent are: 1, MSP responses, 2, UBLOX SVINFO packet. +// The largest thing that needs to be sent is an MSP response. -// Size must be a power of two due to various optimizations which use 'and' instead of 'mod' -// Various serial routines return the buffer occupied size as uint8_t which would need to be extended in order to -// increase size further. +// Size must be a power of two due to various optimizations which use 'and' instead of 'mod'. +// The GPS port gets a larger receive buffer of its own, see gps.c. #define UART1_RX_BUFFER_SIZE 256 #define UART1_TX_BUFFER_SIZE 256 #define UART2_RX_BUFFER_SIZE 256 @@ -66,6 +65,13 @@ typedef struct { USART_TypeDef *USARTx; #endif +#ifdef USE_UART_RX_DMA + struct dmaChannelDescriptor_s *rxDma; // the stream filling rxBuffer, NULL where the byte interrupt does +#endif +#ifdef USE_UART_TX_DMA + struct dmaChannelDescriptor_s *txDma; // the stream emptying txBuffer, NULL where the byte interrupt does + volatile uint32_t txDmaCount; // bytes of the transfer on its way, 0 when the stream is idle +#endif } uartPort_t; void uartGetPortPins(UARTDevice_e device, serialPortPins_t * pins); diff --git a/src/main/drivers/serial_uart_at32f43x.c b/src/main/drivers/serial_uart_at32f43x.c index 438af01c665..5093f68e2e4 100644 --- a/src/main/drivers/serial_uart_at32f43x.c +++ b/src/main/drivers/serial_uart_at32f43x.c @@ -357,9 +357,280 @@ static uartDevice_t* uartHardwareMap[] = { #endif }; +#if defined(USE_UART_RX_DMA) || defined(USE_UART_TX_DMA) +static UARTDevice_e uartDeviceOf(const uartPort_t *s) +{ + for (int device = 0; device < UARTDEV_MAX; device++) { + if (uartHardwareMap[device] && &uartHardwareMap[device]->port == s) { + return device; + } + } + return UARTDEV_MAX; +} + +#endif + +#ifdef USE_UART_RX_DMA +typedef struct { + dmaTag_t tag; + uint32_t request; // this UART's receiver on the DMAMUX +} uartRxDmaConfig_t; + +static const uartRxDmaConfig_t uartRxDmaConfig[UARTDEV_MAX] = { +#ifdef UART1_RX_DMA + [UARTDEV_1] = { UART1_RX_DMA, DMAMUX_DMAREQ_ID_USART1_RX }, +#endif +#ifdef UART2_RX_DMA + [UARTDEV_2] = { UART2_RX_DMA, DMAMUX_DMAREQ_ID_USART2_RX }, +#endif +#ifdef UART3_RX_DMA + [UARTDEV_3] = { UART3_RX_DMA, DMAMUX_DMAREQ_ID_USART3_RX }, +#endif +#ifdef UART4_RX_DMA + [UARTDEV_4] = { UART4_RX_DMA, DMAMUX_DMAREQ_ID_UART4_RX }, +#endif +#ifdef UART5_RX_DMA + [UARTDEV_5] = { UART5_RX_DMA, DMAMUX_DMAREQ_ID_UART5_RX }, +#endif +#ifdef UART6_RX_DMA + [UARTDEV_6] = { UART6_RX_DMA, DMAMUX_DMAREQ_ID_USART6_RX }, +#endif +#ifdef UART7_RX_DMA + [UARTDEV_7] = { UART7_RX_DMA, DMAMUX_DMAREQ_ID_UART7_RX }, +#endif +#ifdef UART8_RX_DMA + [UARTDEV_8] = { UART8_RX_DMA, DMAMUX_DMAREQ_ID_UART8_RX }, +#endif +}; + +#define UART_RX_DMA_FLAGS (DMA_IT_TCIF | DMA_IT_HTIF | DMA_IT_DMEIF) + +void uartRxDmaStop(uartPort_t *s) +{ + if (s->rxDma) { + usart_dma_receiver_enable(s->USARTx, FALSE); + usart_interrupt_enable(s->USARTx, USART_IDLE_INT, FALSE); + dma_interrupt_enable(s->rxDma->ref, DMA_FDT_INT | DMA_HDT_INT, FALSE); + dma_channel_enable(s->rxDma->ref, FALSE); + DMA_CLEAR_FLAG(s->rxDma, UART_RX_DMA_FLAGS); + uartDmaRelease(s->rxDma); + s->rxDma = NULL; + s->port.rxBursts = false; + } +} + +// Half and full ring: a burst longer than half the ring is handed over before the stream overwrites it +static void uartRxDmaHandler(DMA_t dma) +{ + DMA_CLEAR_FLAG(dma, UART_RX_DMA_FLAGS); + uartRxDmaDeliver((uartPort_t *)dma->userParam, microsISR()); +} + +bool uartRxDmaStart(uartPort_t *s) +{ + uartRxDmaStop(s); + + const UARTDevice_e device = uartDeviceOf(s); + if (device == UARTDEV_MAX || !(s->port.mode & MODE_RX) || uartRxDmaRefused(s)) { + return false; + } + + const dmaTag_t tag = uartDmaPick(uartRxDmaConfig[device].tag, NULL, device, &s->port, uartTxDmaOf(s)); + const DMA_t dma = dmaGetByTag(tag); + if (!dma) { + return false; + } + + dmaInit(dma, OWNER_SERIAL, RESOURCE_INDEX(device)); + dma_reset(dma->ref); + + dma_init_type init; + dma_default_para_init(&init); + init.peripheral_base_addr = (uint32_t)&s->USARTx->dt; + init.memory_base_addr = (uint32_t)s->port.rxBuffer; + init.direction = DMA_DIR_PERIPHERAL_TO_MEMORY; + init.buffer_size = s->port.rxBufferSize; + init.peripheral_inc_enable = FALSE; + init.memory_inc_enable = TRUE; + init.peripheral_data_width = DMA_PERIPHERAL_DATA_WIDTH_BYTE; + init.memory_data_width = DMA_MEMORY_DATA_WIDTH_BYTE; + init.loop_mode_enable = TRUE; + init.priority = DMA_PRIORITY_MEDIUM; + dma_init(dma->ref, &init); + dmaMuxEnable(dma, uartRxDmaConfig[device].request); + + s->port.rxBufferHead = s->port.rxBufferTail = 0; + s->rxDma = dma; + + if (s->port.rxCallback) { + // The UART's priority, so the two never hand bytes over at once + dmaSetHandler(dma, uartRxDmaHandler, NVIC_PRIO_SERIALUART, (uint32_t)s); + DMA_CLEAR_FLAG(dma, UART_RX_DMA_FLAGS); + dma_interrupt_enable(dma->ref, DMA_FDT_INT | DMA_HDT_INT, TRUE); + s->port.rxBursts = true; + usart_interrupt_enable(s->USARTx, USART_IDLE_INT, TRUE); + } + + dma_channel_enable(dma->ref, TRUE); + + usart_interrupt_enable(s->USARTx, USART_RDBF_INT, FALSE); + usart_dma_receiver_enable(s->USARTx, TRUE); + return true; +} +#endif + +#ifdef USE_UART_TX_DMA +typedef struct { + dmaTag_t tag; + uint32_t request; // this UART's transmitter on the DMAMUX +} uartTxDmaConfig_t; + +static const uartTxDmaConfig_t uartTxDmaConfig[UARTDEV_MAX] = { +#ifdef UART1_TX_DMA + [UARTDEV_1] = { UART1_TX_DMA, DMAMUX_DMAREQ_ID_USART1_TX }, +#endif +#ifdef UART2_TX_DMA + [UARTDEV_2] = { UART2_TX_DMA, DMAMUX_DMAREQ_ID_USART2_TX }, +#endif +#ifdef UART3_TX_DMA + [UARTDEV_3] = { UART3_TX_DMA, DMAMUX_DMAREQ_ID_USART3_TX }, +#endif +#ifdef UART4_TX_DMA + [UARTDEV_4] = { UART4_TX_DMA, DMAMUX_DMAREQ_ID_UART4_TX }, +#endif +#ifdef UART5_TX_DMA + [UARTDEV_5] = { UART5_TX_DMA, DMAMUX_DMAREQ_ID_UART5_TX }, +#endif +#ifdef UART6_TX_DMA + [UARTDEV_6] = { UART6_TX_DMA, DMAMUX_DMAREQ_ID_USART6_TX }, +#endif +#ifdef UART7_TX_DMA + [UARTDEV_7] = { UART7_TX_DMA, DMAMUX_DMAREQ_ID_UART7_TX }, +#endif +#ifdef UART8_TX_DMA + [UARTDEV_8] = { UART8_TX_DMA, DMAMUX_DMAREQ_ID_UART8_TX }, +#endif +}; + +// Full, half transfer and error flags; the interrupt enables sit at the same bits +#define UART_TX_DMA_FLAGS (DMA_IT_TCIF | DMA_IT_HTIF | DMA_IT_DMEIF) +#define UART_TX_DMA_ENDED (DMA_IT_TCIF | DMA_IT_DMEIF) + +void uartTxDmaStop(uartPort_t *s) +{ + if (s->txDma) { + usart_dma_transmitter_enable(s->USARTx, FALSE); + dma_interrupt_enable(s->txDma->ref, UART_TX_DMA_ENDED, FALSE); + dma_channel_enable(s->txDma->ref, FALSE); + DMA_CLEAR_FLAG(s->txDma, UART_TX_DMA_FLAGS); + uartDmaRelease(s->txDma); + s->txDma = NULL; + s->txDmaCount = 0; + } +} + +void uartStartTxDMA(uartPort_t *s) +{ + // Masked: the end-of-transfer interrupt also starts the next one + ATOMIC_BLOCK(NVIC_PRIO_SERIALUART) { + const uint32_t head = s->port.txBufferHead; + const uint32_t tail = s->port.txBufferTail; + + if (s->txDma && !s->txDmaCount && head != tail) { + // Up to the head, or to the end of the ring if the queue wraps: the rest follows + const uint32_t count = (head > tail) ? head - tail : s->port.txBufferSize - tail; + + s->txDmaCount = count; + // Unlike an STM32 stream, a channel stays enabled after its transfer, and + // takes a new count and address only while disabled + dma_channel_enable(s->txDma->ref, FALSE); + s->txDma->ref->maddr = (uint32_t)&s->port.txBuffer[tail]; + dma_data_number_set(s->txDma->ref, count); + dma_channel_enable(s->txDma->ref, TRUE); + } + } +} + +// An error ends the transfer too, or the port would wait for it forever +static void uartTxDmaHandler(DMA_t dma) +{ + uartPort_t *s = (uartPort_t *)dma->userParam; + + // Clear every flag: one left set re-raises the interrupt, starving USB at this priority + const bool ended = DMA_GET_FLAG_STATUS(dma, UART_TX_DMA_ENDED); + DMA_CLEAR_FLAG(dma, UART_TX_DMA_FLAGS); + + if (ended) { + // After an error the rest goes again, unless nothing went: that error would only repeat + const uint32_t sent = s->txDmaCount - dma_data_number_get(dma->ref); + s->port.txBufferTail = (s->port.txBufferTail + (sent ? sent : s->txDmaCount)) % s->port.txBufferSize; + s->txDmaCount = 0; + uartStartTxDMA(s); + } +} + +bool uartTxDmaStart(uartPort_t *s) +{ + uartTxDmaStop(s); + + const UARTDevice_e device = uartDeviceOf(s); + if (device == UARTDEV_MAX || !(s->port.mode & MODE_TX)) { + return false; + } + + const dmaTag_t tag = uartDmaPick(uartTxDmaConfig[device].tag, NULL, device, &s->port, uartRxDmaOf(s)); + const DMA_t dma = dmaGetByTag(tag); + if (!dma) { + return false; + } +#ifdef USE_UART_RX_DMA + // The channel this port receives on is this UART's too, so it looks free to it + if (dma == s->rxDma) { + return false; + } +#endif + + dmaInit(dma, OWNER_SERIAL, RESOURCE_INDEX(device)); + dmaSetHandler(dma, uartTxDmaHandler, NVIC_PRIO_SERIALUART, (uint32_t)s); + dma_reset(dma->ref); + + dma_init_type init; + dma_default_para_init(&init); + init.peripheral_base_addr = (uint32_t)&s->USARTx->dt; + init.memory_base_addr = (uint32_t)s->port.txBuffer; + init.direction = DMA_DIR_MEMORY_TO_PERIPHERAL; + init.peripheral_inc_enable = FALSE; + init.memory_inc_enable = TRUE; + init.peripheral_data_width = DMA_PERIPHERAL_DATA_WIDTH_BYTE; + init.memory_data_width = DMA_MEMORY_DATA_WIDTH_BYTE; + init.loop_mode_enable = FALSE; + init.priority = DMA_PRIORITY_MEDIUM; + dma_init(dma->ref, &init); + dmaMuxEnable(dma, uartTxDmaConfig[device].request); + DMA_CLEAR_FLAG(dma, UART_TX_DMA_FLAGS); + dma_interrupt_enable(dma->ref, UART_TX_DMA_ENDED, TRUE); + + s->txDmaCount = 0; + s->txDma = dma; + + usart_interrupt_enable(s->USARTx, USART_TDBE_INT, FALSE); + usart_dma_transmitter_enable(s->USARTx, TRUE); + return true; +} +#endif + void uartIrqHandler(uartPort_t *s) { - if (usart_flag_get(s->USARTx, USART_RDBF_FLAG) == SET) { +#ifdef USE_UART_RX_DMA + // Clearing the flag reads dt: after a character of silence the stream has already taken the last byte + if (s->USARTx->ctrl1_bit.idleien && usart_flag_get(s->USARTx, USART_IDLEF_FLAG) == SET) { + uartClearIdleFlag(s); + uartRxDmaIdle(s); + } +#endif + + // With RX on DMA the interrupt still fires for TX: an RX byte is the stream's to take + if (usart_flag_get(s->USARTx, USART_RDBF_FLAG) == SET && !uartRxDmaRunning(s)) { if (s->port.rxCallback) { s->port.rxCallback(s->USARTx->dt, s->port.rxCallbackData); } else { @@ -368,7 +639,8 @@ void uartIrqHandler(uartPort_t *s) } } - if (usart_flag_get(s->USARTx, USART_TDBE_FLAG) == SET) { + // With TX on DMA the ring is the stream's to empty + if (usart_flag_get(s->USARTx, USART_TDBE_FLAG) == SET && !uartTxDmaRunning(s)) { if (s->port.txBufferTail != s->port.txBufferHead) { usart_data_transmit(s->USARTx, s->port.txBuffer[s->port.txBufferTail]); s->port.txBufferTail = (s->port.txBufferTail + 1) % s->port.txBufferSize; diff --git a/src/main/drivers/serial_uart_hal.c b/src/main/drivers/serial_uart_hal.c index 659fbf3c0d3..c5d4a611461 100644 --- a/src/main/drivers/serial_uart_hal.c +++ b/src/main/drivers/serial_uart_hal.c @@ -108,7 +108,14 @@ static void uartReconfigure(uartPort_t *uartPort) HAL_UART_Init(&uartPort->Handle); } - if (uartPort->port.mode & MODE_RX) { + if ((uartPort->port.mode & MODE_RX) && uartRxDmaRunning(uartPort)) { + /* The de-init cleared the DMA request, and the idle interrupt of a receiver taking bursts */ + SET_BIT(uartPort->USARTx->CR3, USART_CR3_DMAR); + if (uartPort->port.rxBursts) { + SET_BIT(uartPort->USARTx->CR1, USART_CR1_IDLEIE); + } + } + else if (uartPort->port.mode & MODE_RX) { /* Enable the UART Parity Error Interrupt */ SET_BIT(uartPort->USARTx->CR1, USART_CR1_PEIE); @@ -120,7 +127,11 @@ static void uartReconfigure(uartPort_t *uartPort) } // Transmit IRQ - if (uartPort->port.mode & MODE_TX) { + if ((uartPort->port.mode & MODE_TX) && uartTxDmaRunning(uartPort)) { + /* The de-init cleared the DMA request */ + SET_BIT(uartPort->USARTx->CR3, USART_CR3_DMAT); + } + else if (uartPort->port.mode & MODE_TX) { /* Enable the UART Transmit Data Register Empty Interrupt */ SET_BIT(uartPort->USARTx->CR1, USART_CR1_TXEIE); } @@ -169,6 +180,10 @@ serialPort_t *uartOpen(USART_TypeDef *USARTx, serialReceiveCallbackPtr callback, } + // Streams still running would go on with the rings about to be reset, and a receiver's with the old callback + uartTxDmaStop(s); + uartRxDmaStop(s); + // common serial initialisation code should move to serialPort::init() s->port.rxBufferHead = s->port.rxBufferTail = 0; s->port.txBufferHead = s->port.txBufferTail = 0; @@ -179,6 +194,9 @@ serialPort_t *uartOpen(USART_TypeDef *USARTx, serialReceiveCallbackPtr callback, s->port.baudRate = baudRate; s->port.options = options; + // Before the UART, so it comes up with the stream already running + uartRxDmaStart(s); + uartTxDmaStart(s); uartReconfigure(s); return (serialPort_t *)s; @@ -209,10 +227,12 @@ uint32_t uartTotalRxBytesWaiting(const serialPort_t *instance) { uartPort_t *s = (uartPort_t*)instance; - if (s->port.rxBufferHead >= s->port.rxBufferTail) { - return s->port.rxBufferHead - s->port.rxBufferTail; + const uint32_t head = uartRxBufferHead(s); + + if (head >= s->port.rxBufferTail) { + return head - s->port.rxBufferTail; } else { - return s->port.rxBufferSize + s->port.rxBufferHead - s->port.rxBufferTail; + return s->port.rxBufferSize + head - s->port.rxBufferTail; } } @@ -253,17 +273,52 @@ uint8_t uartRead(serialPort_t *instance) return ch; } -void uartWrite(serialPort_t *instance, uint8_t ch) +static void uartQueue(uartPort_t *s, uint8_t ch) { - uartPort_t *s = (uartPort_t *)instance; s->port.txBuffer[s->port.txBufferHead] = ch; if (s->port.txBufferHead + 1 >= s->port.txBufferSize) { s->port.txBufferHead = 0; } else { s->port.txBufferHead++; } +} + +void uartWrite(serialPort_t *instance, uint8_t ch) +{ + uartPort_t *s = (uartPort_t *)instance; + uartQueue(s, ch); + + if (uartTxDmaRunning(s)) { + uartStartTxDMA(s); + } else { + __HAL_UART_ENABLE_IT(&s->Handle, UART_IT_TXE); + } +} + +// Queued whole before the stream starts, so a message goes out in one transfer +static void uartWriteBuf(serialPort_t *instance, const void *data, int count) +{ + uartPort_t *s = (uartPort_t *)instance; + const uint8_t *p = data; + + for (; count > 0; count--, p++) { + while (!uartTotalTxBytesFree(instance)) { + // The ring is full: sending is what makes room + if (uartTxDmaRunning(s)) { + uartStartTxDMA(s); + } + } + + if (uartTxDmaRunning(s)) { + uartQueue(s, *p); + } else { + uartWrite(instance, *p); + } + } - __HAL_UART_ENABLE_IT(&s->Handle, UART_IT_TXE); + if (uartTxDmaRunning(s)) { + uartStartTxDMA(s); + } } bool isUartIdle(serialPort_t *instance) @@ -288,9 +343,11 @@ const struct serialPortVTable uartVTable[] = { .setMode = uartSetMode, .setOptions = uartSetOptions, .isConnected = NULL, - .writeBuf = NULL, + .writeBuf = uartWriteBuf, .beginWrite = NULL, .endWrite = NULL, .isIdle = isUartIdle, + .setRxBuffer = uartSetRxBuffer, + .release = uartRelease, } }; diff --git a/src/main/drivers/serial_uart_hal_at32f43x.c b/src/main/drivers/serial_uart_hal_at32f43x.c index f30726332f4..f89d24c3e17 100644 --- a/src/main/drivers/serial_uart_hal_at32f43x.c +++ b/src/main/drivers/serial_uart_hal_at32f43x.c @@ -138,6 +138,10 @@ serialPort_t *uartOpen(usart_type *USARTx, serialReceiveCallbackPtr rxCallback, return (serialPort_t *)s; } + // Streams still running would go on with the rings about to be reset, and a receiver's with the old callback + uartTxDmaStop(s); + uartRxDmaStop(s); + // common serial initialisation code should move to serialPort::init() s->port.rxBufferHead = s->port.rxBufferTail = 0; s->port.txBufferHead = s->port.txBufferTail = 0; @@ -150,13 +154,13 @@ serialPort_t *uartOpen(usart_type *USARTx, serialReceiveCallbackPtr rxCallback, uartReconfigure(s); - if (mode & MODE_RX) { + if ((mode & MODE_RX) && !uartRxDmaStart(s)) { usart_flag_clear(s->USARTx, USART_RDBF_FLAG); usart_interrupt_enable (s->USARTx, USART_RDBF_INT, TRUE); } - if (mode & MODE_TX) { + if ((mode & MODE_TX) && !uartTxDmaStart(s)) { usart_interrupt_enable (s->USARTx, USART_TDBE_INT, TRUE); } @@ -190,10 +194,12 @@ uint32_t uartTotalRxBytesWaiting(const serialPort_t *instance) { const uartPort_t *s = (const uartPort_t*)instance; - if (s->port.rxBufferHead >= s->port.rxBufferTail) { - return s->port.rxBufferHead - s->port.rxBufferTail; + const uint32_t head = uartRxBufferHead(s); + + if (head >= s->port.rxBufferTail) { + return head - s->port.rxBufferTail; } else { - return s->port.rxBufferSize + s->port.rxBufferHead - s->port.rxBufferTail; + return s->port.rxBufferSize + head - s->port.rxBufferTail; } } @@ -233,18 +239,52 @@ uint8_t uartRead(serialPort_t *instance) return ch; } -void uartWrite(serialPort_t *instance, uint8_t ch) +static void uartQueue(uartPort_t *s, uint8_t ch) { - uartPort_t *s = (uartPort_t *)instance; s->port.txBuffer[s->port.txBufferHead] = ch; if (s->port.txBufferHead + 1 >= s->port.txBufferSize) { s->port.txBufferHead = 0; } else { s->port.txBufferHead++; } +} + +void uartWrite(serialPort_t *instance, uint8_t ch) +{ + uartPort_t *s = (uartPort_t *)instance; + uartQueue(s, ch); + + if (uartTxDmaRunning(s)) { + uartStartTxDMA(s); + } else { + usart_interrupt_enable (s->USARTx, USART_TDBE_INT, TRUE); + } +} - usart_interrupt_enable (s->USARTx, USART_TDBE_INT, TRUE); +// Queued whole before the stream starts, so a message goes out in one transfer +static void uartWriteBuf(serialPort_t *instance, const void *data, int count) +{ + uartPort_t *s = (uartPort_t *)instance; + const uint8_t *p = data; + + for (; count > 0; count--, p++) { + while (!uartTotalTxBytesFree(instance)) { + // The ring is full: sending is what makes room + if (uartTxDmaRunning(s)) { + uartStartTxDMA(s); + } + } + + if (uartTxDmaRunning(s)) { + uartQueue(s, *p); + } else { + uartWrite(instance, *p); + } + } + if (uartTxDmaRunning(s)) { + uartStartTxDMA(s); + } } bool isUartIdle(serialPort_t *instance) @@ -270,9 +310,11 @@ const struct serialPortVTable uartVTable[] = { .setMode = uartSetMode, .setOptions = uartSetOptions, .isConnected = NULL, - .writeBuf = NULL, + .writeBuf = uartWriteBuf, .beginWrite = NULL, .endWrite = NULL, .isIdle = isUartIdle, + .setRxBuffer = uartSetRxBuffer, + .release = uartRelease, } }; diff --git a/src/main/drivers/serial_uart_impl.h b/src/main/drivers/serial_uart_impl.h index d983570285b..b44f0ece640 100644 --- a/src/main/drivers/serial_uart_impl.h +++ b/src/main/drivers/serial_uart_impl.h @@ -19,10 +19,379 @@ // device specific uart implementation is defined here +#include "build/atomic.h" + +#include "drivers/nvic.h" +#include "drivers/time.h" + extern const struct serialPortVTable uartVTable[]; +#if defined(USE_UART_RX_DMA) || defined(USE_UART_TX_DMA) +#include "common/utils.h" +#include "drivers/dma.h" +#include "drivers/pwm_mapping.h" +#include "drivers/timer.h" + +#if defined(STM32F4) || defined(STM32F7) +// Fixed streams and channel per UART receiver and transmitter (reference manual request +// tables): a target naming another gets a build error instead of a silent port +#define UART_DMA_IS(tag, dma, stream, channel) ((tag) == DMA_TAG_AUTO || (tag) == DMA_NONE || \ + (DMATAG_GET_DMA(tag) == (dma) && DMATAG_GET_STREAM(tag) == (stream) && DMATAG_GET_CHANNEL(tag) == (channel))) +#ifdef UART1_RX_DMA +STATIC_ASSERT(UART_DMA_IS(UART1_RX_DMA, 2, 2, 4) || UART_DMA_IS(UART1_RX_DMA, 2, 5, 4), UART1_RX_DMA_is_DMA2_stream_2_or_5_channel_4); +#endif +#ifdef UART2_RX_DMA +STATIC_ASSERT(UART_DMA_IS(UART2_RX_DMA, 1, 5, 4), UART2_RX_DMA_is_DMA1_stream_5_channel_4); +#endif +#ifdef UART3_RX_DMA +STATIC_ASSERT(UART_DMA_IS(UART3_RX_DMA, 1, 1, 4), UART3_RX_DMA_is_DMA1_stream_1_channel_4); +#endif +#ifdef UART4_RX_DMA +STATIC_ASSERT(UART_DMA_IS(UART4_RX_DMA, 1, 2, 4), UART4_RX_DMA_is_DMA1_stream_2_channel_4); +#endif +#ifdef UART5_RX_DMA +STATIC_ASSERT(UART_DMA_IS(UART5_RX_DMA, 1, 0, 4), UART5_RX_DMA_is_DMA1_stream_0_channel_4); +#endif +#ifdef UART6_RX_DMA +STATIC_ASSERT(UART_DMA_IS(UART6_RX_DMA, 2, 1, 5) || UART_DMA_IS(UART6_RX_DMA, 2, 2, 5), UART6_RX_DMA_is_DMA2_stream_1_or_2_channel_5); +#endif +#ifdef UART7_RX_DMA +STATIC_ASSERT(UART_DMA_IS(UART7_RX_DMA, 1, 3, 5), UART7_RX_DMA_is_DMA1_stream_3_channel_5); +#endif +#ifdef UART8_RX_DMA +STATIC_ASSERT(UART_DMA_IS(UART8_RX_DMA, 1, 6, 5), UART8_RX_DMA_is_DMA1_stream_6_channel_5); +#endif +#ifdef UART1_TX_DMA +STATIC_ASSERT(UART_DMA_IS(UART1_TX_DMA, 2, 7, 4), UART1_TX_DMA_is_DMA2_stream_7_channel_4); +#endif +#ifdef UART2_TX_DMA +STATIC_ASSERT(UART_DMA_IS(UART2_TX_DMA, 1, 6, 4), UART2_TX_DMA_is_DMA1_stream_6_channel_4); +#endif +#ifdef UART3_TX_DMA +STATIC_ASSERT(UART_DMA_IS(UART3_TX_DMA, 1, 3, 4) || UART_DMA_IS(UART3_TX_DMA, 1, 4, 7), UART3_TX_DMA_is_DMA1_stream_3_channel_4_or_stream_4_channel_7); +#endif +#ifdef UART4_TX_DMA +STATIC_ASSERT(UART_DMA_IS(UART4_TX_DMA, 1, 4, 4), UART4_TX_DMA_is_DMA1_stream_4_channel_4); +#endif +#ifdef UART5_TX_DMA +STATIC_ASSERT(UART_DMA_IS(UART5_TX_DMA, 1, 7, 4), UART5_TX_DMA_is_DMA1_stream_7_channel_4); +#endif +#ifdef UART6_TX_DMA +STATIC_ASSERT(UART_DMA_IS(UART6_TX_DMA, 2, 6, 5) || UART_DMA_IS(UART6_TX_DMA, 2, 7, 5), UART6_TX_DMA_is_DMA2_stream_6_or_7_channel_5); +#endif +#ifdef UART7_TX_DMA +STATIC_ASSERT(UART_DMA_IS(UART7_TX_DMA, 1, 1, 5), UART7_TX_DMA_is_DMA1_stream_1_channel_5); +#endif +#ifdef UART8_TX_DMA +STATIC_ASSERT(UART_DMA_IS(UART8_TX_DMA, 1, 0, 5), UART8_TX_DMA_is_DMA1_stream_0_channel_5); +#endif +#endif + +// Every stop gives its stream back, so a port holds only the streams it runs on +#define uartDmaRelease(dma) dmaInit((dma), OWNER_FREE, 0) + +// Free, and not one a timer output claims later in init +static inline bool uartDmaStreamAvailable(DMA_t dma) +{ + if (pwmIsDmaStreamReserved(dma)) { + return false; + } + return dmaGetOwner(dma) == OWNER_FREE; +} + +#ifdef USE_UART_RX_DMA +#define uartRxDmaOf(s) ((s)->rxDma) +#else +#define uartRxDmaOf(s) ((DMA_t)NULL) +#endif +#ifdef USE_UART_TX_DMA +#define uartTxDmaOf(s) ((s)->txDma) +#else +#define uartTxDmaOf(s) ((DMA_t)NULL) +#endif + +// Streams tried, in order, for a port whose target names none (DMA_TAG_AUTO) +#if defined(STM32F4) || defined(STM32F7) +#define UART_DMA_CANDIDATES 2 +static const dmaTag_t uartRxDmaCandidates[UARTDEV_MAX][UART_DMA_CANDIDATES] = { + [UARTDEV_1] = { DMA_TAG(2, 2, 4), DMA_TAG(2, 5, 4) }, + [UARTDEV_2] = { DMA_TAG(1, 5, 4) }, + [UARTDEV_3] = { DMA_TAG(1, 1, 4) }, + [UARTDEV_4] = { DMA_TAG(1, 2, 4) }, + [UARTDEV_5] = { DMA_TAG(1, 0, 4) }, + [UARTDEV_6] = { DMA_TAG(2, 1, 5), DMA_TAG(2, 2, 5) }, + [UARTDEV_7] = { DMA_TAG(1, 3, 5) }, + [UARTDEV_8] = { DMA_TAG(1, 6, 5) }, +}; +static const dmaTag_t uartTxDmaCandidates[UARTDEV_MAX][UART_DMA_CANDIDATES] = { + [UARTDEV_1] = { DMA_TAG(2, 7, 4) }, + [UARTDEV_2] = { DMA_TAG(1, 6, 4) }, + [UARTDEV_3] = { DMA_TAG(1, 3, 4), DMA_TAG(1, 4, 7) }, + [UARTDEV_4] = { DMA_TAG(1, 4, 4) }, + [UARTDEV_5] = { DMA_TAG(1, 7, 4) }, + [UARTDEV_6] = { DMA_TAG(2, 6, 5), DMA_TAG(2, 7, 5) }, + [UARTDEV_7] = { DMA_TAG(1, 1, 5) }, + [UARTDEV_8] = { DMA_TAG(1, 0, 5) }, +}; +#define uartDmaCandidate(table, device, i) ((table)[device][i]) +#elif defined(STM32H7) +// The DMAMUX routes any UART to any stream; DMA2 streams 0-2 are the ADCs' +#define UART_DMA_CANDIDATES 13 +static const dmaTag_t uartDmaStreams[UART_DMA_CANDIDATES] = { + DMA_TAG(1, 0, 0), DMA_TAG(1, 1, 0), DMA_TAG(1, 2, 0), DMA_TAG(1, 3, 0), DMA_TAG(1, 4, 0), DMA_TAG(1, 5, 0), + DMA_TAG(1, 6, 0), DMA_TAG(1, 7, 0), DMA_TAG(2, 3, 0), DMA_TAG(2, 4, 0), DMA_TAG(2, 5, 0), DMA_TAG(2, 6, 0), + DMA_TAG(2, 7, 0), +}; +#define uartDmaCandidate(table, device, i) (uartDmaStreams[i]) +#elif defined(AT32F43x) +// The DMAMUX routes any UART to any channel; the ADC's is excluded below +#define UART_DMA_CANDIDATES 14 +static const dmaTag_t uartDmaStreams[UART_DMA_CANDIDATES] = { + DMA_TAG(1, 1, 0), DMA_TAG(1, 2, 0), DMA_TAG(1, 3, 0), DMA_TAG(1, 4, 0), DMA_TAG(1, 5, 0), DMA_TAG(1, 6, 0), + DMA_TAG(1, 7, 0), DMA_TAG(2, 1, 0), DMA_TAG(2, 2, 0), DMA_TAG(2, 3, 0), DMA_TAG(2, 4, 0), DMA_TAG(2, 5, 0), + DMA_TAG(2, 6, 0), DMA_TAG(2, 7, 0), +}; +#define uartDmaCandidate(table, device, i) (uartDmaStreams[i]) +#endif + +// Streams claimed after the first serial ports open: the SD card would give up on a taken one, the AT32 ADC takes it anyway +static inline bool uartDmaStreamUsedElsewhere(DMA_t dma) +{ +#if (defined(STM32F4) || defined(STM32F7)) && defined(USE_SDCARD_SDIO) +#ifdef SDCARD_SDIO_DMA + if (dma == dmaGetByTag(SDCARD_SDIO_DMA)) { + return true; + } +#else + if (dma == dmaGetByTag(DMA_TAG(2, 3, 0)) || dma == dmaGetByTag(DMA_TAG(2, 6, 0))) { + return true; + } +#endif +#endif +#if defined(AT32F43x) +#ifdef ADC1_DMA_STREAM + if (dma == dmaGetByRef(ADC1_DMA_STREAM)) { +#else + if (dma == dmaGetByRef(DMA2_CHANNEL1)) { +#endif + return true; + } +#endif + UNUSED(dma); + return false; +} + +// Streams a target names for some port: an automatic pick leaves them alone, whichever port opens first +static const dmaTag_t uartNamedDmaTags[] = { +#if defined(UART1_RX_DMA) && (UART1_RX_DMA != DMA_TAG_AUTO) && (UART1_RX_DMA != DMA_NONE) + UART1_RX_DMA, +#endif +#if defined(UART1_TX_DMA) && (UART1_TX_DMA != DMA_TAG_AUTO) && (UART1_TX_DMA != DMA_NONE) + UART1_TX_DMA, +#endif +#if defined(UART2_RX_DMA) && (UART2_RX_DMA != DMA_TAG_AUTO) && (UART2_RX_DMA != DMA_NONE) + UART2_RX_DMA, +#endif +#if defined(UART2_TX_DMA) && (UART2_TX_DMA != DMA_TAG_AUTO) && (UART2_TX_DMA != DMA_NONE) + UART2_TX_DMA, +#endif +#if defined(UART3_RX_DMA) && (UART3_RX_DMA != DMA_TAG_AUTO) && (UART3_RX_DMA != DMA_NONE) + UART3_RX_DMA, +#endif +#if defined(UART3_TX_DMA) && (UART3_TX_DMA != DMA_TAG_AUTO) && (UART3_TX_DMA != DMA_NONE) + UART3_TX_DMA, +#endif +#if defined(UART4_RX_DMA) && (UART4_RX_DMA != DMA_TAG_AUTO) && (UART4_RX_DMA != DMA_NONE) + UART4_RX_DMA, +#endif +#if defined(UART4_TX_DMA) && (UART4_TX_DMA != DMA_TAG_AUTO) && (UART4_TX_DMA != DMA_NONE) + UART4_TX_DMA, +#endif +#if defined(UART5_RX_DMA) && (UART5_RX_DMA != DMA_TAG_AUTO) && (UART5_RX_DMA != DMA_NONE) + UART5_RX_DMA, +#endif +#if defined(UART5_TX_DMA) && (UART5_TX_DMA != DMA_TAG_AUTO) && (UART5_TX_DMA != DMA_NONE) + UART5_TX_DMA, +#endif +#if defined(UART6_RX_DMA) && (UART6_RX_DMA != DMA_TAG_AUTO) && (UART6_RX_DMA != DMA_NONE) + UART6_RX_DMA, +#endif +#if defined(UART6_TX_DMA) && (UART6_TX_DMA != DMA_TAG_AUTO) && (UART6_TX_DMA != DMA_NONE) + UART6_TX_DMA, +#endif +#if defined(UART7_RX_DMA) && (UART7_RX_DMA != DMA_TAG_AUTO) && (UART7_RX_DMA != DMA_NONE) + UART7_RX_DMA, +#endif +#if defined(UART7_TX_DMA) && (UART7_TX_DMA != DMA_TAG_AUTO) && (UART7_TX_DMA != DMA_NONE) + UART7_TX_DMA, +#endif +#if defined(UART8_RX_DMA) && (UART8_RX_DMA != DMA_TAG_AUTO) && (UART8_RX_DMA != DMA_NONE) + UART8_RX_DMA, +#endif +#if defined(UART8_TX_DMA) && (UART8_TX_DMA != DMA_TAG_AUTO) && (UART8_TX_DMA != DMA_NONE) + UART8_TX_DMA, +#endif + DMA_NONE, +}; + +static inline bool uartDmaStreamNamed(DMA_t dma) +{ + for (unsigned i = 0; i < ARRAYLEN(uartNamedDmaTags); i++) { + if (uartNamedDmaTags[i] != DMA_NONE && dma == dmaGetByTag(uartNamedDmaTags[i])) { + return true; + } + } + return false; +} + +// A receiver that takes its bytes in bursts, on a port that is not half duplex +#define uartRxTakesBursts(port) (((port)->options & (SERIAL_RX_BURSTS | SERIAL_BIDIR)) == SERIAL_RX_BURSTS) + +// The named stream, else the first free candidate; byte-wise receivers and half-duplex links were never measured on DMA +static inline dmaTag_t uartDmaPick(dmaTag_t named, const dmaTag_t (*candidates)[UART_DMA_CANDIDATES], UARTDevice_e device, + const serialPort_t *port, DMA_t otherDirection) +{ + if (named != DMA_TAG_AUTO) { + const DMA_t dma = dmaGetByTag(named); + return (named != DMA_NONE && dma && uartDmaStreamAvailable(dma)) ? named : DMA_NONE; + } + if ((port->rxCallback && !uartRxTakesBursts(port)) || (port->options & SERIAL_BIDIR)) { + return DMA_NONE; + } + for (int i = 0; i < UART_DMA_CANDIDATES; i++) { + const dmaTag_t tag = uartDmaCandidate(candidates, device, i); + const DMA_t dma = dmaGetByTag(tag); + if (tag != DMA_NONE && dma && dma != otherDirection && !uartDmaStreamUsedElsewhere(dma) && !uartDmaStreamNamed(dma) && + uartDmaStreamAvailable(dma)) { + return tag; + } + } + UNUSED(candidates); + UNUSED(device); + return DMA_NONE; +} +#endif + +#ifdef USE_UART_RX_DMA +// False leaves the port on the byte interrupt: no stream named or free, no RX, or an +// rxCallback that wants each byte as it lands +bool uartRxDmaStart(uartPort_t *s); +// Stops the stream and its interrupts and gives it back: for a port about to be reset or closed +void uartRxDmaStop(uartPort_t *s); + +#define uartRxDmaRefused(s) ((s)->port.rxCallback && !uartRxTakesBursts(&(s)->port)) + +static inline bool uartRxDmaRunning(const uartPort_t *s) +{ + return s->rxDma != NULL; +} + +// NDTR counts down the rest of the lap round the ring +static inline uint32_t uartRxBufferHead(const uartPort_t *s) +{ + if (s->rxDma) { +#if defined(AT32F43x) + const uint32_t left = s->rxDma->ref->dtcnt; +#else + const uint32_t left = s->rxDma->ref->NDTR; +#endif + return (s->port.rxBufferSize - left) % s->port.rxBufferSize; + } + return s->port.rxBufferHead; +} + +// Start bit, 8 data bits, parity and stop bits +static inline uint32_t uartCharNs(const uartPort_t *s) +{ + const uint32_t bits = 10 + ((s->port.options & SERIAL_PARITY_EVEN) ? 1 : 0) + ((s->port.options & SERIAL_STOPBITS_2) ? 1 : 0); + return bits * (1000000000U / s->port.baudRate); +} + +// Hands rxCallback what the stream wrote since the last call. The newest byte ended at lastEndUs and +// the others are timed back from it a character apart: a gap of a character would have ended the burst +static inline void uartRxDmaDeliver(uartPort_t *s, timeUs_t lastEndUs) +{ + // A passthrough drops the callback and reads the ring itself, as on the byte interrupt + if (!s->port.rxCallback) { + return; + } + + const uint32_t size = s->port.rxBufferSize; + const uint32_t charNs = uartCharNs(s); + uint32_t tail = s->port.rxBufferTail; + + for (uint32_t left = (uartRxBufferHead(s) + size - tail) % size; left > 0; left--) { + s->port.rxByteTimeUs = lastEndUs - ((left - 1) * charNs) / 1000; + s->port.rxCallback(s->port.rxBuffer[tail], s->port.rxCallbackData); + tail = (tail + 1) % size; + } + s->port.rxBufferTail = tail; +} + +// The UART flags idle one character after the last stop bit +static inline void uartRxDmaIdle(uartPort_t *s) +{ + uartRxDmaDeliver(s, microsISR() - uartCharNs(s) / 1000); +} +#else +static inline bool uartRxDmaStart(uartPort_t *s) { (void)s; return false; } +static inline void uartRxDmaStop(uartPort_t *s) { (void)s; } +static inline bool uartRxDmaRunning(const uartPort_t *s) { (void)s; return false; } +static inline uint32_t uartRxBufferHead(const uartPort_t *s) { return s->port.rxBufferHead; } +#endif + +#ifdef USE_UART_TX_DMA +// False leaves the port on the byte interrupt: no stream named or free, or no TX +bool uartTxDmaStart(uartPort_t *s); +// Stops the stream, dropping any transfer on its way, and gives it back: for a port about to be reset or closed +void uartTxDmaStop(uartPort_t *s); +// Sends what is queued, unless a transfer is already on its way: its end starts the next void uartStartTxDMA(uartPort_t *s); +static inline bool uartTxDmaRunning(const uartPort_t *s) +{ + return s->txDma != NULL; +} + +#if defined(STM32F7) || defined(STM32H7) +// The stream reads RAM, and a buffer a feature swaps in (MSP DisplayPort, gimbal) can still have its bytes in the D-cache +static inline void uartTxDmaCleanCache(const volatile uint8_t *data, uint32_t count) +{ + const uint32_t start = (uint32_t)data & ~31U; // this CMSIS does not align to the 32 byte line + SCB_CleanDCache_by_Addr((uint32_t *)start, (uint32_t)data + count - start); +} +#endif +#else +static inline bool uartTxDmaStart(uartPort_t *s) { (void)s; return false; } +static inline void uartTxDmaStop(uartPort_t *s) { (void)s; } +static inline void uartStartTxDMA(uartPort_t *s) { (void)s; } +static inline bool uartTxDmaRunning(const uartPort_t *s) { (void)s; return false; } +#endif + +// What a closed port had still to send or had received is dropped, as reopening it would +static inline void uartRelease(serialPort_t *instance) +{ + uartTxDmaStop((uartPort_t *)instance); + uartRxDmaStop((uartPort_t *)instance); +} + +// A stream keeps writing where it was started, so it is started again on the new ring +static inline void uartSetRxBuffer(serialPort_t *instance, volatile uint8_t *buffer, uint32_t size) +{ + uartPort_t *s = (uartPort_t *)instance; + + // Restarted in the same block: an idle or half ring interrupt in between would read the new ring with the old count + ATOMIC_BLOCK(NVIC_PRIO_MAX) { + s->port.rxBuffer = buffer; + s->port.rxBufferSize = size; + s->port.rxBufferHead = 0; + s->port.rxBufferTail = 0; + if (uartRxDmaRunning(s)) { + uartRxDmaStart(s); + } + } +} + uartPort_t *serialUART1(uint32_t baudRate, portMode_t mode, portOptions_t options); uartPort_t *serialUART2(uint32_t baudRate, portMode_t mode, portOptions_t options); uartPort_t *serialUART3(uint32_t baudRate, portMode_t mode, portOptions_t options); diff --git a/src/main/drivers/serial_uart_stm32f4xx.c b/src/main/drivers/serial_uart_stm32f4xx.c index 0f54fb69142..49798da99f1 100644 --- a/src/main/drivers/serial_uart_stm32f4xx.c +++ b/src/main/drivers/serial_uart_stm32f4xx.c @@ -213,8 +213,269 @@ static uartDevice_t* uartHardwareMap[] = { #endif }; +#if defined(USE_UART_RX_DMA) || defined(USE_UART_TX_DMA) +static UARTDevice_e uartDeviceOf(const uartPort_t *s) +{ + for (int device = 0; device < UARTDEV_MAX; device++) { + if (uartHardwareMap[device] && &uartHardwareMap[device]->port == s) { + return device; + } + } + return UARTDEV_MAX; +} + +#endif + +#ifdef USE_UART_RX_DMA +static const dmaTag_t uartRxDmaTag[UARTDEV_MAX] = { +#ifdef UART1_RX_DMA + [UARTDEV_1] = UART1_RX_DMA, +#endif +#ifdef UART2_RX_DMA + [UARTDEV_2] = UART2_RX_DMA, +#endif +#ifdef UART3_RX_DMA + [UARTDEV_3] = UART3_RX_DMA, +#endif +#ifdef UART4_RX_DMA + [UARTDEV_4] = UART4_RX_DMA, +#endif +#ifdef UART5_RX_DMA + [UARTDEV_5] = UART5_RX_DMA, +#endif +#ifdef UART6_RX_DMA + [UARTDEV_6] = UART6_RX_DMA, +#endif +#ifdef UART7_RX_DMA + [UARTDEV_7] = UART7_RX_DMA, +#endif +#ifdef UART8_RX_DMA + [UARTDEV_8] = UART8_RX_DMA, +#endif +}; + +#define UART_RX_DMA_FLAGS (DMA_IT_TCIF | DMA_IT_HTIF | DMA_IT_TEIF | DMA_IT_DMEIF | DMA_IT_FEIF) + +void uartRxDmaStop(uartPort_t *s) +{ + if (s->rxDma) { + USART_DMACmd(s->USARTx, USART_DMAReq_Rx, DISABLE); + USART_ITConfig(s->USARTx, USART_IT_IDLE, DISABLE); + DMA_ITConfig(s->rxDma->ref, DMA_IT_HT | DMA_IT_TC, DISABLE); + DMA_Cmd(s->rxDma->ref, DISABLE); + while (DMA_GetCmdStatus(s->rxDma->ref) != DISABLE); + DMA_CLEAR_FLAG(s->rxDma, UART_RX_DMA_FLAGS); + uartDmaRelease(s->rxDma); + s->rxDma = NULL; + s->port.rxBursts = false; + } +} + +// Half and full ring: a burst longer than half the ring is handed over before the stream overwrites it +static void uartRxDmaHandler(DMA_t dma) +{ + DMA_CLEAR_FLAG(dma, UART_RX_DMA_FLAGS); + uartRxDmaDeliver((uartPort_t *)dma->userParam, microsISR()); +} + +bool uartRxDmaStart(uartPort_t *s) +{ + uartRxDmaStop(s); + + const UARTDevice_e device = uartDeviceOf(s); + if (device == UARTDEV_MAX || !(s->port.mode & MODE_RX) || uartRxDmaRefused(s)) { + return false; + } + + const dmaTag_t tag = uartDmaPick(uartRxDmaTag[device], uartRxDmaCandidates, device, &s->port, uartTxDmaOf(s)); + const DMA_t dma = dmaGetByTag(tag); + if (!dma) { + return false; + } + + dmaInit(dma, OWNER_SERIAL, RESOURCE_INDEX(device)); + DMA_DeInit(dma->ref); + + DMA_InitTypeDef init; + DMA_StructInit(&init); + init.DMA_Channel = DMATAG_GET_CHANNEL(tag) << 25; // DMA_Channel_n is n in CHSEL + init.DMA_PeripheralBaseAddr = (uint32_t)&s->USARTx->DR; + init.DMA_Memory0BaseAddr = (uint32_t)s->port.rxBuffer; + init.DMA_DIR = DMA_DIR_PeripheralToMemory; + init.DMA_BufferSize = s->port.rxBufferSize; + init.DMA_PeripheralInc = DMA_PeripheralInc_Disable; + init.DMA_MemoryInc = DMA_MemoryInc_Enable; + init.DMA_PeripheralDataSize = DMA_PeripheralDataSize_Byte; + init.DMA_MemoryDataSize = DMA_MemoryDataSize_Byte; + init.DMA_Mode = DMA_Mode_Circular; + init.DMA_Priority = DMA_Priority_Medium; + // No FIFO: a byte is in the ring as soon as the count says so + init.DMA_FIFOMode = DMA_FIFOMode_Disable; + DMA_Init(dma->ref, &init); + + s->port.rxBufferHead = s->port.rxBufferTail = 0; + s->rxDma = dma; + + if (s->port.rxCallback) { + // The UART's priority, so the two never hand bytes over at once + dmaSetHandler(dma, uartRxDmaHandler, NVIC_PRIO_SERIALUART, (uint32_t)s); + DMA_CLEAR_FLAG(dma, UART_RX_DMA_FLAGS); + DMA_ITConfig(dma->ref, DMA_IT_HT | DMA_IT_TC, ENABLE); + s->port.rxBursts = true; + USART_ITConfig(s->USARTx, USART_IT_IDLE, ENABLE); + } + + DMA_Cmd(dma->ref, ENABLE); + + USART_ITConfig(s->USARTx, USART_IT_RXNE, DISABLE); + USART_DMACmd(s->USARTx, USART_DMAReq_Rx, ENABLE); + return true; +} +#endif + +#ifdef USE_UART_TX_DMA +static const dmaTag_t uartTxDmaTag[UARTDEV_MAX] = { +#ifdef UART1_TX_DMA + [UARTDEV_1] = UART1_TX_DMA, +#endif +#ifdef UART2_TX_DMA + [UARTDEV_2] = UART2_TX_DMA, +#endif +#ifdef UART3_TX_DMA + [UARTDEV_3] = UART3_TX_DMA, +#endif +#ifdef UART4_TX_DMA + [UARTDEV_4] = UART4_TX_DMA, +#endif +#ifdef UART5_TX_DMA + [UARTDEV_5] = UART5_TX_DMA, +#endif +#ifdef UART6_TX_DMA + [UARTDEV_6] = UART6_TX_DMA, +#endif +#ifdef UART7_TX_DMA + [UARTDEV_7] = UART7_TX_DMA, +#endif +#ifdef UART8_TX_DMA + [UARTDEV_8] = UART8_TX_DMA, +#endif +}; + +#define UART_TX_DMA_FLAGS (DMA_IT_TCIF | DMA_IT_HTIF | DMA_IT_TEIF | DMA_IT_DMEIF | DMA_IT_FEIF) +#define UART_TX_DMA_ENDED (DMA_IT_TCIF | DMA_IT_TEIF) + +void uartTxDmaStop(uartPort_t *s) +{ + if (s->txDma) { + USART_DMACmd(s->USARTx, USART_DMAReq_Tx, DISABLE); + DMA_ITConfig(s->txDma->ref, DMA_IT_TC | DMA_IT_TE, DISABLE); + DMA_Cmd(s->txDma->ref, DISABLE); + while (DMA_GetCmdStatus(s->txDma->ref) != DISABLE); + DMA_CLEAR_FLAG(s->txDma, UART_TX_DMA_FLAGS); + uartDmaRelease(s->txDma); + s->txDma = NULL; + s->txDmaCount = 0; + } +} + +void uartStartTxDMA(uartPort_t *s) +{ + // Masked: the end-of-transfer interrupt also starts the next one + ATOMIC_BLOCK(NVIC_PRIO_SERIALUART) { + const uint32_t head = s->port.txBufferHead; + const uint32_t tail = s->port.txBufferTail; + + if (s->txDma && !s->txDmaCount && head != tail) { + // Up to the head, or to the end of the ring if the queue wraps: the rest follows + const uint32_t count = (head > tail) ? head - tail : s->port.txBufferSize - tail; + + s->txDmaCount = count; + s->txDma->ref->M0AR = (uint32_t)&s->port.txBuffer[tail]; + DMA_SetCurrDataCounter(s->txDma->ref, count); + DMA_Cmd(s->txDma->ref, ENABLE); + } + } +} + +// An error ends the transfer too, or the port would wait for it forever +static void uartTxDmaHandler(DMA_t dma) +{ + uartPort_t *s = (uartPort_t *)dma->userParam; + + // Clear every flag: one left set re-raises the interrupt, starving USB at this priority + const bool ended = DMA_GET_FLAG_STATUS(dma, UART_TX_DMA_ENDED); + DMA_CLEAR_FLAG(dma, UART_TX_DMA_FLAGS); + + if (ended) { + // After an error the rest goes again, unless nothing went: that error would only repeat + const uint32_t sent = s->txDmaCount - DMA_GetCurrDataCounter(dma->ref); + s->port.txBufferTail = (s->port.txBufferTail + (sent ? sent : s->txDmaCount)) % s->port.txBufferSize; + s->txDmaCount = 0; + uartStartTxDMA(s); + } +} + +bool uartTxDmaStart(uartPort_t *s) +{ + uartTxDmaStop(s); + + const UARTDevice_e device = uartDeviceOf(s); + if (device == UARTDEV_MAX || !(s->port.mode & MODE_TX)) { + return false; + } + + const dmaTag_t tag = uartDmaPick(uartTxDmaTag[device], uartTxDmaCandidates, device, &s->port, uartRxDmaOf(s)); + const DMA_t dma = dmaGetByTag(tag); + if (!dma) { + return false; + } +#ifdef USE_UART_RX_DMA + // The stream this port receives on is this UART's too, so it looks free to it + if (dma == s->rxDma) { + return false; + } +#endif + + dmaInit(dma, OWNER_SERIAL, RESOURCE_INDEX(device)); + dmaSetHandler(dma, uartTxDmaHandler, NVIC_PRIO_SERIALUART, (uint32_t)s); + DMA_DeInit(dma->ref); + + DMA_InitTypeDef init; + DMA_StructInit(&init); + init.DMA_Channel = DMATAG_GET_CHANNEL(tag) << 25; // DMA_Channel_n is n in CHSEL + init.DMA_PeripheralBaseAddr = (uint32_t)&s->USARTx->DR; + init.DMA_Memory0BaseAddr = (uint32_t)s->port.txBuffer; + init.DMA_DIR = DMA_DIR_MemoryToPeripheral; + init.DMA_PeripheralInc = DMA_PeripheralInc_Disable; + init.DMA_MemoryInc = DMA_MemoryInc_Enable; + init.DMA_PeripheralDataSize = DMA_PeripheralDataSize_Byte; + init.DMA_MemoryDataSize = DMA_MemoryDataSize_Byte; + init.DMA_Mode = DMA_Mode_Normal; + init.DMA_Priority = DMA_Priority_Medium; + init.DMA_FIFOMode = DMA_FIFOMode_Disable; + DMA_Init(dma->ref, &init); + DMA_CLEAR_FLAG(dma, UART_TX_DMA_FLAGS); + DMA_ITConfig(dma->ref, DMA_IT_TC | DMA_IT_TE, ENABLE); + + s->txDmaCount = 0; + s->txDma = dma; + + USART_ITConfig(s->USARTx, USART_IT_TXE, DISABLE); + USART_DMACmd(s->USARTx, USART_DMAReq_Tx, ENABLE); + return true; +} +#endif + void uartIrqHandler(uartPort_t *s) { +#ifdef USE_UART_RX_DMA + // Clearing the flag reads DR: after a character of silence the stream has already taken the last byte + if (USART_GetITStatus(s->USARTx, USART_IT_IDLE) == SET) { + uartClearIdleFlag(s); + uartRxDmaIdle(s); + } +#endif + if (USART_GetITStatus(s->USARTx, USART_IT_RXNE) == SET) { if (s->port.rxCallback) { s->port.rxCallback(s->USARTx->DR, s->port.rxCallbackData); diff --git a/src/main/drivers/serial_uart_stm32f7xx.c b/src/main/drivers/serial_uart_stm32f7xx.c index 944634c31ea..12538cb93c3 100644 --- a/src/main/drivers/serial_uart_stm32f7xx.c +++ b/src/main/drivers/serial_uart_stm32f7xx.c @@ -251,11 +251,340 @@ static uartDevice_t* uartHardwareMap[] = { #endif }; +#if defined(USE_UART_RX_DMA) || defined(USE_UART_TX_DMA) +static UARTDevice_e uartDeviceOf(const uartPort_t *s) +{ + for (int device = 0; device < UARTDEV_MAX; device++) { + if (uartHardwareMap[device] && &uartHardwareMap[device]->port == s) { + return device; + } + } + return UARTDEV_MAX; +} + +#endif + +#ifdef USE_UART_RX_DMA +static const dmaTag_t uartRxDmaTag[UARTDEV_MAX] = { +#ifdef UART1_RX_DMA + [UARTDEV_1] = UART1_RX_DMA, +#endif +#ifdef UART2_RX_DMA + [UARTDEV_2] = UART2_RX_DMA, +#endif +#ifdef UART3_RX_DMA + [UARTDEV_3] = UART3_RX_DMA, +#endif +#ifdef UART4_RX_DMA + [UARTDEV_4] = UART4_RX_DMA, +#endif +#ifdef UART5_RX_DMA + [UARTDEV_5] = UART5_RX_DMA, +#endif +#ifdef UART6_RX_DMA + [UARTDEV_6] = UART6_RX_DMA, +#endif +#ifdef UART7_RX_DMA + [UARTDEV_7] = UART7_RX_DMA, +#endif +#ifdef UART8_RX_DMA + [UARTDEV_8] = UART8_RX_DMA, +#endif +}; + +// Only with the D-cache on, which leaves DMA_RAM uncached: from the cache the CPU would not see what the stream wrote +#ifdef DMA_RAM_UNCACHED +#if defined(UART1_RX_DMA) && (UART1_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart1RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif +#if defined(UART2_RX_DMA) && (UART2_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart2RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif +#if defined(UART3_RX_DMA) && (UART3_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart3RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif +#if defined(UART4_RX_DMA) && (UART4_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart4RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif +#if defined(UART5_RX_DMA) && (UART5_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart5RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif +#if defined(UART6_RX_DMA) && (UART6_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart6RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif +#if defined(UART7_RX_DMA) && (UART7_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart7RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif +#if defined(UART8_RX_DMA) && (UART8_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart8RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif + +static volatile uint8_t * const uartRxDmaBuffer[UARTDEV_MAX] = { +#if defined(UART1_RX_DMA) && (UART1_RX_DMA != DMA_NONE) + [UARTDEV_1] = uart1RxDmaBuffer, +#endif +#if defined(UART2_RX_DMA) && (UART2_RX_DMA != DMA_NONE) + [UARTDEV_2] = uart2RxDmaBuffer, +#endif +#if defined(UART3_RX_DMA) && (UART3_RX_DMA != DMA_NONE) + [UARTDEV_3] = uart3RxDmaBuffer, +#endif +#if defined(UART4_RX_DMA) && (UART4_RX_DMA != DMA_NONE) + [UARTDEV_4] = uart4RxDmaBuffer, +#endif +#if defined(UART5_RX_DMA) && (UART5_RX_DMA != DMA_NONE) + [UARTDEV_5] = uart5RxDmaBuffer, +#endif +#if defined(UART6_RX_DMA) && (UART6_RX_DMA != DMA_NONE) + [UARTDEV_6] = uart6RxDmaBuffer, +#endif +#if defined(UART7_RX_DMA) && (UART7_RX_DMA != DMA_NONE) + [UARTDEV_7] = uart7RxDmaBuffer, +#endif +#if defined(UART8_RX_DMA) && (UART8_RX_DMA != DMA_NONE) + [UARTDEV_8] = uart8RxDmaBuffer, +#endif +}; +#endif + +#define UART_RX_DMA_FLAGS (DMA_IT_TCIF | DMA_IT_HTIF | DMA_IT_TEIF | DMA_IT_DMEIF | DMA_IT_FEIF) + +void uartRxDmaStop(uartPort_t *s) +{ + if (s->rxDma) { + const uint32_t stream = DMATAG_GET_STREAM(s->rxDma->tag); // LL_DMA_STREAM_n is n + CLEAR_BIT(s->USARTx->CR3, USART_CR3_DMAR); + CLEAR_BIT(s->USARTx->CR1, USART_CR1_IDLEIE); + LL_DMA_DisableIT_HT(s->rxDma->dma, stream); + LL_DMA_DisableIT_TC(s->rxDma->dma, stream); + LL_DMA_DisableStream(s->rxDma->dma, stream); + while (LL_DMA_IsEnabledStream(s->rxDma->dma, stream)); + DMA_CLEAR_FLAG(s->rxDma, UART_RX_DMA_FLAGS); + uartDmaRelease(s->rxDma); + s->rxDma = NULL; + s->port.rxBursts = false; + } +} + +// Half and full ring: a burst longer than half the ring is handed over before the stream overwrites it +static void uartRxDmaHandler(DMA_t dma) +{ + DMA_CLEAR_FLAG(dma, UART_RX_DMA_FLAGS); + uartRxDmaDeliver((uartPort_t *)dma->userParam, microsISR()); +} + +// USART_CR3_DMAR is set again in uartReconfigure(): the HAL clears CR3 whenever it reprograms the port +bool uartRxDmaStart(uartPort_t *s) +{ + uartRxDmaStop(s); + + const UARTDevice_e device = uartDeviceOf(s); + if (device == UARTDEV_MAX || !(s->port.mode & MODE_RX) || uartRxDmaRefused(s)) { + return false; + } + + const dmaTag_t tag = uartDmaPick(uartRxDmaTag[device], uartRxDmaCandidates, device, &s->port, uartTxDmaOf(s)); + const DMA_t dma = dmaGetByTag(tag); + if (!dma) { + return false; + } + + dmaInit(dma, OWNER_SERIAL, RESOURCE_INDEX(device)); + + const uint32_t stream = DMATAG_GET_STREAM(tag); + LL_DMA_DeInit(dma->dma, stream); + + LL_DMA_InitTypeDef init; + LL_DMA_StructInit(&init); + init.Channel = DMATAG_GET_CHANNEL(tag) << DMA_SxCR_CHSEL_Pos; // LL_DMA_CHANNEL_n is n in CHSEL + init.PeriphOrM2MSrcAddress = (uint32_t)&s->USARTx->RDR; + init.MemoryOrM2MDstAddress = (uint32_t)s->port.rxBuffer; + init.Direction = LL_DMA_DIRECTION_PERIPH_TO_MEMORY; + init.Mode = LL_DMA_MODE_CIRCULAR; + init.PeriphOrM2MSrcIncMode = LL_DMA_PERIPH_NOINCREMENT; + init.MemoryOrM2MDstIncMode = LL_DMA_MEMORY_INCREMENT; + init.PeriphOrM2MSrcDataSize = LL_DMA_PDATAALIGN_BYTE; + init.MemoryOrM2MDstDataSize = LL_DMA_MDATAALIGN_BYTE; + init.NbData = s->port.rxBufferSize; + init.Priority = LL_DMA_PRIORITY_MEDIUM; + // No FIFO: a byte is in the ring as soon as the count says so + init.FIFOMode = LL_DMA_FIFOMODE_DISABLE; + LL_DMA_Init(dma->dma, stream, &init); + + s->port.rxBufferHead = s->port.rxBufferTail = 0; + s->rxDma = dma; + + if (s->port.rxCallback) { + // The UART's priority, so the two never hand bytes over at once + dmaSetHandler(dma, uartRxDmaHandler, NVIC_PRIO_SERIALUART, (uint32_t)s); + DMA_CLEAR_FLAG(dma, UART_RX_DMA_FLAGS); + LL_DMA_EnableIT_HT(dma->dma, stream); + LL_DMA_EnableIT_TC(dma->dma, stream); + s->port.rxBursts = true; + SET_BIT(s->USARTx->CR1, USART_CR1_IDLEIE); + } + + LL_DMA_EnableStream(dma->dma, stream); + // Restarted on a new ring (serialSetRxBuffer) the port gets no reprogramming to set it + SET_BIT(s->USARTx->CR3, USART_CR3_DMAR); + return true; +} +#endif + +#ifdef USE_UART_TX_DMA +static const dmaTag_t uartTxDmaTag[UARTDEV_MAX] = { +#ifdef UART1_TX_DMA + [UARTDEV_1] = UART1_TX_DMA, +#endif +#ifdef UART2_TX_DMA + [UARTDEV_2] = UART2_TX_DMA, +#endif +#ifdef UART3_TX_DMA + [UARTDEV_3] = UART3_TX_DMA, +#endif +#ifdef UART4_TX_DMA + [UARTDEV_4] = UART4_TX_DMA, +#endif +#ifdef UART5_TX_DMA + [UARTDEV_5] = UART5_TX_DMA, +#endif +#ifdef UART6_TX_DMA + [UARTDEV_6] = UART6_TX_DMA, +#endif +#ifdef UART7_TX_DMA + [UARTDEV_7] = UART7_TX_DMA, +#endif +#ifdef UART8_TX_DMA + [UARTDEV_8] = UART8_TX_DMA, +#endif +}; + +#define UART_TX_DMA_FLAGS (DMA_IT_TCIF | DMA_IT_HTIF | DMA_IT_TEIF | DMA_IT_DMEIF | DMA_IT_FEIF) +#define UART_TX_DMA_ENDED (DMA_IT_TCIF | DMA_IT_TEIF) + +void uartTxDmaStop(uartPort_t *s) +{ + if (s->txDma) { + const uint32_t stream = DMATAG_GET_STREAM(s->txDma->tag); // LL_DMA_STREAM_n is n + CLEAR_BIT(s->USARTx->CR3, USART_CR3_DMAT); + LL_DMA_DisableIT_TC(s->txDma->dma, stream); + LL_DMA_DisableIT_TE(s->txDma->dma, stream); + LL_DMA_DisableStream(s->txDma->dma, stream); + while (LL_DMA_IsEnabledStream(s->txDma->dma, stream)); + DMA_CLEAR_FLAG(s->txDma, UART_TX_DMA_FLAGS); + uartDmaRelease(s->txDma); + s->txDma = NULL; + s->txDmaCount = 0; + } +} + +void uartStartTxDMA(uartPort_t *s) +{ + // Masked: the end-of-transfer interrupt also starts the next one + ATOMIC_BLOCK(NVIC_PRIO_SERIALUART) { + const uint32_t head = s->port.txBufferHead; + const uint32_t tail = s->port.txBufferTail; + + if (s->txDma && !s->txDmaCount && head != tail) { + // Up to the head, or to the end of the ring if the queue wraps: the rest follows + const uint32_t count = (head > tail) ? head - tail : s->port.txBufferSize - tail; + const uint32_t stream = DMATAG_GET_STREAM(s->txDma->tag); + + s->txDmaCount = count; + LL_DMA_SetMemoryAddress(s->txDma->dma, stream, (uint32_t)&s->port.txBuffer[tail]); + LL_DMA_SetDataLength(s->txDma->dma, stream, count); + // Also orders the writes to the ring before the stream starts + uartTxDmaCleanCache(&s->port.txBuffer[tail], count); + LL_DMA_EnableStream(s->txDma->dma, stream); + } + } +} + +// An error ends the transfer too, or the port would wait for it forever +static void uartTxDmaHandler(DMA_t dma) +{ + uartPort_t *s = (uartPort_t *)dma->userParam; + + // Clear every flag: one left set re-raises the interrupt, starving USB at this priority + const bool ended = DMA_GET_FLAG_STATUS(dma, UART_TX_DMA_ENDED); + DMA_CLEAR_FLAG(dma, UART_TX_DMA_FLAGS); + + if (ended) { + // After an error the rest goes again, unless nothing went: that error would only repeat + const uint32_t sent = s->txDmaCount - LL_DMA_GetDataLength(dma->dma, DMATAG_GET_STREAM(dma->tag)); + s->port.txBufferTail = (s->port.txBufferTail + (sent ? sent : s->txDmaCount)) % s->port.txBufferSize; + s->txDmaCount = 0; + uartStartTxDMA(s); + } +} + +// USART_CR3_DMAT is set in uartReconfigure(), as DMAR is +bool uartTxDmaStart(uartPort_t *s) +{ + uartTxDmaStop(s); + + const UARTDevice_e device = uartDeviceOf(s); + if (device == UARTDEV_MAX || !(s->port.mode & MODE_TX)) { + return false; + } + + const dmaTag_t tag = uartDmaPick(uartTxDmaTag[device], uartTxDmaCandidates, device, &s->port, uartRxDmaOf(s)); + const DMA_t dma = dmaGetByTag(tag); + if (!dma) { + return false; + } +#ifdef USE_UART_RX_DMA + // The stream this port receives on is this UART's too, so it looks free to it + if (dma == s->rxDma) { + return false; + } +#endif + + dmaInit(dma, OWNER_SERIAL, RESOURCE_INDEX(device)); + dmaSetHandler(dma, uartTxDmaHandler, NVIC_PRIO_SERIALUART, (uint32_t)s); + + const uint32_t stream = DMATAG_GET_STREAM(tag); + LL_DMA_DeInit(dma->dma, stream); + + LL_DMA_InitTypeDef init; + LL_DMA_StructInit(&init); + init.Channel = DMATAG_GET_CHANNEL(tag) << DMA_SxCR_CHSEL_Pos; // LL_DMA_CHANNEL_n is n in CHSEL + init.PeriphOrM2MSrcAddress = (uint32_t)&s->USARTx->TDR; + init.MemoryOrM2MDstAddress = (uint32_t)s->port.txBuffer; + init.Direction = LL_DMA_DIRECTION_MEMORY_TO_PERIPH; + init.Mode = LL_DMA_MODE_NORMAL; + init.PeriphOrM2MSrcIncMode = LL_DMA_PERIPH_NOINCREMENT; + init.MemoryOrM2MDstIncMode = LL_DMA_MEMORY_INCREMENT; + init.PeriphOrM2MSrcDataSize = LL_DMA_PDATAALIGN_BYTE; + init.MemoryOrM2MDstDataSize = LL_DMA_MDATAALIGN_BYTE; + init.Priority = LL_DMA_PRIORITY_MEDIUM; + init.FIFOMode = LL_DMA_FIFOMODE_DISABLE; + LL_DMA_Init(dma->dma, stream, &init); + DMA_CLEAR_FLAG(dma, UART_TX_DMA_FLAGS); + LL_DMA_EnableIT_TC(dma->dma, stream); + LL_DMA_EnableIT_TE(dma->dma, stream); + + s->txDmaCount = 0; + s->txDma = dma; + return true; +} +#endif + void uartIrqHandler(uartPort_t *s) { UART_HandleTypeDef *huart = &s->Handle; + +#ifdef USE_UART_RX_DMA + // End of a burst on a receiver that takes them; elsewhere the flag is left to serialIsIdle() + if (READ_BIT(huart->Instance->CR1, USART_CR1_IDLEIE) && __HAL_UART_GET_FLAG(huart, UART_FLAG_IDLE)) { + __HAL_UART_CLEAR_IDLEFLAG(huart); + uartRxDmaIdle(s); + } +#endif + /* UART in mode Receiver ---------------------------------------------------*/ - if ((__HAL_UART_GET_IT(huart, UART_IT_RXNE) != RESET)) { + // With RX on DMA the interrupt still fires for TX: an RX byte is the stream's to take + if ((__HAL_UART_GET_IT(huart, UART_IT_RXNE) != RESET) && !uartRxDmaRunning(s)) { uint8_t rbyte = (uint8_t)(huart->Instance->RDR & (uint8_t) 0xff); if (s->port.rxCallback) { @@ -293,7 +622,8 @@ void uartIrqHandler(uartPort_t *s) } /* UART in mode Transmitter ------------------------------------------------*/ - if (__HAL_UART_GET_IT(huart, UART_IT_TXE) != RESET) { + // With TX on DMA the ring is the stream's to empty + if ((__HAL_UART_GET_IT(huart, UART_IT_TXE) != RESET) && !uartTxDmaRunning(s)) { /* Check that a Tx process is ongoing */ if (huart->gState != HAL_UART_STATE_BUSY_TX) { if (s->port.txBufferTail == s->port.txBufferHead) { @@ -351,6 +681,11 @@ uartPort_t *serialUART(UARTDevice_e device, uint32_t baudRate, portMode_t mode, s->port.baudRate = baudRate; s->port.rxBuffer = uart->rxBuffer; +#if defined(USE_UART_RX_DMA) && defined(DMA_RAM_UNCACHED) + if (uartRxDmaBuffer[device]) { + s->port.rxBuffer = uartRxDmaBuffer[device]; + } +#endif s->port.txBuffer = uart->txBuffer; s->port.rxBufferSize = sizeof(uart->rxBuffer); s->port.txBufferSize = sizeof(uart->txBuffer); diff --git a/src/main/drivers/serial_uart_stm32h7xx.c b/src/main/drivers/serial_uart_stm32h7xx.c index 233aa7057a1..ae8d6758580 100644 --- a/src/main/drivers/serial_uart_stm32h7xx.c +++ b/src/main/drivers/serial_uart_stm32h7xx.c @@ -297,11 +297,350 @@ void uartGetPortPins(UARTDevice_e device, serialPortPins_t * pins) } } +#if defined(USE_UART_RX_DMA) || defined(USE_UART_TX_DMA) +static UARTDevice_e uartDeviceOf(const uartPort_t *s) +{ + for (int device = 0; device < UARTDEV_MAX; device++) { + if (uartHardwareMap[device] && &uartHardwareMap[device]->port == s) { + return device; + } + } + return UARTDEV_MAX; +} + +#endif + +#ifdef USE_UART_RX_DMA +// DMA rings go in D2 SRAM, which the MPU marks shareable and so uncached: in AXI SRAM the CPU +// would read the cache rather than what the stream wrote +#if defined(UART1_RX_DMA) && (UART1_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart1RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif +#if defined(UART2_RX_DMA) && (UART2_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart2RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif +#if defined(UART3_RX_DMA) && (UART3_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart3RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif +#if defined(UART4_RX_DMA) && (UART4_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart4RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif +#if defined(UART5_RX_DMA) && (UART5_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart5RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif +#if defined(UART6_RX_DMA) && (UART6_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart6RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif +#if defined(UART7_RX_DMA) && (UART7_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart7RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif +#if defined(UART8_RX_DMA) && (UART8_RX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart8RxDmaBuffer[UART_RX_BUFFER_SIZE]; +#endif + +typedef struct { + dmaTag_t tag; + uint32_t request; // this UART's receiver on the DMAMUX + volatile uint8_t *buffer; +} uartRxDmaConfig_t; + +static const uartRxDmaConfig_t uartRxDmaConfig[UARTDEV_MAX] = { +#if defined(UART1_RX_DMA) && (UART1_RX_DMA != DMA_NONE) + [UARTDEV_1] = { UART1_RX_DMA, LL_DMAMUX1_REQ_USART1_RX, uart1RxDmaBuffer }, +#endif +#if defined(UART2_RX_DMA) && (UART2_RX_DMA != DMA_NONE) + [UARTDEV_2] = { UART2_RX_DMA, LL_DMAMUX1_REQ_USART2_RX, uart2RxDmaBuffer }, +#endif +#if defined(UART3_RX_DMA) && (UART3_RX_DMA != DMA_NONE) + [UARTDEV_3] = { UART3_RX_DMA, LL_DMAMUX1_REQ_USART3_RX, uart3RxDmaBuffer }, +#endif +#if defined(UART4_RX_DMA) && (UART4_RX_DMA != DMA_NONE) + [UARTDEV_4] = { UART4_RX_DMA, LL_DMAMUX1_REQ_UART4_RX, uart4RxDmaBuffer }, +#endif +#if defined(UART5_RX_DMA) && (UART5_RX_DMA != DMA_NONE) + [UARTDEV_5] = { UART5_RX_DMA, LL_DMAMUX1_REQ_UART5_RX, uart5RxDmaBuffer }, +#endif +#if defined(UART6_RX_DMA) && (UART6_RX_DMA != DMA_NONE) + [UARTDEV_6] = { UART6_RX_DMA, LL_DMAMUX1_REQ_USART6_RX, uart6RxDmaBuffer }, +#endif +#if defined(UART7_RX_DMA) && (UART7_RX_DMA != DMA_NONE) + [UARTDEV_7] = { UART7_RX_DMA, LL_DMAMUX1_REQ_UART7_RX, uart7RxDmaBuffer }, +#endif +#if defined(UART8_RX_DMA) && (UART8_RX_DMA != DMA_NONE) + [UARTDEV_8] = { UART8_RX_DMA, LL_DMAMUX1_REQ_UART8_RX, uart8RxDmaBuffer }, +#endif +}; + +#define UART_RX_DMA_FLAGS (DMA_IT_TCIF | DMA_IT_HTIF | DMA_IT_TEIF | DMA_IT_DMEIF | DMA_IT_FEIF) + +void uartRxDmaStop(uartPort_t *s) +{ + if (s->rxDma) { + const uint32_t stream = DMATAG_GET_STREAM(s->rxDma->tag); // LL_DMA_STREAM_n is n + CLEAR_BIT(s->USARTx->CR3, USART_CR3_DMAR); + CLEAR_BIT(s->USARTx->CR1, USART_CR1_IDLEIE); + LL_DMA_DisableIT_HT(s->rxDma->dma, stream); + LL_DMA_DisableIT_TC(s->rxDma->dma, stream); + LL_DMA_DisableStream(s->rxDma->dma, stream); + while (LL_DMA_IsEnabledStream(s->rxDma->dma, stream)); + DMA_CLEAR_FLAG(s->rxDma, UART_RX_DMA_FLAGS); + uartDmaRelease(s->rxDma); + s->rxDma = NULL; + s->port.rxBursts = false; + } +} + +// Half and full ring: a burst longer than half the ring is handed over before the stream overwrites it +static void uartRxDmaHandler(DMA_t dma) +{ + DMA_CLEAR_FLAG(dma, UART_RX_DMA_FLAGS); + uartRxDmaDeliver((uartPort_t *)dma->userParam, microsISR()); +} + +// USART_CR3_DMAR is set again in uartReconfigure(): the HAL clears CR3 whenever it reprograms the port +bool uartRxDmaStart(uartPort_t *s) +{ + uartRxDmaStop(s); + + const UARTDevice_e device = uartDeviceOf(s); + if (device == UARTDEV_MAX || !(s->port.mode & MODE_RX) || uartRxDmaRefused(s)) { + return false; + } + + const dmaTag_t tag = uartDmaPick(uartRxDmaConfig[device].tag, NULL, device, &s->port, uartTxDmaOf(s)); + const DMA_t dma = dmaGetByTag(tag); + if (!dma) { + return false; + } + + dmaInit(dma, OWNER_SERIAL, RESOURCE_INDEX(device)); + + const uint32_t stream = DMATAG_GET_STREAM(tag); + LL_DMA_DeInit(dma->dma, stream); + + LL_DMA_InitTypeDef init; + LL_DMA_StructInit(&init); + init.PeriphRequest = uartRxDmaConfig[device].request; + init.PeriphOrM2MSrcAddress = (uint32_t)&s->USARTx->RDR; + init.MemoryOrM2MDstAddress = (uint32_t)s->port.rxBuffer; + init.Direction = LL_DMA_DIRECTION_PERIPH_TO_MEMORY; + init.Mode = LL_DMA_MODE_CIRCULAR; + init.PeriphOrM2MSrcIncMode = LL_DMA_PERIPH_NOINCREMENT; + init.MemoryOrM2MDstIncMode = LL_DMA_MEMORY_INCREMENT; + init.PeriphOrM2MSrcDataSize = LL_DMA_PDATAALIGN_BYTE; + init.MemoryOrM2MDstDataSize = LL_DMA_MDATAALIGN_BYTE; + init.NbData = s->port.rxBufferSize; + init.Priority = LL_DMA_PRIORITY_MEDIUM; + // No FIFO: a byte is in the ring as soon as the count says so + init.FIFOMode = LL_DMA_FIFOMODE_DISABLE; + LL_DMA_Init(dma->dma, stream, &init); + + s->port.rxBufferHead = s->port.rxBufferTail = 0; + s->rxDma = dma; + + if (s->port.rxCallback) { + // The UART's priority, so the two never hand bytes over at once + dmaSetHandler(dma, uartRxDmaHandler, NVIC_PRIO_SERIALUART, (uint32_t)s); + DMA_CLEAR_FLAG(dma, UART_RX_DMA_FLAGS); + LL_DMA_EnableIT_HT(dma->dma, stream); + LL_DMA_EnableIT_TC(dma->dma, stream); + s->port.rxBursts = true; + SET_BIT(s->USARTx->CR1, USART_CR1_IDLEIE); + } + + LL_DMA_EnableStream(dma->dma, stream); + // Restarted on a new ring (serialSetRxBuffer) the port gets no reprogramming to set it + SET_BIT(s->USARTx->CR3, USART_CR3_DMAR); + return true; +} +#endif + +#ifdef USE_UART_TX_DMA +// TX rings go in uncached D2 SRAM too, so the stream reads what the CPU wrote +#if defined(UART1_TX_DMA) && (UART1_TX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart1TxDmaBuffer[UART_TX_BUFFER_SIZE]; +#endif +#if defined(UART2_TX_DMA) && (UART2_TX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart2TxDmaBuffer[UART_TX_BUFFER_SIZE]; +#endif +#if defined(UART3_TX_DMA) && (UART3_TX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart3TxDmaBuffer[UART_TX_BUFFER_SIZE]; +#endif +#if defined(UART4_TX_DMA) && (UART4_TX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart4TxDmaBuffer[UART_TX_BUFFER_SIZE]; +#endif +#if defined(UART5_TX_DMA) && (UART5_TX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart5TxDmaBuffer[UART_TX_BUFFER_SIZE]; +#endif +#if defined(UART6_TX_DMA) && (UART6_TX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart6TxDmaBuffer[UART_TX_BUFFER_SIZE]; +#endif +#if defined(UART7_TX_DMA) && (UART7_TX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart7TxDmaBuffer[UART_TX_BUFFER_SIZE]; +#endif +#if defined(UART8_TX_DMA) && (UART8_TX_DMA != DMA_NONE) +static DMA_RAM uint8_t uart8TxDmaBuffer[UART_TX_BUFFER_SIZE]; +#endif + +typedef struct { + dmaTag_t tag; + uint32_t request; // this UART's transmitter on the DMAMUX + volatile uint8_t *buffer; +} uartTxDmaConfig_t; + +static const uartTxDmaConfig_t uartTxDmaConfig[UARTDEV_MAX] = { +#if defined(UART1_TX_DMA) && (UART1_TX_DMA != DMA_NONE) + [UARTDEV_1] = { UART1_TX_DMA, LL_DMAMUX1_REQ_USART1_TX, uart1TxDmaBuffer }, +#endif +#if defined(UART2_TX_DMA) && (UART2_TX_DMA != DMA_NONE) + [UARTDEV_2] = { UART2_TX_DMA, LL_DMAMUX1_REQ_USART2_TX, uart2TxDmaBuffer }, +#endif +#if defined(UART3_TX_DMA) && (UART3_TX_DMA != DMA_NONE) + [UARTDEV_3] = { UART3_TX_DMA, LL_DMAMUX1_REQ_USART3_TX, uart3TxDmaBuffer }, +#endif +#if defined(UART4_TX_DMA) && (UART4_TX_DMA != DMA_NONE) + [UARTDEV_4] = { UART4_TX_DMA, LL_DMAMUX1_REQ_UART4_TX, uart4TxDmaBuffer }, +#endif +#if defined(UART5_TX_DMA) && (UART5_TX_DMA != DMA_NONE) + [UARTDEV_5] = { UART5_TX_DMA, LL_DMAMUX1_REQ_UART5_TX, uart5TxDmaBuffer }, +#endif +#if defined(UART6_TX_DMA) && (UART6_TX_DMA != DMA_NONE) + [UARTDEV_6] = { UART6_TX_DMA, LL_DMAMUX1_REQ_USART6_TX, uart6TxDmaBuffer }, +#endif +#if defined(UART7_TX_DMA) && (UART7_TX_DMA != DMA_NONE) + [UARTDEV_7] = { UART7_TX_DMA, LL_DMAMUX1_REQ_UART7_TX, uart7TxDmaBuffer }, +#endif +#if defined(UART8_TX_DMA) && (UART8_TX_DMA != DMA_NONE) + [UARTDEV_8] = { UART8_TX_DMA, LL_DMAMUX1_REQ_UART8_TX, uart8TxDmaBuffer }, +#endif +}; + +#define UART_TX_DMA_FLAGS (DMA_IT_TCIF | DMA_IT_HTIF | DMA_IT_TEIF | DMA_IT_DMEIF | DMA_IT_FEIF) +#define UART_TX_DMA_ENDED (DMA_IT_TCIF | DMA_IT_TEIF) + +void uartTxDmaStop(uartPort_t *s) +{ + if (s->txDma) { + const uint32_t stream = DMATAG_GET_STREAM(s->txDma->tag); // LL_DMA_STREAM_n is n + CLEAR_BIT(s->USARTx->CR3, USART_CR3_DMAT); + LL_DMA_DisableIT_TC(s->txDma->dma, stream); + LL_DMA_DisableIT_TE(s->txDma->dma, stream); + LL_DMA_DisableStream(s->txDma->dma, stream); + while (LL_DMA_IsEnabledStream(s->txDma->dma, stream)); + DMA_CLEAR_FLAG(s->txDma, UART_TX_DMA_FLAGS); + uartDmaRelease(s->txDma); + s->txDma = NULL; + s->txDmaCount = 0; + } +} + +void uartStartTxDMA(uartPort_t *s) +{ + // Masked: the end-of-transfer interrupt also starts the next one + ATOMIC_BLOCK(NVIC_PRIO_SERIALUART) { + const uint32_t head = s->port.txBufferHead; + const uint32_t tail = s->port.txBufferTail; + + if (s->txDma && !s->txDmaCount && head != tail) { + // Up to the head, or to the end of the ring if the queue wraps: the rest follows + const uint32_t count = (head > tail) ? head - tail : s->port.txBufferSize - tail; + const uint32_t stream = DMATAG_GET_STREAM(s->txDma->tag); + + s->txDmaCount = count; + LL_DMA_SetMemoryAddress(s->txDma->dma, stream, (uint32_t)&s->port.txBuffer[tail]); + LL_DMA_SetDataLength(s->txDma->dma, stream, count); + // Also orders the writes to the ring before the stream starts + uartTxDmaCleanCache(&s->port.txBuffer[tail], count); + LL_DMA_EnableStream(s->txDma->dma, stream); + } + } +} + +// An error ends the transfer too, or the port would wait for it forever +static void uartTxDmaHandler(DMA_t dma) +{ + uartPort_t *s = (uartPort_t *)dma->userParam; + + // Clear every flag: one left set re-raises the interrupt, starving USB at this priority + const bool ended = DMA_GET_FLAG_STATUS(dma, UART_TX_DMA_ENDED); + DMA_CLEAR_FLAG(dma, UART_TX_DMA_FLAGS); + + if (ended) { + // After an error the rest goes again, unless nothing went: that error would only repeat + const uint32_t sent = s->txDmaCount - LL_DMA_GetDataLength(dma->dma, DMATAG_GET_STREAM(dma->tag)); + s->port.txBufferTail = (s->port.txBufferTail + (sent ? sent : s->txDmaCount)) % s->port.txBufferSize; + s->txDmaCount = 0; + uartStartTxDMA(s); + } +} + +// USART_CR3_DMAT is set in uartReconfigure(), as DMAR is +bool uartTxDmaStart(uartPort_t *s) +{ + uartTxDmaStop(s); + + const UARTDevice_e device = uartDeviceOf(s); + if (device == UARTDEV_MAX || !(s->port.mode & MODE_TX)) { + return false; + } + + const dmaTag_t tag = uartDmaPick(uartTxDmaConfig[device].tag, NULL, device, &s->port, uartRxDmaOf(s)); + const DMA_t dma = dmaGetByTag(tag); + if (!dma) { + return false; + } +#ifdef USE_UART_RX_DMA + // The stream this port receives on is this UART's too, so it looks free to it + if (dma == s->rxDma) { + return false; + } +#endif + + dmaInit(dma, OWNER_SERIAL, RESOURCE_INDEX(device)); + dmaSetHandler(dma, uartTxDmaHandler, NVIC_PRIO_SERIALUART, (uint32_t)s); + + const uint32_t stream = DMATAG_GET_STREAM(tag); + LL_DMA_DeInit(dma->dma, stream); + + LL_DMA_InitTypeDef init; + LL_DMA_StructInit(&init); + init.PeriphRequest = uartTxDmaConfig[device].request; + init.PeriphOrM2MSrcAddress = (uint32_t)&s->USARTx->TDR; + init.MemoryOrM2MDstAddress = (uint32_t)s->port.txBuffer; + init.Direction = LL_DMA_DIRECTION_MEMORY_TO_PERIPH; + init.Mode = LL_DMA_MODE_NORMAL; + init.PeriphOrM2MSrcIncMode = LL_DMA_PERIPH_NOINCREMENT; + init.MemoryOrM2MDstIncMode = LL_DMA_MEMORY_INCREMENT; + init.PeriphOrM2MSrcDataSize = LL_DMA_PDATAALIGN_BYTE; + init.MemoryOrM2MDstDataSize = LL_DMA_MDATAALIGN_BYTE; + init.Priority = LL_DMA_PRIORITY_MEDIUM; + init.FIFOMode = LL_DMA_FIFOMODE_DISABLE; + LL_DMA_Init(dma->dma, stream, &init); + DMA_CLEAR_FLAG(dma, UART_TX_DMA_FLAGS); + LL_DMA_EnableIT_TC(dma->dma, stream); + LL_DMA_EnableIT_TE(dma->dma, stream); + + s->txDmaCount = 0; + s->txDma = dma; + return true; +} +#endif + void uartIrqHandler(uartPort_t *s) { UART_HandleTypeDef *huart = &s->Handle; + +#ifdef USE_UART_RX_DMA + // End of a burst on a receiver that takes them; elsewhere the flag is left to serialIsIdle() + if (READ_BIT(huart->Instance->CR1, USART_CR1_IDLEIE) && __HAL_UART_GET_FLAG(huart, UART_FLAG_IDLE)) { + __HAL_UART_CLEAR_IDLEFLAG(huart); + uartRxDmaIdle(s); + } +#endif + /* UART in mode Receiver ---------------------------------------------------*/ - if ((__HAL_UART_GET_IT(huart, UART_IT_RXNE) != RESET)) { + // With RX on DMA the interrupt still fires for TX: an RX byte is the stream's to take + if ((__HAL_UART_GET_IT(huart, UART_IT_RXNE) != RESET) && !uartRxDmaRunning(s)) { uint8_t rbyte = (uint8_t)(huart->Instance->RDR & (uint8_t) 0xff); if (s->port.rxCallback) { @@ -339,7 +678,8 @@ void uartIrqHandler(uartPort_t *s) } /* UART in mode Transmitter ------------------------------------------------*/ - if (__HAL_UART_GET_IT(huart, UART_IT_TXE) != RESET) { + // With TX on DMA the data register is the stream's to feed + if ((__HAL_UART_GET_IT(huart, UART_IT_TXE) != RESET) && !uartTxDmaRunning(s)) { /* Check that a Tx process is ongoing */ if (huart->gState != HAL_UART_STATE_BUSY_TX) { if (s->port.txBufferTail == s->port.txBufferHead) { @@ -383,7 +723,17 @@ uartPort_t *serialUART(UARTDevice_e device, uint32_t baudRate, portMode_t mode, s->port.baudRate = baudRate; s->port.rxBuffer = uart->rxBuffer; +#ifdef USE_UART_RX_DMA + if (uartRxDmaConfig[device].buffer) { + s->port.rxBuffer = uartRxDmaConfig[device].buffer; + } +#endif s->port.txBuffer = uart->txBuffer; +#ifdef USE_UART_TX_DMA + if (uartTxDmaConfig[device].buffer) { + s->port.txBuffer = uartTxDmaConfig[device].buffer; + } +#endif s->port.rxBufferSize = sizeof(uart->rxBuffer); s->port.txBufferSize = sizeof(uart->txBuffer); diff --git a/src/main/fc/fc_msp.c b/src/main/fc/fc_msp.c index 3bf520b977a..db3c4647463 100644 --- a/src/main/fc/fc_msp.c +++ b/src/main/fc/fc_msp.c @@ -223,6 +223,10 @@ static void mspSerialPassthroughFn(serialPort_t *serialPort) { serialPort_t *passthroughPort = mspFindPassthroughSerialPort(); if (passthroughPort && serialPort) { + // As in the CLI: a receiver's callback would take the bytes the session passes on; it is back after +++ + const serialReceiveCallbackPtr rxCallback = passthroughPort->rxCallback; + passthroughPort->rxCallback = NULL; + // The port the request came in on goes first, as it does in the CLI. Both of the // things serialPassthrough() does for whoever opened the session are done for its // first port only: the +++ that ends the session is looked for there, and a USB @@ -230,6 +234,7 @@ static void mspSerialPassthroughFn(serialPort_t *serialPort) // way round, a session opened over MSP could not be closed and could not raise the // rate of the port it opened, which is what an SRXL2 ESC negotiates up to 400000 serialPassthrough(serialPort, passthroughPort, NULL, NULL); + passthroughPort->rxCallback = rxCallback; } } diff --git a/src/main/io/gps.c b/src/main/io/gps.c index 89657541eed..7cdeab04ef0 100755 --- a/src/main/io/gps.c +++ b/src/main/io/gps.c @@ -91,6 +91,22 @@ gpsStatistics_t gpsStats; gpsSolutionData_t gpsSolDRV; //filled by driver gpsSolutionData_t gpsSol; //used in the rest of the code +#ifdef USE_GPS_PROTO_UBLOX +// Covers 230400; at 460800 and above a NAV-PVT plus a large NAV-SIG can still overflow it +#define GPS_RX_BUFFER_SIZE 512 +STATIC_ASSERT((GPS_RX_BUFFER_SIZE & (GPS_RX_BUFFER_SIZE - 1)) == 0, gps_rx_buffer_size_not_power_of_2); +// A ring a stream can reach and the D-cache does not hide: not DTCM or cached RAM on F7 and H7, not CCM on F4 +// (RAM1 on AT32F43x is unknown) +#if defined(USE_UART_RX_DMA) && (defined(STM32F7) || defined(STM32H7)) +#define GPS_RX_BUFFER_RAM DMA_RAM +#elif defined(USE_UART_RX_DMA) && (defined(STM32F4) || defined(AT32F43x)) +#define GPS_RX_BUFFER_RAM +#else +#define GPS_RX_BUFFER_RAM FASTRAM +#endif +static GPS_RX_BUFFER_RAM volatile uint8_t gpsRxBuffer[GPS_RX_BUFFER_SIZE]; +#endif + // Map gpsBaudRate_e index to baudRate_e baudRate_e gpsToSerialBaudRate[GPS_BAUDRATE_COUNT] = { BAUD_115200, BAUD_57600, BAUD_38400, BAUD_19200, BAUD_9600, BAUD_230400, BAUD_460800, BAUD_921600 }; @@ -526,6 +542,10 @@ void gpsInit(void) return; } +#ifdef USE_GPS_PROTO_UBLOX + serialSetRxBuffer(gpsState.gpsPort, gpsRxBuffer, sizeof(gpsRxBuffer)); +#endif + gpsSetState(GPS_INITIALIZING); } diff --git a/src/main/io/gps_ublox.c b/src/main/io/gps_ublox.c index 703242d10dd..193ccf3e998 100755 --- a/src/main/io/gps_ublox.c +++ b/src/main/io/gps_ublox.c @@ -955,6 +955,12 @@ static uint16_t hz2rate(uint8_t hz) return 1000 / hz; } +// Satellite details only feed gpssats and the OSD's GPS extra stats: about 2 Hz, in epochs +static uint8_t ubloxSatInfoRate(void) +{ + return MAX(1, (gpsState.gpsConfig->ubloxNavHz + 1) / 2); +} + STATIC_PROTOTHREAD(gpsConfigure) { ptBegin(gpsConfigure); @@ -1073,7 +1079,7 @@ STATIC_PROTOTHREAD(gpsConfigure) {UBLOX_CFG_MSGOUT_NAV_VELNED_UART1, 0}, // 2 {UBLOX_CFG_MSGOUT_NAV_TIMEUTC_UART1, 0}, // 3 {UBLOX_CFG_MSGOUT_NAV_PVT_UART1, 1}, // 4 - {UBLOX_CFG_MSGOUT_NAV_SIG_UART1, 1}, // 5 + {UBLOX_CFG_MSGOUT_NAV_SIG_UART1, ubloxSatInfoRate()}, // 5 {UBLOX_CFG_MSGOUT_NAV_SAT_UART1, 0} // 6 }; @@ -1101,7 +1107,7 @@ STATIC_PROTOTHREAD(gpsConfigure) configureMSG(MSG_CLASS_UBX, MSG_PVT, 1); ptWait(_ack_state == UBX_ACK_GOT_ACK || _ack_state == UBX_ACK_GOT_NAK); - configureMSG(MSG_CLASS_UBX, MSG_NAV_SAT, 1); + configureMSG(MSG_CLASS_UBX, MSG_NAV_SAT, ubloxSatInfoRate()); ptWait(_ack_state == UBX_ACK_GOT_ACK || _ack_state == UBX_ACK_GOT_NAK); } else { // Really old stuff, consider upgrading :), ols setting API, no PVT or NAV_SAT or NAV_SIG // TODO: remove in INAV 9.0.0 diff --git a/src/main/io/serial.c b/src/main/io/serial.c index c9c73a4eab8..87e6a1589a9 100644 --- a/src/main/io/serial.c +++ b/src/main/io/serial.c @@ -445,6 +445,8 @@ void closeSerialPort(serialPort_t *serialPort) // TODO wait until data has been transmitted. serialPort->rxCallback = NULL; + // A stream would go on filling a ring nobody reads, and stay out of reach of the ports opened later + serialRelease(serialPort); serialPortUsage->function = FUNCTION_NONE; serialPortUsage->serialPort = NULL; diff --git a/src/main/rx/crsf.c b/src/main/rx/crsf.c index e5855205642..962a2e8b05d 100755 --- a/src/main/rx/crsf.c +++ b/src/main/rx/crsf.c @@ -141,7 +141,7 @@ STATIC_UNIT_TESTED void crsfDataReceive(uint16_t c, void *rxCallbackData) UNUSED(rxCallbackData); static uint8_t crsfFramePosition = 0; - const timeUs_t currentTimeUs = microsISR(); + const timeUs_t currentTimeUs = serialRxByteTimeUs(serialPort); #ifdef DEBUG_CRSF_PACKETS debug[2] = now - crsfFrameStartAt; @@ -335,7 +335,7 @@ bool crsfRxInit(const rxConfig_t *rxConfig, rxRuntimeConfig_t *rxRuntimeConfig) NULL, CRSF_BAUDRATE, CRSF_PORT_MODE, - CRSF_PORT_OPTIONS | (tristateWithDefaultOffIsActive(rxConfig->halfDuplex) ? SERIAL_BIDIR : 0) + CRSF_PORT_OPTIONS | SERIAL_RX_BURSTS | (tristateWithDefaultOffIsActive(rxConfig->halfDuplex) ? SERIAL_BIDIR : 0) ); return serialPort != NULL; diff --git a/src/main/rx/sbus.c b/src/main/rx/sbus.c index f08267eeb1d..4837a321b6e 100644 --- a/src/main/rx/sbus.c +++ b/src/main/rx/sbus.c @@ -67,6 +67,7 @@ typedef struct sbusFrameData_s { uint8_t buffer[SBUS_FRAME_SIZE]; uint8_t position; timeUs_t lastActivityTimeUs; + serialPort_t *port; } sbusFrameData_t; static uint8_t sbus2ActiveTelemetryPage = 0; @@ -78,7 +79,7 @@ timeUs_t frameTime = 0; static void sbusDataReceive(uint16_t c, void *data) { sbusFrameData_t *sbusFrameData = data; - const timeUs_t currentTimeUs = micros(); + const timeUs_t currentTimeUs = serialRxByteTimeUs(sbusFrameData->port); const timeDelta_t timeSinceLastByteUs = cmpTimeUs(currentTimeUs, sbusFrameData->lastActivityTimeUs); sbusFrameData->lastActivityTimeUs = currentTimeUs; @@ -255,11 +256,12 @@ static bool sbusInitEx(const rxConfig_t *rxConfig, rxRuntimeConfig_t *rxRuntimeC &sbusFrameData, sbusBaudRate, (portShared || rxConfig->serialrx_provider == SERIALRX_SBUS2) ? MODE_RXTX : MODE_RX, - SBUS_PORT_OPTIONS | + SBUS_PORT_OPTIONS | SERIAL_RX_BURSTS | (rxConfig->serialrx_inverted ? 0 : SERIAL_INVERTED) | ((rxConfig->serialrx_provider == SERIALRX_SBUS2) ? SERIAL_BIDIR : 0) | (tristateWithDefaultOffIsActive(rxConfig->halfDuplex) ? SERIAL_BIDIR : 0) ); + sbusFrameData.port = sBusPort; #ifdef USE_TELEMETRY if (portShared || (rxConfig->serialrx_provider == SERIALRX_SBUS2)) { diff --git a/src/main/target/TBS_LUCID_H7_WING/target.h b/src/main/target/TBS_LUCID_H7_WING/target.h index 93d3888770e..ef84c003c43 100644 --- a/src/main/target/TBS_LUCID_H7_WING/target.h +++ b/src/main/target/TBS_LUCID_H7_WING/target.h @@ -46,6 +46,9 @@ #define USE_UART2 #define UART2_TX_PIN PD5 #define UART2_RX_PIN PD6 +// Outputs are mapped to DMA1 0-7 and DMA2 1-4, the ADC uses DMA2 0 +#define UART2_RX_DMA DMA_TAG(2, 5, 0) +#define UART2_TX_DMA DMA_TAG(2, 7, 0) #define USE_UART3 #define UART3_TX_PIN PD8 diff --git a/src/main/target/common_post.h b/src/main/target/common_post.h index 183b44a3908..1598f765f80 100644 --- a/src/main/target/common_post.h +++ b/src/main/target/common_post.h @@ -185,6 +185,70 @@ extern uint8_t __config_end; #endif #endif +// A port with no stream named gets a free one at runtime; DMA_NONE keeps it on the interrupt +#if defined(STM32F4) || defined(STM32F7) || defined(STM32H7) || defined(AT32F43x) +#if defined(USE_UART1) && !defined(UART1_RX_DMA) +#define UART1_RX_DMA DMA_TAG_AUTO +#endif +#if defined(USE_UART1) && !defined(UART1_TX_DMA) +#define UART1_TX_DMA DMA_TAG_AUTO +#endif +#if defined(USE_UART2) && !defined(UART2_RX_DMA) +#define UART2_RX_DMA DMA_TAG_AUTO +#endif +#if defined(USE_UART2) && !defined(UART2_TX_DMA) +#define UART2_TX_DMA DMA_TAG_AUTO +#endif +#if defined(USE_UART3) && !defined(UART3_RX_DMA) +#define UART3_RX_DMA DMA_TAG_AUTO +#endif +#if defined(USE_UART3) && !defined(UART3_TX_DMA) +#define UART3_TX_DMA DMA_TAG_AUTO +#endif +#if defined(USE_UART4) && !defined(UART4_RX_DMA) +#define UART4_RX_DMA DMA_TAG_AUTO +#endif +#if defined(USE_UART4) && !defined(UART4_TX_DMA) +#define UART4_TX_DMA DMA_TAG_AUTO +#endif +#if defined(USE_UART5) && !defined(UART5_RX_DMA) +#define UART5_RX_DMA DMA_TAG_AUTO +#endif +#if defined(USE_UART5) && !defined(UART5_TX_DMA) +#define UART5_TX_DMA DMA_TAG_AUTO +#endif +#if defined(USE_UART6) && !defined(UART6_RX_DMA) +#define UART6_RX_DMA DMA_TAG_AUTO +#endif +#if defined(USE_UART6) && !defined(UART6_TX_DMA) +#define UART6_TX_DMA DMA_TAG_AUTO +#endif +#if defined(USE_UART7) && !defined(UART7_RX_DMA) +#define UART7_RX_DMA DMA_TAG_AUTO +#endif +#if defined(USE_UART7) && !defined(UART7_TX_DMA) +#define UART7_TX_DMA DMA_TAG_AUTO +#endif +#if defined(USE_UART8) && !defined(UART8_RX_DMA) +#define UART8_RX_DMA DMA_TAG_AUTO +#endif +#if defined(USE_UART8) && !defined(UART8_TX_DMA) +#define UART8_TX_DMA DMA_TAG_AUTO +#endif +#endif + +// Which stream is free depends on the board: the timers, the ADC and the SD card share them +#if defined(UART1_RX_DMA) || defined(UART2_RX_DMA) || defined(UART3_RX_DMA) || defined(UART4_RX_DMA) || \ + defined(UART5_RX_DMA) || defined(UART6_RX_DMA) || defined(UART7_RX_DMA) || defined(UART8_RX_DMA) +#define USE_UART_RX_DMA +#endif + +// The same for sending (UARTx_TX_DMA): one interrupt per transfer instead of one per byte +#if defined(UART1_TX_DMA) || defined(UART2_TX_DMA) || defined(UART3_TX_DMA) || defined(UART4_TX_DMA) || \ + defined(UART5_TX_DMA) || defined(UART6_TX_DMA) || defined(UART7_TX_DMA) || defined(UART8_TX_DMA) +#define USE_UART_TX_DMA +#endif + #ifdef USE_ESC_SENSOR #define USE_RPM_FILTER #endif diff --git a/src/test/unit/gps_heartbeat_unittest.cc b/src/test/unit/gps_heartbeat_unittest.cc index 75505ec1ff0..e79dc19a9ae 100644 --- a/src/test/unit/gps_heartbeat_unittest.cc +++ b/src/test/unit/gps_heartbeat_unittest.cc @@ -72,6 +72,7 @@ void serialWriteBuf(serialPort_t *instance, const uint8_t *data, int count) { UN void serialPrint(serialPort_t *instance, const char *str) { UNUSED(instance); UNUSED(str); } void serialSetMode(serialPort_t *instance, portMode_t mode) { UNUSED(instance); UNUSED(mode); } void serialSetBaudRate(serialPort_t *instance, uint32_t baudRate) { UNUSED(instance); UNUSED(baudRate); } +void serialSetRxBuffer(serialPort_t *instance, volatile uint8_t *buffer, uint32_t size) { UNUSED(instance); UNUSED(buffer); UNUSED(size); } serialPort_t *openSerialPort(serialPortIdentifier_e identifier, serialPortFunction_e function, serialReceiveCallbackPtr rxCallback, void *rxCallbackData, diff --git a/src/test/unit/gps_null_port_unittest.cc b/src/test/unit/gps_null_port_unittest.cc index 288b68995c9..cd41bb2e6ff 100644 --- a/src/test/unit/gps_null_port_unittest.cc +++ b/src/test/unit/gps_null_port_unittest.cc @@ -189,6 +189,13 @@ void serialSetBaudRate(serialPort_t *instance, uint32_t baudRate) UNUSED(baudRate); } +void serialSetRxBuffer(serialPort_t *instance, volatile uint8_t *buffer, uint32_t size) +{ + UNUSED(instance); + UNUSED(buffer); + UNUSED(size); +} + serialPort_t *openSerialPort(serialPortIdentifier_e identifier, serialPortFunction_e function, serialReceiveCallbackPtr rxCallback, diff --git a/src/test/unit/uart_dma_stream_unittest.cc b/src/test/unit/uart_dma_stream_unittest.cc new file mode 100644 index 00000000000..3190f4c0af3 --- /dev/null +++ b/src/test/unit/uart_dma_stream_unittest.cc @@ -0,0 +1,736 @@ +/* + * This file is part of INAV. + * + * INAV is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * INAV is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with INAV. If not, see . + */ + +/* + * Which DMA streams a UART may take: pwmIsDmaStreamReserved() (pwm_mapping.c) and uartDmaStreamAvailable() + * (serial_uart_impl.h). pwm_mapping.c is not built for the host, so, as in pwm_mapping_beeper_unittest.cc, the + * logic is mirrored on plain data and a SourceSync test checks that the live function still reads the same. + */ + +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "unittest_macros.h" + +namespace { + +enum { TIM_USE_MOTOR = 1 << 2, TIM_USE_SERVO = 1 << 3, TIM_USE_LED = 1 << 24 }; +enum { OUTPUT_MODE_AUTO = 0, OUTPUT_MODE_LED = 3 }; +enum owner_e { OWNER_FREE, OWNER_TIMER, OWNER_SERIAL }; + +struct pad_t { + int timer; // index into the per-timer output modes + int stream; // the DMA stream the pad is mapped to, -1 for none + uint32_t usageFlags; +}; + +struct board_t { + const pad_t *pads; + int padCount; + uint8_t outputMode[4]; + bool dshot; + bool ledStrip; + bool motorsInitialised; +}; + +// Mirror of pwmIsDmaStreamReserved() +bool reserved(const board_t &b, int stream) +{ + const bool motorsWillClaim = !b.motorsInitialised && b.dshot; + + for (int i = 0; i < b.padCount; i++) { + const pad_t *timHw = &b.pads[i]; + if (timHw->stream != stream) { + continue; + } + if (motorsWillClaim) { + return true; + } + if (b.ledStrip && ((timHw->usageFlags & TIM_USE_LED) || b.outputMode[timHw->timer] == OUTPUT_MODE_LED)) { + return true; + } + } + return false; +} + +// Mirror of uartDmaStreamAvailable() +bool available(const board_t &b, int stream, owner_e owner) +{ + if (reserved(b, stream)) { + return false; + } + return owner == OWNER_FREE; +} + +// Four motor-capable outputs on streams 0-3, a LED pad on stream 4 (timer 2), stream 6 unmapped +const pad_t pads[] = { + { 0, 0, TIM_USE_MOTOR | TIM_USE_SERVO }, + { 0, 1, TIM_USE_MOTOR | TIM_USE_SERVO }, + { 1, 2, TIM_USE_MOTOR | TIM_USE_SERVO }, + { 1, 3, TIM_USE_MOTOR | TIM_USE_SERVO }, + { 2, 4, TIM_USE_LED }, +}; + +board_t board(bool dshot, bool ledStrip, bool motorsInitialised) +{ + return board_t{ pads, (int)(sizeof(pads) / sizeof(pads[0])), { OUTPUT_MODE_AUTO, OUTPUT_MODE_AUTO, OUTPUT_MODE_AUTO, OUTPUT_MODE_AUTO }, + dshot, ledStrip, motorsInitialised }; +} + +} // namespace + +TEST(UartDmaStream, DshotMotorsNotStartedReserveEveryOutputStream) +{ + const board_t b = board(true, false, false); + for (int s = 0; s <= 4; s++) { + EXPECT_TRUE(reserved(b, s)) << "stream " << s; + } + EXPECT_FALSE(reserved(b, 6)); +} + +TEST(UartDmaStream, StartedMotorsLeaveTheRestToOwnership) +{ + const board_t b = board(true, false, true); + for (int s = 0; s <= 4; s++) { + EXPECT_FALSE(reserved(b, s)) << "stream " << s; + } + // The started motors own their streams; an unused output's stream is free + EXPECT_FALSE(available(b, 0, OWNER_TIMER)); + EXPECT_TRUE(available(b, 3, OWNER_FREE)); +} + +TEST(UartDmaStream, MotorsOffDshotNeverReserveMotorStreams) +{ + const board_t b = board(false, false, false); + for (int s = 0; s <= 4; s++) { + EXPECT_FALSE(reserved(b, s)) << "stream " << s; + } +} + +TEST(UartDmaStream, LedStripKeepsItsStreamOnlyWhenEnabled) +{ + EXPECT_TRUE(reserved(board(false, true, false), 4)); + EXPECT_TRUE(reserved(board(true, true, true), 4)); + EXPECT_FALSE(reserved(board(false, false, false), 4)); + EXPECT_FALSE(reserved(board(true, false, true), 4)); +} + +TEST(UartDmaStream, LedOutputModeReservesTheTimersPads) +{ + board_t b = board(false, true, true); + b.outputMode[1] = OUTPUT_MODE_LED; + EXPECT_TRUE(reserved(b, 2)); + EXPECT_TRUE(reserved(b, 3)); + EXPECT_FALSE(reserved(b, 0)); +} + +TEST(UartDmaStream, OwnershipStillDecidesFreeStreams) +{ + const board_t b = board(false, false, true); + EXPECT_TRUE(available(b, 6, OWNER_FREE)); + // A port holds only the streams it runs on, this one's included: every stop gives its stream back + EXPECT_FALSE(available(b, 6, OWNER_SERIAL)); + EXPECT_FALSE(available(b, 6, OWNER_TIMER)); +} + +/* + * SourceSync: the live pwmIsDmaStreamReserved() must read as below, whitespace and line comments aside, so the + * mirror above keeps testing the real logic. + */ +namespace { + +std::string normalize(const std::string &text) +{ + std::string noComments; + for (size_t i = 0; i < text.size();) { + if (text[i] == '/' && i + 1 < text.size() && text[i + 1] == '/') { + while (i < text.size() && text[i] != '\n') { + i++; + } + continue; + } + noComments += text[i++]; + } + std::string result; + bool lastWasSpace = true; + for (char c : noComments) { + if (std::isspace(static_cast(c))) { + if (!lastWasSpace) { + result += ' '; + lastWasSpace = true; + } + } else { + result += c; + lastWasSpace = false; + } + } + while (!result.empty() && result.back() == ' ') { + result.pop_back(); + } + return result; +} + +const char *expectedFunction = R"( +bool pwmIsDmaStreamReserved(DMA_t dma) +{ + const bool motorsWillClaim = !motorsInitialised && getMotorProtocolProperties(motorConfig()->motorPwmProtocol)->isDSHOT; + + for (int i = 0; i < timerHardwareCount; i++) { + const timerHardware_t *timHw = &timerHardware[i]; + if (dmaGetByTag(timHw->dmaTag) != dma) { + continue; + } + if (motorsWillClaim) { + return true; + } + if (feature(FEATURE_LED_STRIP) && (TIM_IS_LED(timHw->usageFlags) || timerOverrides(timer2id(timHw->tim))->outputMode == OUTPUT_MODE_LED)) { + return true; + } + } + return false; +} +)"; + +const char *expectedAvailable = R"( +static inline bool uartDmaStreamAvailable(DMA_t dma) +{ + if (pwmIsDmaStreamReserved(dma)) { + return false; + } + return dmaGetOwner(dma) == OWNER_FREE; +} +)"; + +::testing::AssertionResult liveSourceContains(const char *relativePath, const char *snippet) +{ + std::string thisFile = __FILE__; + const size_t slash = thisFile.find_last_of("/\\"); + const std::string path = (slash == std::string::npos ? std::string(".") : thisFile.substr(0, slash)) + + "/../../main/drivers/" + relativePath; + std::ifstream file(path); + if (!file.is_open()) { + return ::testing::AssertionFailure() << "cannot open " << path; + } + std::ostringstream buffer; + buffer << file.rdbuf(); + if (normalize(buffer.str()).find(normalize(snippet)) == std::string::npos) { + return ::testing::AssertionFailure() << relativePath << " changed: update the mirror in this test"; + } + return ::testing::AssertionSuccess(); +} + +} // namespace + +TEST(UartDmaStreamSourceSync, ReservedMatchesMirror) +{ + EXPECT_TRUE(liveSourceContains("pwm_mapping.c", expectedFunction)); +} + +TEST(UartDmaStreamSourceSync, AvailableMatchesMirror) +{ + EXPECT_TRUE(liveSourceContains("serial_uart_impl.h", expectedAvailable)); +} + +/* + * uartDmaPick() (serial_uart_impl.h): the stream a port starts on. Mirrored on plain tags: a stream is "unavailable" + * when uartDmaStreamAvailable() would refuse it, "used elsewhere" when it is the SD card's (or the AT32 ADC's). + */ +namespace { + +const uint32_t TAG_NONE = 0; +const uint32_t TAG_AUTO = 0xFFFF0000U; + +uint32_t tag(int dma, int stream, int channel) +{ + return ((dma & 0x03) << 12) | ((stream & 0x0F) << 8) | (channel & 0xFF); +} + +// dmaGetByTag() matches controller and stream; a tag with no controller has no stream +int streamOf(uint32_t t) +{ + return ((t >> 12) & 0x03) ? (int)((t >> 8) & 0x3F) : -1; +} + +struct dmaWorld_t { + int unavailable[4]; + int usedElsewhere[2]; + int named[2]; // streams a target names for some port +}; + +bool listed(const int *list, int count, int stream) +{ + for (int i = 0; i < count; i++) { + if (list[i] == stream) { + return true; + } + } + return false; +} + +const uint32_t rxCandidates[8][2] = { + { tag(2, 2, 4), tag(2, 5, 4) }, { tag(1, 5, 4) }, { tag(1, 1, 4) }, { tag(1, 2, 4) }, + { tag(1, 0, 4) }, { tag(2, 1, 5), tag(2, 2, 5) }, { tag(1, 3, 5) }, { tag(1, 6, 5) }, +}; +const uint32_t txCandidates[8][2] = { + { tag(2, 7, 4) }, { tag(1, 6, 4) }, { tag(1, 3, 4), tag(1, 4, 7) }, { tag(1, 4, 4) }, + { tag(1, 7, 4) }, { tag(2, 6, 5), tag(2, 7, 5) }, { tag(1, 1, 5) }, { tag(1, 0, 5) }, +}; + +// Mirror of uartDmaPick() for F4/F7; bursts is SERIAL_RX_BURSTS +uint32_t pick(uint32_t named, const uint32_t (*candidates)[2], int device, bool rxCallback, bool halfDuplex, + int otherDirection, const dmaWorld_t &w, bool bursts = false) +{ + if (named != TAG_AUTO) { + const int dma = streamOf(named); + return (named != TAG_NONE && dma >= 0 && !listed(w.unavailable, 4, dma)) ? named : TAG_NONE; + } + if ((rxCallback && !(bursts && !halfDuplex)) || halfDuplex) { + return TAG_NONE; + } + for (int i = 0; i < 2; i++) { + const uint32_t t = candidates[device][i]; + const int dma = streamOf(t); + if (t != TAG_NONE && dma >= 0 && dma != otherDirection && !listed(w.usedElsewhere, 2, dma) && !listed(w.named, 2, dma) && + !listed(w.unavailable, 4, dma)) { + return t; + } + } + return TAG_NONE; +} + +const dmaWorld_t freeWorld = { { -1, -1, -1, -1 }, { -1, -1 }, { -1, -1 } }; + +int s(int dma, int stream) +{ + return streamOf(tag(dma, stream, 0)); +} + +} // namespace + +TEST(UartDmaPick, NamedStreamIsUsedOnlyWhenFree) +{ + EXPECT_EQ(tag(1, 5, 4), pick(tag(1, 5, 4), rxCandidates, 1, false, false, -1, freeWorld)); + const dmaWorld_t taken = { { s(1, 5), -1, -1, -1 }, { -1, -1 }, { -1, -1 } }; + EXPECT_EQ(TAG_NONE, pick(tag(1, 5, 4), rxCandidates, 1, false, false, -1, taken)); + EXPECT_EQ(TAG_NONE, pick(TAG_NONE, rxCandidates, 1, false, false, -1, freeWorld)); +} + +TEST(UartDmaPick, NamedStreamAlsoServesReceiversAndHalfDuplex) +{ + EXPECT_EQ(tag(1, 6, 4), pick(tag(1, 6, 4), txCandidates, 1, true, true, -1, freeWorld)); +} + +TEST(UartDmaPick, AutoLeavesReceiversAndHalfDuplexOnTheInterrupt) +{ + EXPECT_EQ(TAG_NONE, pick(TAG_AUTO, rxCandidates, 0, true, false, -1, freeWorld)); + EXPECT_EQ(TAG_NONE, pick(TAG_AUTO, txCandidates, 0, false, true, -1, freeWorld)); +} + +TEST(UartDmaPick, AutoGivesAReceiverTakingBurstsAStream) +{ + EXPECT_EQ(tag(2, 2, 4), pick(TAG_AUTO, rxCandidates, 0, true, false, -1, freeWorld, true)); + EXPECT_EQ(tag(2, 7, 4), pick(TAG_AUTO, txCandidates, 0, true, false, -1, freeWorld, true)); +} + +TEST(UartDmaPick, AutoLeavesAHalfDuplexReceiverOnTheInterruptEvenTakingBursts) +{ + EXPECT_EQ(TAG_NONE, pick(TAG_AUTO, rxCandidates, 0, true, true, -1, freeWorld, true)); +} + +TEST(UartDmaPick, AutoTakesTheFirstFreeCandidate) +{ + EXPECT_EQ(tag(2, 2, 4), pick(TAG_AUTO, rxCandidates, 0, false, false, -1, freeWorld)); + const dmaWorld_t firstTaken = { { s(2, 2), -1, -1, -1 }, { -1, -1 }, { -1, -1 } }; + EXPECT_EQ(tag(2, 5, 4), pick(TAG_AUTO, rxCandidates, 0, false, false, -1, firstTaken)); + const dmaWorld_t bothTaken = { { s(2, 2), s(2, 5), -1, -1 }, { -1, -1 }, { -1, -1 } }; + EXPECT_EQ(TAG_NONE, pick(TAG_AUTO, rxCandidates, 0, false, false, -1, bothTaken)); +} + +TEST(UartDmaPick, AutoNeverTakesAnEmptySecondCandidate) +{ + const dmaWorld_t taken = { { s(1, 5), -1, -1, -1 }, { -1, -1 }, { -1, -1 } }; + EXPECT_EQ(TAG_NONE, pick(TAG_AUTO, rxCandidates, 1, false, false, -1, taken)); +} + +TEST(UartDmaPick, AutoNeverTakesAStreamAnotherPortNames) +{ + // UART6 names DMA2 stream 2 for its receiver: UART1 moves on to stream 5 even when it opens first + const dmaWorld_t named = { { -1, -1, -1, -1 }, { -1, -1 }, { s(2, 2), -1 } }; + EXPECT_EQ(tag(2, 5, 4), pick(TAG_AUTO, rxCandidates, 0, false, false, -1, named)); +} + +TEST(UartDmaPick, AutoSkipsTheOtherDirectionAndTheSdCard) +{ + EXPECT_EQ(tag(2, 7, 5), pick(TAG_AUTO, txCandidates, 5, false, false, s(2, 6), freeWorld)); + const dmaWorld_t sdcard = { { -1, -1, -1, -1 }, { s(2, 6), -1 }, { -1, -1 } }; + EXPECT_EQ(tag(2, 7, 5), pick(TAG_AUTO, txCandidates, 5, false, false, -1, sdcard)); +} + +namespace { + +const char *expectedPick = R"( +static inline dmaTag_t uartDmaPick(dmaTag_t named, const dmaTag_t (*candidates)[UART_DMA_CANDIDATES], UARTDevice_e device, + const serialPort_t *port, DMA_t otherDirection) +{ + if (named != DMA_TAG_AUTO) { + const DMA_t dma = dmaGetByTag(named); + return (named != DMA_NONE && dma && uartDmaStreamAvailable(dma)) ? named : DMA_NONE; + } + if ((port->rxCallback && !uartRxTakesBursts(port)) || (port->options & SERIAL_BIDIR)) { + return DMA_NONE; + } + for (int i = 0; i < UART_DMA_CANDIDATES; i++) { + const dmaTag_t tag = uartDmaCandidate(candidates, device, i); + const DMA_t dma = dmaGetByTag(tag); + if (tag != DMA_NONE && dma && dma != otherDirection && !uartDmaStreamUsedElsewhere(dma) && !uartDmaStreamNamed(dma) && + uartDmaStreamAvailable(dma)) { + return tag; + } + } + UNUSED(candidates); + UNUSED(device); + return DMA_NONE; +} +)"; + +const char *expectedCandidates = R"( +static const dmaTag_t uartRxDmaCandidates[UARTDEV_MAX][UART_DMA_CANDIDATES] = { + [UARTDEV_1] = { DMA_TAG(2, 2, 4), DMA_TAG(2, 5, 4) }, + [UARTDEV_2] = { DMA_TAG(1, 5, 4) }, + [UARTDEV_3] = { DMA_TAG(1, 1, 4) }, + [UARTDEV_4] = { DMA_TAG(1, 2, 4) }, + [UARTDEV_5] = { DMA_TAG(1, 0, 4) }, + [UARTDEV_6] = { DMA_TAG(2, 1, 5), DMA_TAG(2, 2, 5) }, + [UARTDEV_7] = { DMA_TAG(1, 3, 5) }, + [UARTDEV_8] = { DMA_TAG(1, 6, 5) }, +}; +static const dmaTag_t uartTxDmaCandidates[UARTDEV_MAX][UART_DMA_CANDIDATES] = { + [UARTDEV_1] = { DMA_TAG(2, 7, 4) }, + [UARTDEV_2] = { DMA_TAG(1, 6, 4) }, + [UARTDEV_3] = { DMA_TAG(1, 3, 4), DMA_TAG(1, 4, 7) }, + [UARTDEV_4] = { DMA_TAG(1, 4, 4) }, + [UARTDEV_5] = { DMA_TAG(1, 7, 4) }, + [UARTDEV_6] = { DMA_TAG(2, 6, 5), DMA_TAG(2, 7, 5) }, + [UARTDEV_7] = { DMA_TAG(1, 1, 5) }, + [UARTDEV_8] = { DMA_TAG(1, 0, 5) }, +}; +)"; + +const char *expectedTakesBursts = R"( +#define uartRxTakesBursts(port) (((port)->options & (SERIAL_RX_BURSTS | SERIAL_BIDIR)) == SERIAL_RX_BURSTS) +)"; + +// Not mirrored: checked so that a change to them shows up here +const char *expectedStreamList = R"( +static const dmaTag_t uartDmaStreams[UART_DMA_CANDIDATES] = { + DMA_TAG(1, 0, 0), DMA_TAG(1, 1, 0), DMA_TAG(1, 2, 0), DMA_TAG(1, 3, 0), DMA_TAG(1, 4, 0), DMA_TAG(1, 5, 0), + DMA_TAG(1, 6, 0), DMA_TAG(1, 7, 0), DMA_TAG(2, 3, 0), DMA_TAG(2, 4, 0), DMA_TAG(2, 5, 0), DMA_TAG(2, 6, 0), + DMA_TAG(2, 7, 0), +}; +)"; + +const char *expectedNamed = R"( +static inline bool uartDmaStreamNamed(DMA_t dma) +{ + for (unsigned i = 0; i < ARRAYLEN(uartNamedDmaTags); i++) { + if (uartNamedDmaTags[i] != DMA_NONE && dma == dmaGetByTag(uartNamedDmaTags[i])) { + return true; + } + } + return false; +} +)"; + +} // namespace + +TEST(UartDmaStreamSourceSync, H7StreamListMatches) +{ + EXPECT_TRUE(liveSourceContains("serial_uart_impl.h", expectedStreamList)); +} + +TEST(UartDmaStreamSourceSync, NamedMatches) +{ + EXPECT_TRUE(liveSourceContains("serial_uart_impl.h", expectedNamed)); +} + +// Without it a closed or reopened port would keep a stream the pick above counts as taken +TEST(UartDmaStreamSourceSync, EveryStopGivesItsStreamBack) +{ + for (const char *file : { "serial_uart_stm32f4xx.c", "serial_uart_stm32f7xx.c", "serial_uart_stm32h7xx.c", "serial_uart_at32f43x.c" }) { + EXPECT_TRUE(liveSourceContains(file, "uartDmaRelease(s->rxDma); s->rxDma = NULL;")); + EXPECT_TRUE(liveSourceContains(file, "uartDmaRelease(s->txDma); s->txDma = NULL;")); + } +} + +TEST(UartDmaStreamSourceSync, PickMatchesMirror) +{ + EXPECT_TRUE(liveSourceContains("serial_uart_impl.h", expectedPick)); + EXPECT_TRUE(liveSourceContains("serial_uart_impl.h", expectedTakesBursts)); +} + +TEST(UartDmaStreamSourceSync, CandidateTablesMatchMirror) +{ + EXPECT_TRUE(liveSourceContains("serial_uart_impl.h", expectedCandidates)); +} + +namespace { + +const char *expectedUsedElsewhere = R"( +static inline bool uartDmaStreamUsedElsewhere(DMA_t dma) +{ +#if (defined(STM32F4) || defined(STM32F7)) && defined(USE_SDCARD_SDIO) +#ifdef SDCARD_SDIO_DMA + if (dma == dmaGetByTag(SDCARD_SDIO_DMA)) { + return true; + } +#else + if (dma == dmaGetByTag(DMA_TAG(2, 3, 0)) || dma == dmaGetByTag(DMA_TAG(2, 6, 0))) { + return true; + } +#endif +#endif +#if defined(AT32F43x) +#ifdef ADC1_DMA_STREAM + if (dma == dmaGetByRef(ADC1_DMA_STREAM)) { +#else + if (dma == dmaGetByRef(DMA2_CHANNEL1)) { +#endif + return true; + } +#endif + UNUSED(dma); + return false; +} +)"; + +const char *expectedAt32StreamList = R"( +static const dmaTag_t uartDmaStreams[UART_DMA_CANDIDATES] = { + DMA_TAG(1, 1, 0), DMA_TAG(1, 2, 0), DMA_TAG(1, 3, 0), DMA_TAG(1, 4, 0), DMA_TAG(1, 5, 0), DMA_TAG(1, 6, 0), + DMA_TAG(1, 7, 0), DMA_TAG(2, 1, 0), DMA_TAG(2, 2, 0), DMA_TAG(2, 3, 0), DMA_TAG(2, 4, 0), DMA_TAG(2, 5, 0), + DMA_TAG(2, 6, 0), DMA_TAG(2, 7, 0), +}; +)"; + +// Built rather than pasted, so a UART or a direction left out shows up +std::string streamName(int uart, const char *direction) +{ + return "UART" + std::to_string(uart) + "_" + direction + "_DMA"; +} + +std::string expectedNamedList() +{ + std::string text = "static const dmaTag_t uartNamedDmaTags[] = {\n"; + for (int uart = 1; uart <= 8; uart++) { + for (const char *direction : { "RX", "TX" }) { + const std::string name = streamName(uart, direction); + text += "#if defined(" + name + ") && (" + name + " != DMA_TAG_AUTO) && (" + name + " != DMA_NONE)\n " + name + ",\n#endif\n"; + } + } + return text + " DMA_NONE,\n};\n"; // keeps the array from being empty +} + +std::string expectedAutoDefaults() +{ + std::string text = "#if defined(STM32F4) || defined(STM32F7) || defined(STM32H7) || defined(AT32F43x)\n"; + for (int uart = 1; uart <= 8; uart++) { + for (const char *direction : { "RX", "TX" }) { + const std::string name = streamName(uart, direction); + text += "#if defined(USE_UART" + std::to_string(uart) + ") && !defined(" + name + ")\n#define " + name + " DMA_TAG_AUTO\n#endif\n"; + } + } + return text + "#endif\n"; +} + +// The first uartDmaPick() call after the anchor must be the expected one: catches a direction given the other's table +::testing::AssertionResult livePickAfter(const char *relativePath, const char *anchor, const char *call) +{ + std::string thisFile = __FILE__; + const size_t slash = thisFile.find_last_of("/\\"); + const std::string path = (slash == std::string::npos ? std::string(".") : thisFile.substr(0, slash)) + + "/../../main/drivers/" + relativePath; + std::ifstream file(path); + if (!file.is_open()) { + return ::testing::AssertionFailure() << "cannot open " << path; + } + std::ostringstream buffer; + buffer << file.rdbuf(); + const std::string text = normalize(buffer.str()); + const size_t at = text.find(normalize(anchor)); + if (at == std::string::npos) { + return ::testing::AssertionFailure() << relativePath << ": no " << anchor; + } + const size_t next = text.find("uartDmaPick(", at); + const std::string expected = normalize(call); + if (next == std::string::npos || text.compare(next, expected.size(), expected) != 0) { + return ::testing::AssertionFailure() << relativePath << ": " << anchor << " no longer calls " << call; + } + return ::testing::AssertionSuccess(); +} + +} // namespace + +TEST(UartDmaStreamSourceSync, UsedElsewhereMatches) +{ + EXPECT_TRUE(liveSourceContains("serial_uart_impl.h", expectedUsedElsewhere)); +} + +TEST(UartDmaStreamSourceSync, At32StreamListMatches) +{ + EXPECT_TRUE(liveSourceContains("serial_uart_impl.h", expectedAt32StreamList)); +} + +TEST(UartDmaStreamSourceSync, NamedListHasEveryPortAndDirection) +{ + EXPECT_TRUE(liveSourceContains("serial_uart_impl.h", expectedNamedList().c_str())); +} + +TEST(UartDmaStreamSourceSync, EveryPortAndDirectionDefaultsToAuto) +{ + EXPECT_TRUE(liveSourceContains("../target/common_post.h", expectedAutoDefaults().c_str())); +} + +TEST(UartDmaStreamSourceSync, EachDirectionPicksFromItsOwnTable) +{ + const char *rxTables = "uartDmaPick(uartRxDmaTag[device], uartRxDmaCandidates, device, &s->port, uartTxDmaOf(s))"; + const char *txTables = "uartDmaPick(uartTxDmaTag[device], uartTxDmaCandidates, device, &s->port, uartRxDmaOf(s))"; + const char *rxConfig = "uartDmaPick(uartRxDmaConfig[device].tag, NULL, device, &s->port, uartTxDmaOf(s))"; + const char *txConfig = "uartDmaPick(uartTxDmaConfig[device].tag, NULL, device, &s->port, uartRxDmaOf(s))"; + for (const char *driver : { "serial_uart_stm32f4xx.c", "serial_uart_stm32f7xx.c" }) { + EXPECT_TRUE(livePickAfter(driver, "bool uartRxDmaStart(uartPort_t *s)", rxTables)); + EXPECT_TRUE(livePickAfter(driver, "bool uartTxDmaStart(uartPort_t *s)", txTables)); + } + for (const char *driver : { "serial_uart_stm32h7xx.c", "serial_uart_at32f43x.c" }) { + EXPECT_TRUE(livePickAfter(driver, "bool uartRxDmaStart(uartPort_t *s)", rxConfig)); + EXPECT_TRUE(livePickAfter(driver, "bool uartTxDmaStart(uartPort_t *s)", txConfig)); + } +} + +/* + * uartCharNs() and uartRxDmaDeliver() (serial_uart_impl.h): which bytes a burst hands over, and the time each is given. + * Mirrored on a plain ring; the source check below keeps the mirror honest. + */ +namespace { + +uint32_t charNs(uint32_t baudRate, bool parityEven, bool stopBits2) +{ + const uint32_t bits = 10 + (parityEven ? 1 : 0) + (stopBits2 ? 1 : 0); + return bits * (1000000000U / baudRate); +} + +struct handed_t { + uint8_t byte; + uint32_t timeUs; +}; + +// Mirror of uartRxDmaDeliver(): appends what the callback would get to out, returns the new tail +uint32_t deliver(const uint8_t *ring, uint32_t size, uint32_t tail, uint32_t head, uint32_t lastEndUs, uint32_t cns, + handed_t *out, uint32_t *count) +{ + for (uint32_t left = (head + size - tail) % size; left > 0; left--) { + out[*count].timeUs = lastEndUs - ((left - 1) * cns) / 1000; + out[*count].byte = ring[tail]; + (*count)++; + tail = (tail + 1) % size; + } + return tail; +} + +const char *expectedCharNs = R"( +static inline uint32_t uartCharNs(const uartPort_t *s) +{ + const uint32_t bits = 10 + ((s->port.options & SERIAL_PARITY_EVEN) ? 1 : 0) + ((s->port.options & SERIAL_STOPBITS_2) ? 1 : 0); + return bits * (1000000000U / s->port.baudRate); +} +)"; + +const char *expectedDeliver = R"( + const uint32_t size = s->port.rxBufferSize; + const uint32_t charNs = uartCharNs(s); + uint32_t tail = s->port.rxBufferTail; + + for (uint32_t left = (uartRxBufferHead(s) + size - tail) % size; left > 0; left--) { + s->port.rxByteTimeUs = lastEndUs - ((left - 1) * charNs) / 1000; + s->port.rxCallback(s->port.rxBuffer[tail], s->port.rxCallbackData); + tail = (tail + 1) % size; + } + s->port.rxBufferTail = tail; +)"; + +} // namespace + +TEST(UartRxDmaDeliver, CharacterTimesOfCrsfAndSbus) +{ + EXPECT_EQ(23800U, charNs(420000, false, false)); // 8N1, 23.8 us + EXPECT_EQ(120000U, charNs(100000, true, true)); // 8E2, 120 us +} + +TEST(UartRxDmaDeliver, BurstIsTimedBackFromItsEnd) +{ + uint8_t ring[256]; + for (int i = 0; i < 256; i++) { + ring[i] = (uint8_t)i; + } + handed_t out[256]; + uint32_t count = 0; + + // A 26 byte CRSF frame at 420000 baud, its last byte ending at 10000 us + EXPECT_EQ(26U, deliver(ring, 256, 0, 26, 10000, charNs(420000, false, false), out, &count)); + ASSERT_EQ(26U, count); + EXPECT_EQ(0, out[0].byte); + EXPECT_EQ(10000U - 595U, out[0].timeUs); // 25 characters before the last + EXPECT_EQ(10000U - 23U, out[24].timeUs); + EXPECT_EQ(10000U, out[25].timeUs); +} + +TEST(UartRxDmaDeliver, BurstWrapsRoundTheRing) +{ + uint8_t ring[256]; + for (int i = 0; i < 256; i++) { + ring[i] = (uint8_t)i; + } + handed_t out[256]; + uint32_t count = 0; + + EXPECT_EQ(4U, deliver(ring, 256, 250, 4, 5000, charNs(100000, true, true), out, &count)); + ASSERT_EQ(10U, count); + EXPECT_EQ(250, out[0].byte); + EXPECT_EQ(255, out[5].byte); + EXPECT_EQ(0, out[6].byte); + EXPECT_EQ(3, out[9].byte); + EXPECT_EQ(5000U - 9U * 120U, out[0].timeUs); +} + +TEST(UartRxDmaDeliver, NothingNewHandsNothing) +{ + uint8_t ring[256] = { 0 }; + handed_t out[1]; + uint32_t count = 0; + + EXPECT_EQ(17U, deliver(ring, 256, 17, 17, 5000, charNs(420000, false, false), out, &count)); + EXPECT_EQ(0U, count); +} + +TEST(UartDmaStreamSourceSync, DeliverMatchesMirror) +{ + EXPECT_TRUE(liveSourceContains("serial_uart_impl.h", expectedCharNs)); + EXPECT_TRUE(liveSourceContains("serial_uart_impl.h", expectedDeliver)); +}