Commit 7b496c84 authored by Jens Korinth's avatar Jens Korinth
Browse files

WIP: implement DMA engine support in TLKM

* independent of platform, using status core base addresses
* generic interrupt mechanism
* not implemented yet: DMA engines do not request interrupts
parent 02511715
......@@ -44,7 +44,7 @@ namespace eval addressmap {
}
proc get_known_platform_components {} {
set f [open "$::env(TAPASCO_HOME)/platform/common/include/platform_types.h" "r"]
set f [open "$::env(TAPASCO_HOME)/platform/common/include/platform_components.h" "r"]
set fl [split [read $f] "\n"]
foreach line $fl {
if {[regexp {.*(PLATFORM_COMPONENT_[^\s,]*)} $line _ name]} {
......
#ifndef PLATFORM_COMPONENTS_H__
#define PLATFORM_COMPONENTS_H__
/**
* Platform component identifiers.
* NOTE: This will be parsed by a simple regex in Tcl, which uses the order of
* appearance to determine the value of the constant; make sure not to change
* the values by assigning explicitly, or start at something other than 0.
**/
typedef enum {
/** TaPaSCo Status Core: bitstream information. **/
PLATFORM_COMPONENT_STATUS = 0,
/** ATS/PRI checker. **/
PLATFORM_COMPONENT_ATSPRI,
/** Interrupt controller #0. **/
PLATFORM_COMPONENT_INTC0,
/** Interrupt controller #1. **/
PLATFORM_COMPONENT_INTC1,
/** Interrupt controller #2. **/
PLATFORM_COMPONENT_INTC2,
/** Interrupt controller #3. **/
PLATFORM_COMPONENT_INTC3,
/** MSI-X Interrupt controller #0. **/
PLATFORM_COMPONENT_MSIX0,
/** MSI-X Interrupt controller #1. **/
PLATFORM_COMPONENT_MSIX1,
/** MSI-X Interrupt controller #2. **/
PLATFORM_COMPONENT_MSIX2,
/** MSI-X Interrupt controller #3. **/
PLATFORM_COMPONENT_MSIX3,
/** DMA engine #0. **/
PLATFORM_COMPONENT_DMA0,
/** DMA engine #1. **/
PLATFORM_COMPONENT_DMA1,
/** DMA engine #2. **/
PLATFORM_COMPONENT_DMA2,
/** DMA engine #3. **/
PLATFORM_COMPONENT_DMA3,
} platform_component_t;
#endif /* PLATFORM_COMPONENTS_H__ */
......@@ -23,6 +23,7 @@
#include <stdlib.h>
#include <tlkm_access.h>
#include <tlkm_ioctl_cmds.h>
#include "platform_components.h"
#define PE_LOCAL_FLAG 2
......@@ -75,43 +76,6 @@ typedef enum {
PLATFORM_MONITOR_ACCESS = TLKM_ACCESS_MONITOR,
} platform_access_t;
/**
* Platform component identifiers.
* NOTE: This will be parsed by a simple regex in Tcl, which uses the order of
* appearance to determine the value of the constant; make sure not to change
* the values by assigning explicitly, or start at something other than 0.
**/
typedef enum {
/** TaPaSCo Status Core: bitstream information. **/
PLATFORM_COMPONENT_STATUS = 0,
/** ATS/PRI checker. **/
PLATFORM_COMPONENT_ATSPRI,
/** Interrupt controller #0. **/
PLATFORM_COMPONENT_INTC0,
/** Interrupt controller #1. **/
PLATFORM_COMPONENT_INTC1,
/** Interrupt controller #2. **/
PLATFORM_COMPONENT_INTC2,
/** Interrupt controller #3. **/
PLATFORM_COMPONENT_INTC3,
/** MSI-X Interrupt controller #0. **/
PLATFORM_COMPONENT_MSIX0,
/** MSI-X Interrupt controller #1. **/
PLATFORM_COMPONENT_MSIX1,
/** MSI-X Interrupt controller #2. **/
PLATFORM_COMPONENT_MSIX2,
/** MSI-X Interrupt controller #3. **/
PLATFORM_COMPONENT_MSIX3,
/** DMA engine #0. **/
PLATFORM_COMPONENT_DMA0,
/** DMA engine #1. **/
PLATFORM_COMPONENT_DMA1,
/** DMA engine #2. **/
PLATFORM_COMPONENT_DMA2,
/** DMA engine #3. **/
PLATFORM_COMPONENT_DMA3,
} platform_component_t;
typedef enum {
/** no flags **/
PLATFORM_ALLOC_FLAGS_NONE = 0,
......
......@@ -29,7 +29,10 @@ tlkm-objs := \
zynq/zynq_mmap.o \
pcie/pcie.o \
pcie/pcie_device.o \
pcie/pcie_irq.o
pcie/pcie_irq.o \
dma/tlkm_dma.o \
dma/dual_dma.o \
dma/blue_dma.o
CPPFLAGS+=-I$(TAPASCO_HOME)/tlkm -I$(TAPASCO_HOME)/tlkm/device -I$(TAPASCO_HOME)/tlkm/tlkm -I$(TAPASCO_HOME)/tlkm/user -I$(TAPASCO_HOME)/platform/include
......
......@@ -43,7 +43,9 @@
_PC(total_ctl_writes) \
_PC(total_ctl_reads) \
_PC(link_width) \
_PC(link_speed)
_PC(link_speed) \
_PC(dma_reads) \
_PC(dma_writes)
#ifndef NPERFC
#include <linux/types.h>
......
//
// Copyright (C) 2017 Jaco A. Hofmann, TU Darmstadt
//
// This file is part of Tapasco (TPC).
//
// Tapasco is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Tapasco is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with Tapasco. If not, see <http://www.gnu.org/licenses/>.
//
#include <linux/sched.h>
#include "tlkm_dma.h"
#include "tlkm_logging.h"
/* Register Map and commands */
#define REG_HOST_ADDR 0x00 /* slv_reg0 = PCIe addr */
#define REG_FPGA_ADDR 0x08 /* slv_reg1 = FPGA addr */
#define REG_BTT 0x10 /* slv_reg2 = bytes to transfer */
#define REG_CMD 0x20 /* slv_reg3 = CMD */
#define CMD_READ 0x10001000 /* from m64 fpga memory to m64 host memory */
#define CMD_WRITE 0x10000001 /* from m64 host memory to m64 fpga memory */
irqreturn_t blue_dma_intr_handler_read(int irq, void * dev_id)
{
struct dma_engine *dma = (struct dma_engine *)dev_id;
BUG_ON(dma->irq_no != irq);
atomic64_inc(&dma->rq_processed);
wake_up_interruptible_sync(&dma->rq);
return IRQ_HANDLED;
}
irqreturn_t blue_dma_intr_handler_write(int irq, void * dev_id)
{
struct dma_engine *dma = (struct dma_engine *)dev_id;
BUG_ON(dma->irq_no != irq);
atomic64_inc(&dma->wq_processed);
wake_up_interruptible_sync(&dma->wq);
return IRQ_HANDLED;
}
ssize_t blue_dma_copy_from(struct dma_engine *dma, void __user *usr_addr, dev_addr_t dev_addr, size_t len)
{
LOG(TLKM_LF_DMA, "dev_addr = 0x%08llx, usr_addr = 0x%08llx, len: %zu bytes", (u64)dev_addr, (u64)usr_addr, len);
if(mutex_lock_interruptible(&dma->regs_mutex)) {
WRN("got killed while aquiring the mutex");
return len;
}
*(u64 *)(dma->regs + REG_FPGA_ADDR) = dev_addr;
*(u64 *)(dma->regs + REG_HOST_ADDR) = (u64)usr_addr;
*(u64 *)(dma->regs + REG_BTT) = len;
wmb();
*(u64 *)(dma->regs + REG_CMD) = CMD_READ;
mutex_unlock(&dma->regs_mutex);
return atomic64_read(&dma->rq_processed) + 1;
}
ssize_t blue_dma_copy_to(struct dma_engine *dma, dev_addr_t dev_addr, const void __user *usr_addr, size_t len)
{
LOG(TLKM_LF_DMA, "dev_addr = 0x%08llx, usr_addr = 0x%08llx, len: %zu bytes", (u64)dev_addr, (u64)usr_addr, len);
if(mutex_lock_interruptible(&dma->regs_mutex)) {
WRN("got killed while aquiring the mutex");
return len;
}
*(u64 *)(dma->regs + REG_FPGA_ADDR) = dev_addr;
*(u64 *)(dma->regs + REG_HOST_ADDR) = (u64)usr_addr;
*(u64 *)(dma->regs + REG_BTT) = len;
wmb();
*(u64 *)(dma->regs + REG_CMD) = CMD_WRITE;
mutex_unlock(&dma->regs_mutex);
return atomic64_read(&dma->wq_processed) + 1;
}
//
// Copyright (C) 2017 Jaco A. Hofmann, TU Darmstadt
//
// This file is part of Tapasco (TPC).
//
// Tapasco is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Tapasco is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with Tapasco. If not, see <http://www.gnu.org/licenses/>.
//
#ifndef BLUE_DMA_H__
#define BLUE_DMA_H__
#include <linux/interrupt.h>
#include "tlkm_dma.h"
#include "tlkm_types.h"
irqreturn_t blue_dma_intr_handler_read(int irq, void * dev_id);
irqreturn_t blue_dma_intr_handler_write(int irq, void * dev_id);
ssize_t blue_dma_copy_from(struct dma_engine *dma, void *krn_addr, dev_addr_t dev_addr, size_t len);
ssize_t blue_dma_copy_to(struct dma_engine *dma, dev_addr_t dev_addr, const void *krn_addr, size_t len);
#endif /* BLUE_DMA_H__ */
//
// Copyright (C) 2014 David de la Chevallerie, TU Darmstadt
//
// This file is part of Tapasco (TPC).
//
// Tapasco is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Tapasco is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with Tapasco. If not, see <http://www.gnu.org/licenses/>.
//
/**
* @file char_device_dma.h
* @brief Composition of everything needed to handle char-device calls for dma transfers
here all definitions of functions and structs are given, which are used by the char-device
the user can adapt the method used for transfers (bounce-/double-buffering)
in addition the size of the internal buffers can be choosen and the upper bound, when double buffering will be used
the base addresses of the dma engine must match the physical address map of the pci-express design in vivado
do not confuse these addresses with the address of the pcie_core given by bios
* */
#ifndef __CHAR_DEVICE_DMA_H
#define __CHAR_DEVICE_DMA_H
/******************************************************************************/
/* Include section */
/* Includes from linux headers */
#include <linux/module.h>
#include <linux/version.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/kdev_t.h>
#include <linux/fs.h>
#include <linux/device.h>
#include <linux/cdev.h>
#include <linux/mm.h>
//#include <linux/pagemap.h>
//#include <linux/gfp.h>
//#include <linux/dma-mapping.h>
//#include <linux/delay.h>
#include <linux/slab.h>
//#include <linux/clk.h>
#include <asm/io.h>
#include <asm/atomic.h>
#include <linux/aio.h>
#include <linux/uio.h>
#include <linux/highmem.h>
#include <linux/interrupt.h>
#include <linux/mutex.h>
#include <linux/semaphore.h>
#include <linux/spinlock.h>
#include <linux/errno.h>
//#include <linux/kernel.h>
//#include <asm/system.h>
#include <asm/uaccess.h>
#include <linux/delay.h>
#include <linux/pci.h>
#include <linux/sched.h>
/* Includes from local files */
#include "common/debug_print.h"
#include "common/device_dma.h"
#include "common/device_pcie.h"
#include "common/dma_ctrl.h"
#include "include/dma_ioctl_calls.h"
/******************************************************************************/
/* physical address of dma core in register map from vivado */
#define DMA_BASE_ADDR_0 0x300000
#define AXI_CTRL_BASE_ADDR 0x100000
/* BRAM standard address */
#define RAM_BASE_ADDR_0 0x80000000
/* to change buffer_size increase or decrease the order of pages */
#define BUFFER_ORDER (MAX_ORDER -1)
#define BUFFER_SIZE (PAGE_SIZE * (1 << BUFFER_ORDER))
/* device name to register fops with
* fops will append a number to name for multiple minor nodes */
#define FFLINK_DMA_NAME "FFLINK_DMA_DEVICE"
/******************************************************************************/
/* struct array to hold data over multiple fops-calls */
struct priv_data_struct {
void * kvirt_h2l;
void * kvirt_l2h;
dma_addr_t dma_handle_h2l;
dma_addr_t dma_handle_l2h;
void * mem_addr_l2h;
void * mem_addr_h2l;
void * device_base_addr;
void * ctrl_base_addr;
wait_queue_head_t read_wait_queue;
atomic64_t reads_processed;
struct mutex read_mutex;
wait_queue_head_t write_wait_queue;
atomic64_t writes_processed;
struct mutex write_mutex;
unsigned int cache_lsize;
unsigned int cache_mask;
};
/******************************************************************************/
/* functions for user-space interaction */
static int dma_open(struct inode *, struct file *);
static int dma_close(struct inode *, struct file *);
static long dma_ioctl(struct file *, unsigned int, unsigned long);
static ssize_t dma_read(struct file *, char __user *, size_t count, loff_t *);
static ssize_t dma_write(struct file *, const char __user *, size_t count, loff_t *);
/******************************************************************************/
/* helper functions called for sys-calls */
static unsigned int dma_cache_fit(unsigned int btt);
static int dma_alloc_pbufs(void** p, dma_addr_t *handle, gfp_t zone, int direction);
static void dma_free_pbufs(void *p, dma_addr_t handle, int direction);
static void transmit_to_user(void *, void *, dma_addr_t, int);
static void transmit_from_user(void *, void *, dma_addr_t, int);
/******************************************************************************/
/* overload function to exchange bounce-/double-buffering */
static int read_with_bounce(int count, char __user *buf, void * mem_addr, struct priv_data_struct *p);
static inline int read_device(int count, char __user *buf, void * mem_addr, struct priv_data_struct *p);
static int write_with_bounce(int count, const char __user *buf, void * mem_addr, struct priv_data_struct *p);
static inline int write_device(int count, const char __user *buf, void * mem_addr, struct priv_data_struct *p);
/******************************************************************************/
#endif // __CHAR_DEVICE_DMA_H
//
// Copyright (C) 2014 David de la Chevallerie, TU Darmstadt
//
// This file is part of Tapasco (TPC).
//
// Tapasco is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Tapasco is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with Tapasco. If not, see <http://www.gnu.org/licenses/>.
//
#include <linux/sched.h>
#include "tlkm_logging.h"
#include "dual_dma.h"
/* Register Map and commands */
#define REG_HOST_ADDR_LOW 0x00 /* slv_reg0 = PCIe addr (under) */
#define REG_HOST_ADDR_HIGH 0x04 /* slv_reg1 = PCIe addr (upper) */
#define REG_FPGA_ADDR_LOW 0x08 /* slv_reg2 = FPGA addr */
#define REG_BTT 0x10 /* slv_reg4 = bytes to transfer */
#define REG_ID 0x14 /* slv_reg5 = ID */
#define REG_CMD 0x18 /* slv_reg6 = CMD */
#define REG_STATUS 0x20 /* slv_reg8 = return status */
#define CMD_READ 0x10001000 /* from m32 fpga memory to m64 host memory */
#define CMD_WRITE 0x10000001 /* from m64 host memory to m32 fpga memory */
#define CMD_ACK 0x10011001 /* acknowledge data transfer to toggle interrupt */
irqreturn_t dual_dma_intr_handler_dma(int irq, void * dev_id)
{
struct dma_engine *dma = (struct dma_engine *)dev_id;
BUG_ON(dma->irq_no != irq);
*(u32 *)(dma->regs + REG_CMD) = CMD_ACK;
mutex_unlock(&dma->regs_mutex);
atomic64_inc(&dma->wq_processed);
wake_up_interruptible(&dma->wq);
return IRQ_HANDLED;
}
ssize_t dual_dma_copy_from(struct dma_engine *dma, void __user *usr_addr, dev_addr_t dev_addr, size_t len)
{
u64 usr = (u64)usr_addr;
LOG(TLKM_LF_DMA, "dev_addr = 0x%08llx, usr_addr = 0x%08llx, len: %zu bytes", (u64)dev_addr, (u64)usr_addr, len);
if(mutex_lock_interruptible(&dma->regs_mutex)) {
WRN("got killed while aquiring the mutex");
return len;
}
*(u32 *)(dma->regs + REG_FPGA_ADDR_LOW) = (u32)dev_addr;
*(u32 *)(dma->regs + REG_HOST_ADDR_LOW) = (u32)usr;
*(u32 *)(dma->regs + REG_HOST_ADDR_HIGH) = (u32)(usr >> 32);
*(u32 *)(dma->regs + REG_BTT) = (u32)len;
wmb();
*(u32 *)(dma->regs + REG_CMD) = CMD_READ;
return atomic64_read(&dma->rq_processed) + 1;
}
ssize_t dual_dma_copy_to(struct dma_engine *dma, dev_addr_t dev_addr, const void __user *usr_addr, size_t len)
{
u64 usr = (u64)usr_addr;
LOG(TLKM_LF_DMA, "dev_addr = 0x%08llx, usr_addr = 0x%08llx, len: %zu bytes", (u64)dev_addr, (u64)usr_addr, len);
if(mutex_lock_interruptible(&dma->regs_mutex)) {
WRN("got killed while aquiring the mutex");
return len;
}
*(u32 *)(dma->regs + REG_HOST_ADDR_LOW) = (u32)usr;
*(u32 *)(dma->regs + REG_HOST_ADDR_HIGH) = (u32)(usr >> 32);
*(u32 *)(dma->regs + REG_FPGA_ADDR_LOW) = (u32)dev_addr;
*(u32 *)(dma->regs + REG_BTT) = (u32)len;
wmb();
*(u32 *)(dma->regs + REG_CMD) = CMD_WRITE;
return atomic64_read(&dma->wq_processed) + 1;
}
//
// Copyright (C) 2014 David de la Chevallerie, TU Darmstadt
// Copyright (C) 2018 Jens Korinth, TU Darmstadt
//
// This file is part of Tapasco (TaPaSCo).
//
// Tapasco is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Tapasco is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with Tapasco. If not, see <http://www.gnu.org/licenses/>.
//
#ifndef DUAL_DMA_H__
#define DUAL_DMA_H__
#include <linux/interrupt.h>
#include "tlkm_dma.h"
#include "tlkm_types.h"
irqreturn_t dual_dma_intr_handler_dma(int irq, void * dev_id);
ssize_t dual_dma_copy_from(struct dma_engine *dma, void *krn_addr, dev_addr_t dev_addr, size_t len);
ssize_t dual_dma_copy_to(struct dma_engine *dma, dev_addr_t dev_addr, const void *krn_addr, size_t len);
#endif /* DUAL_DMA_H__ */
#include <linux/gfp.h>
#include <linux/uaccess.h>
#include "tlkm_dma.h"
#include "tlkm_logging.h"
#include "blue_dma.h"
#include "dual_dma.h"
#define REG_ID 0x18
#define DMA_SZ 0x10000
static const struct dma_operations dma_ops[] = {
[DMA_USED_DUAL] = {
.intr_read = dual_dma_intr_handler_dma, // Dual DMA can not read and write in parallel
.intr_write = dual_dma_intr_handler_dma,
.copy_from = dual_dma_copy_from,
.copy_to = dual_dma_copy_to,
},
[DMA_USED_BLUE] = {
.intr_read = blue_dma_intr_handler_read,
.intr_write = blue_dma_intr_handler_write,
.copy_from = blue_dma_copy_from,
.copy_to = blue_dma_copy_to,
},
};
int tlkm_dma_init(struct dma_engine *dma, dev_id_t dev_id, void *base, int irq_no)
{
uint64_t id;
int i, ret = 0;
BUG_ON(! dma);
DEVLOG(dev_id, TLKM_LF_DMA, "initializing DMA engine 0x%08llx (#%d) ...", (u64)base, irq_no);
DEVLOG(dev_id, TLKM_LF_DMA, "I/O remapping 0x%08llx - 0x%08llx...", (u64)base, (u64)base + DMA_SZ - 1);
dma->regs = ioremap_nocache((resource_size_t)base, DMA_SZ);
if (IS_ERR(dma->regs)) {
DEVERR(dev_id, "failed to map 0x%08llx - 0x%08llx: %ld",
(u64)base, (u64)base + DMA_SZ - 1, PTR_ERR(dma->regs));
return PTR_ERR(dma->regs);
}
DEVLOG(dev_id, TLKM_LF_DMA, "allocating DMA buffers of %zd bytes ...", TLKM_DMA_BUF_SZ);
for (i = 0; i < TLKM_DMA_BUFS; ++i) {
dma->dma_buf[i] = kzalloc(TLKM_DMA_BUF_SZ, GFP_KERNEL | GFP_DMA);
if (IS_ERR(dma->dma_buf[i])) {
ret = PTR_ERR(dma->dma_buf[i]);
goto err_dma_bufs;
}
}
DEVLOG(dev_id, TLKM_LF_DMA, "detecting DMA engine type ...");
id = *(u64 *)(dma->regs + REG_ID);
if ((id & 0xFFFFFFFF) == 0xE5A0023) {
dma->dma_used = DMA_USED_BLUE;
DEVLOG(dev_id, TLKM_LF_DMA, "detected BlueDMA");
DEVLOG(dev_id, TLKM_LF_DMA, "PCIe beats per burst: %u", (uint8_t)(id >> 32));
DEVLOG(dev_id, TLKM_LF_DMA, "FPGA beats per burst: %u", (uint8_t)(id >> 40));
DEVLOG(dev_id, TLKM_LF_DMA, "smallest alignment: %u", (uint8_t)(id >> 48));
} else {
dma->dma_used = DMA_USED_DUAL;
DEVLOG(dev_id, TLKM_LF_DMA, "detected DualDMA");
}
dma->ops = dma_ops[dma->dma_used];
init_waitqueue_head(&dma->rq);
init_waitqueue_head(&dma->wq);
mutex_init(&dma->regs_mutex);
mutex_init(&dma->rq_mutex);
mutex_init(&dma->wq_mutex);
dma->dev_id = dev_id;
dma->base = base;
dma->irq_no = irq_no;
atomic64_set(&dma->rq_processed, 0);
atomic64_set(&dma->wq_processed, 0);
return 0;
err_dma_bufs:
for (; i >= 0; --i)
kfree(dma->dma_buf[i]);
iounmap(dma->regs);
return ret;
}
void tlkm_dma_exit(struct dma_engine *dma)
{
int i;
dev_id_t dev_id = dma->dev_id;
for (i = 0; i < TLKM_DMA_BUFS; ++i)
kfree(dma->dma_buf[i]);
iounmap(dma->regs);
memset(dma, 0, sizeof(*dma));
DEVLOG(dev_id, TLKM_LF_DMA, "deinitialized DMA engine");
}
ssize_t tlkm_dma_copy_to(struct dma_engine *dma, dev_addr_t dev_addr, const void __user *usr_addr, size_t len)
{
size_t cpy_sz = len;
ssize_t t_id;
while (len > 0) {
DEVLOG(dma->dev_id, TLKM_LF_DMA, "outstanding bytes: %zd - usr_addr = 0x%08llx, dev_addr = 0x%08llx",
len, (u64)usr_addr, (u64)dev_addr);
cpy_sz = len < TLKM_DMA_BUF_SZ ? len : TLKM_DMA_BUF_SZ;
if (copy_from_user(dma->dma_buf[0], usr_addr, cpy_sz)) {
DEVERR(dma->dev_id, "could not copy data from user");
return -EAGAIN;
} else {
t_id = dma->ops.copy_to(dma, dev_addr, dma->dma_buf[0], cpy_sz);
if (wait_event_interruptible(dma->rq, atomic64_read(&dma->wq_processed) > t_id)) {
WRN("got killed while hanging in waiting queue");