【Xilinx】基于MPSoC的OpenAMP实现(二）

前言

参考前文
【Xilinx】基于MPSoC的OpenAMP实现（一）

在上一篇文章里面主要是采用Xilinx提供的现成的openamp.dtsi和已经编译好的image_echo_test固件、echo_test应用程序进行快速测试验证，目的是证明软件环境、硬件环境都能够正常运行OpenAMP。

以此作为基线，我们进一步研究如何修改定制image_echo_test、echo_test、openamp.dtsi，以及内部工作原理。

一、修改echo_test

1、查找源代码

首先我们先找到源代码

xlx@u16:/opt/work/p202/xilinx-zcu106-2020.2$ find . -name echo_test.c
./components/yocto/layers/meta-openamp/recipes-openamp/rpmsg-examples/rpmsg-echo-test/echo_test.c

cd ./components/yocto/layers/meta-openamp/recipes-openamp/rpmsg-examples/
xlx@u16:/opt/work/p202/xilinx-zcu106-2020.2/components/yocto/layers/meta-openamp/recipes-openamp/rpmsg-examples$ tree
.
├── rpmsg-echo-test
│   ├── echo_test.c
│   ├── LICENSE
│   └── Makefile
├── rpmsg-echo-test_1.0.bb
├── rpmsg-mat-mul
│   ├── LICENSE
│   ├── Makefile
│   └── mat_mul_demo.c
├── rpmsg-mat-mul_1.0.bb
├── rpmsg-proxy-app
│   ├── LICENSE
│   ├── Makefile
│   ├── proxy_app.c
│   └── proxy_app.h
└── rpmsg-proxy-app_1.0.bb

可以看到3个测试例子都在components/yocto/layers/meta-openamp/recipes-openamp/rpmsg-examples目录下，代码结构很简单，一个makefie加上一个c文件

Makefile如下

APP = echo_test
APP_OBJS = echo_test.o

# Add any other object files to this list below


all: $(APP)

$(APP): $(APP_OBJS)
	$(CC) $(LDFLAGS) -o $@ $(APP_OBJS) $(LDLIBS)

clean:
	rm -rf $(APP) *.o

%.o: %.c
	$(CC) -c $(CFLAGS) -o $@ $<

echo_test.c

/*
 * echo_test.c
 *
 *  Created on: Oct 4, 2014
 *      Author: etsam
 */

/*
 * Test application that data integraty of inter processor
 * communication from linux userspace to a remote software
 * context. The application sends chunks of data to the
 * remote processor. The remote side echoes the data back
 * to application which then validates the data returned.
 */

#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 

struct _payload {
	unsigned long num;
	unsigned long size;
	char data[];
};

static int charfd = -1, fd = -1, err_cnt;

struct _payload *i_payload;
struct _payload *r_payload;

#define RPMSG_GET_KFIFO_SIZE 1
#define RPMSG_GET_AVAIL_DATA_SIZE 2
#define RPMSG_GET_FREE_SPACE 3

#define RPMSG_HEADER_LEN 16
#define MAX_RPMSG_BUFF_SIZE (512 - RPMSG_HEADER_LEN)
#define PAYLOAD_MIN_SIZE	1
#define PAYLOAD_MAX_SIZE	(MAX_RPMSG_BUFF_SIZE - 24)
#define NUM_PAYLOADS		(PAYLOAD_MAX_SIZE/PAYLOAD_MIN_SIZE)

#define RPMSG_BUS_SYS "/sys/bus/rpmsg"

static int rpmsg_create_ept(int rpfd, struct rpmsg_endpoint_info *eptinfo)
{
	int ret;

	ret = ioctl(rpfd, RPMSG_CREATE_EPT_IOCTL, eptinfo);
	if (ret)
		perror("Failed to create endpoint.\n");
	return ret;
}

static char *get_rpmsg_ept_dev_name(const char *rpmsg_char_name,
				    const char *ept_name,
				    char *ept_dev_name)
{
	char sys_rpmsg_ept_name_path[64];
	char svc_name[64];
	char *sys_rpmsg_path = "/sys/class/rpmsg";
	FILE *fp;
	int i;
	int ept_name_len;

	for (i = 0; i < 128; i++) {
		sprintf(sys_rpmsg_ept_name_path, "%s/%s/rpmsg%d/name",
			sys_rpmsg_path, rpmsg_char_name, i);
		printf("checking %s\n", sys_rpmsg_ept_name_path);
		if (access(sys_rpmsg_ept_name_path, F_OK) < 0)
			continue;
		fp = fopen(sys_rpmsg_ept_name_path, "r");
		if (!fp) {
			printf("failed to open %s\n", sys_rpmsg_ept_name_path);
			break;
		}
		fgets(svc_name, sizeof(svc_name), fp);
		fclose(fp);
		printf("svc_name: %s.\n",svc_name);
		ept_name_len = strlen(ept_name);
		if (ept_name_len > sizeof(svc_name))
			ept_name_len = sizeof(svc_name);
		if (!strncmp(svc_name, ept_name, ept_name_len)) {
			sprintf(ept_dev_name, "rpmsg%d", i);
			return ept_dev_name;
		}
	}

	printf("Not able to RPMsg endpoint file for %s:%s.\n",
	       rpmsg_char_name, ept_name);
	return NULL;
}

static int bind_rpmsg_chrdev(const char *rpmsg_dev_name)
{
	char fpath[256];
	char *rpmsg_chdrv = "rpmsg_chrdev";
	int fd;
	int ret;


	/* rpmsg dev overrides path */
	sprintf(fpath, "%s/devices/%s/driver_override",
		RPMSG_BUS_SYS, rpmsg_dev_name);
	fd = open(fpath, O_WRONLY);
	if (fd < 0) {
		fprintf(stderr, "Failed to open %s, %s\n",
			fpath, strerror(errno));
		return -EINVAL;
	}
	ret = write(fd, rpmsg_chdrv, strlen(rpmsg_chdrv) + 1);
	if (ret < 0) {
		fprintf(stderr, "Failed to write %s to %s, %s\n",
			rpmsg_chdrv, fpath, strerror(errno));
		return -EINVAL;
	}
	close(fd);

	/* bind the rpmsg device to rpmsg char driver */
	sprintf(fpath, "%s/drivers/%s/bind", RPMSG_BUS_SYS, rpmsg_chdrv);
	fd = open(fpath, O_WRONLY);
	if (fd < 0) {
		fprintf(stderr, "Failed to open %s, %s\n",
			fpath, strerror(errno));
		return -EINVAL;
	}
	ret = write(fd, rpmsg_dev_name, strlen(rpmsg_dev_name) + 1);
	if (ret < 0) {
		fprintf(stderr, "Failed to write %s to %s, %s\n",
			rpmsg_dev_name, fpath, strerror(errno));
		return -EINVAL;
	}
	close(fd);
	return 0;
}

static int get_rpmsg_chrdev_fd(const char *rpmsg_dev_name,
			       char *rpmsg_ctrl_name)
{
	char dpath[256];
	char fpath[256];
	char *rpmsg_ctrl_prefix = "rpmsg_ctrl";
	DIR *dir;
	struct dirent *ent;
	int fd;

	sprintf(dpath, "%s/devices/%s/rpmsg", RPMSG_BUS_SYS, rpmsg_dev_name);
	dir = opendir(dpath);
	if (dir == NULL) {
		fprintf(stderr, "Failed to open dir %s\n", dpath);
		return -EINVAL;
	}
	while ((ent = readdir(dir)) != NULL) {
		if (!strncmp(ent->d_name, rpmsg_ctrl_prefix,
			    strlen(rpmsg_ctrl_prefix))) {
			printf("Opening file %s.\n", ent->d_name);
			sprintf(fpath, "/dev/%s", ent->d_name);
			fd = open(fpath, O_RDWR | O_NONBLOCK);
			if (fd < 0) {
				fprintf(stderr,
					"Failed to open rpmsg char dev %s,%s\n",
					fpath, strerror(errno));
				return fd;
			}
			sprintf(rpmsg_ctrl_name, "%s", ent->d_name);
			return fd;
		}
	}

	fprintf(stderr, "No rpmsg char dev file is found\n");
	return -EINVAL;
}

int main(int argc, char *argv[])
{
	int ret, i, j;
	int size, bytes_rcvd, bytes_sent;
	err_cnt = 0;
	int opt;
	char *rpmsg_dev="virtio0.rpmsg-openamp-demo-channel.-1.0";
	int ntimes = 1;
	char fpath[256];
	char rpmsg_char_name[16];
	struct rpmsg_endpoint_info eptinfo;
	char ept_dev_name[16];
	char ept_dev_path[32];


	while ((opt = getopt(argc, argv, "d:n:")) != -1) {
		switch (opt) {
		case 'd':
			rpmsg_dev = optarg;
			break;
		case 'n':
			ntimes = atoi(optarg);
			break;
		default:
			printf("getopt return unsupported option: -%c\n",opt);
			break;
		}
	}
	printf("\r\n Echo test start \r\n");

	/* Load rpmsg_char driver */
	printf("\r\nMaster>probe rpmsg_char\r\n");
	ret = system("modprobe rpmsg_char");
	if (ret < 0) {
		perror("Failed to load rpmsg_char driver.\n");
		return -EINVAL;
	}

	printf("\r\n Open rpmsg dev %s! \r\n", rpmsg_dev);
	sprintf(fpath, "%s/devices/%s", RPMSG_BUS_SYS, rpmsg_dev);
	if (access(fpath, F_OK)) {
		fprintf(stderr, "Not able to access rpmsg device %s, %s\n",
			fpath, strerror(errno));
		return -EINVAL;
	}
	ret = bind_rpmsg_chrdev(rpmsg_dev);
	if (ret < 0)
		return ret;
	charfd = get_rpmsg_chrdev_fd(rpmsg_dev, rpmsg_char_name);
	if (charfd < 0)
		return charfd;

	/* Create endpoint from rpmsg char driver */
	strcpy(eptinfo.name, "rpmsg-openamp-demo-channel");
	eptinfo.src = 0;
	eptinfo.dst = 0xFFFFFFFF;
	ret = rpmsg_create_ept(charfd, &eptinfo);
	if (ret) {
		printf("failed to create RPMsg endpoint.\n");
		return -EINVAL;
	}
	if (!get_rpmsg_ept_dev_name(rpmsg_char_name, eptinfo.name,
				    ept_dev_name))
		return -EINVAL;
	sprintf(ept_dev_path, "/dev/%s", ept_dev_name);
	fd = open(ept_dev_path, O_RDWR | O_NONBLOCK);
	if (fd < 0) {
		perror("Failed to open rpmsg device.");
		close(charfd);
		return -1;
	}

	i_payload = (struct _payload *)malloc(2 * sizeof(unsigned long) + PAYLOAD_MAX_SIZE);
	r_payload = (struct _payload *)malloc(2 * sizeof(unsigned long) + PAYLOAD_MAX_SIZE);

	if (i_payload == 0 || r_payload == 0) {
		printf("ERROR: Failed to allocate memory for payload.\n");
		return -1;
	}

	for (j=0; j < ntimes; j++){
		printf("\r\n **********************************");
		printf("****\r\n");
		printf("\r\n  Echo Test Round %d \r\n", j);
		printf("\r\n **********************************");
		printf("****\r\n");
		for (i = 0, size = PAYLOAD_MIN_SIZE; i < NUM_PAYLOADS;
		i++, size++) {
			int k;

			i_payload->num = i;
			i_payload->size = size;

			/* Mark the data buffer. */
			memset(&(i_payload->data[0]), 0xA5, size);

			printf("\r\n sending payload number");
			printf(" %ld of size %d\r\n", i_payload->num,
			(2 * sizeof(unsigned long)) + size);

			bytes_sent = write(fd, i_payload,
			(2 * sizeof(unsigned long)) + size);

			if (bytes_sent <= 0) {
				printf("\r\n Error sending data");
				printf(" .. \r\n");
				break;
			}
			printf("echo test: sent : %d\n", bytes_sent);

			r_payload->num = 0;
			bytes_rcvd = read(fd, r_payload,
					(2 * sizeof(unsigned long)) + PAYLOAD_MAX_SIZE);
			while (bytes_rcvd <= 0) {
				usleep(10000);
				bytes_rcvd = read(fd, r_payload,
					(2 * sizeof(unsigned long)) + PAYLOAD_MAX_SIZE);
			}
			printf(" received payload number ");
			printf("%ld of size %d\r\n", r_payload->num, bytes_rcvd);

			/* Validate data buffer integrity. */
			for (k = 0; k < r_payload->size; k++) {

				if (r_payload->data[k] != 0xA5) {
					printf(" \r\n Data corruption");
					printf(" at index %d \r\n", k);
					err_cnt++;
					break;
				}
			}
			bytes_rcvd = read(fd, r_payload,
			(2 * sizeof(unsigned long)) + PAYLOAD_MAX_SIZE);

		}
		printf("\r\n **********************************");
		printf("****\r\n");
		printf("\r\n Echo Test Round %d Test Results: Error count = %d\r\n",
		j, err_cnt);
		printf("\r\n **********************************");
		printf("****\r\n");
	}

	free(i_payload);
	free(r_payload);

	close(fd);
	if (charfd >= 0)
		close(charfd);
	return 0;
}

二、创建自己的app

1、新建app

进入petalinux工程主目录，运行下面命令，-n后面是你项目的名称

petalinux-create -t apps -n leo-echo-test --enable

运行完成，会在project-spec/meta-user/recipes-apps创建了leo-echo-test目录

xlx@u16:/opt/work/p202/xilinx-zcu106-2020.2/project-spec/meta-user/recipes-apps/leo-echo-test$ tree
.
├── files
│   ├── leo-echo-test.c
│   └── Makefile
├── leo-echo-test.bb
└── README

1 directory, 4 files

2、 编译代码

把echo-test.c的代码全选复制到leo-echo-test.c，然后编译

2.1 用petalinux-build编译

petalinux-build -c leo-echo-test

petalinux-build -c leo-echo-test -x do_install
petalinux-build -c rootfs
petalinux-build -x package

但是这种方式需要遍历很多东西，编译非常慢，每次需要几分钟

2.2 方法二：创建sdk，用sdk编译（重点）

2.2.1 编译并安装sdk

编译sdk需要时间比较长，但是强烈建议不要用网上下载的其他编译器，可能会有未知的兼容问题。

time petalinux-build --sdk
time petalinux-package --sysroot

我安装完成后sdk的完整路径是

/opt/work/p202/xilinx-zcu106-2020.2/images/linux/sdk

库文件的搜索路径是
/opt/work/p202/xilinx-zcu106-2020.2/images/linux/sdk/sysroots/aarch64-xilinx-linux/usr/include

2.2.2 用sdk编译应用程序

使用前要先初始化sdk的环境变量
source运行environment-setup-aarch64-xilinx-linux配置

source /opt/work/p202/xilinx-zcu106-2020.2/images/linux/sdk/environment-setup-aarch64-xilinx-linux

有可能会提示下面的错误

Your environment is misconfigured, you probably need to 'unset LD_LIBRARY_PATH'
but please check why this was set in the first place and that it's safe to unset.
The SDK will not operate correctly in most cases when LD_LIBRARY_PATH is set.
For more references see:
  http://tldp.org/HOWTO/Program-Library-HOWTO/shared-libraries.html#AEN80
  http://xahlee.info/UnixResource_dir/_/ldpath.html

查看$LD_LIBRARY_PATH变量，非空的话用unset LD_LIBRARY_PATH取消掉。我是因为装有qt的原因。

#查看LD_LIBRARY_PATH
echo $LD_LIBRARY_PATH
/opt/work/Qt5.13.2/Tools/QtCreator/lib:

#删除LD_LIBRARY_PATH
unset LD_LIBRARY_PATH

然后就可以用make来编译应用程序了

xlx@u16:/opt/work/p202/xilinx-zcu106-2020.2/project-spec/meta-user/recipes-apps/leo-echo-test/files$ make
aarch64-xilinx-linux-gcc  -march=armv8-a+crc -mtune=cortex-a72.cortex-a53 --sysroot=/opt/work/p202/xilinx-zcu106-2020.2/images/linux/sdk/sysroots/aarch64-xilinx-linux  -O2 -pipe -g -feliminate-unused-debug-types    -c -o leo-echo-test.o leo-echo-test.c
aarch64-xilinx-linux-gcc  -march=armv8-a+crc -mtune=cortex-a72.cortex-a53 --sysroot=/opt/work/p202/xilinx-zcu106-2020.2/images/linux/sdk/sysroots/aarch64-xilinx-linux -o leo-echo-test leo-echo-test.o -Wl,-O1 -Wl,--hash-style=gnu -Wl,--as-needed 


#用ll或者ls -l查看，已经生成了leo-echo-test
xlx@u16:/opt/work/p202/xilinx-zcu106-2020.2/project-spec/meta-user/recipes-apps/leo-echo-test/files$ ls -l
total 100
-rwxrwxr-x 1 xlx xlx 37208 9月  19 07:11 leo-echo-test
-rw-rw-r-- 1 xlx xlx  8319 9月  18 23:01 leo-echo-test.c
-rw-rw-r-- 1 xlx xlx 44576 9月  19 07:11 leo-echo-test.o
-rw-rw-r-- 1 xlx xlx   216 9月  18 22:56 Makefile

#用readelf命令可以查看文件格式，可以看到是AArch64
xlx@u16:/opt/work/p202/xilinx-zcu106-2020.2/project-spec/meta-user/recipes-apps/leo-echo-test/files$ readelf -h leo-echo-test
ELF Header:
  Magic:   7f 45 4c 46 02 01 01 00 00 00 00 00 00 00 00 00 
  Class:                             ELF64
  Data:                              2's complement, little endian
  Version:                           1 (current)
  OS/ABI:                            UNIX - System V
  ABI Version:                       0
  Type:                              DYN (Shared object file)
  Machine:                           AArch64
  Version:                           0x1
  Entry point address:               0x17e4
  Start of program headers:          64 (bytes into file)
  Start of section headers:          34904 (bytes into file)
  Flags:                             0x0
  Size of this header:               64 (bytes)
  Size of program headers:           56 (bytes)
  Number of program headers:         9
  Size of section headers:           64 (bytes)
  Number of section headers:         36
  Section header string table index: 35

建议用nfs把开发目录映射到开发板上，这样编译完成可以直接在开发板上运行。避免用sd卡或者tftp，需要反复复制下载

详细请参考下面链接
Petalinux快速入门向导 (11) 第十章.配置nfs服务器 https://blog.csdn.net/aatu/article/details/120755209

三、需要注意的地方

ret = system(“modprobe rpmsg_char”);

/sys/devices/platform/ff9a0000.zynqmp-rpu/r5@0/r5@0#vdev0buffer/virtio0/virtio0.rpmsg-openamp-demo-channel.-1.0

/sys/bus/rpmsg/devices/virtio0.rpmsg-openamp-demo-channel.-1.0 -> …/…/…/devices/platform/ff9a0000.zynqmp-rpu/r5@0/r5@0#vdev0buffer/virtio0/virtio0.rpmsg-openamp-demo-channel.-1.0

创建链接
ln -s /sys/devices/platform/ff9a0000.zynqmp-rpu/r5@0/r5@0#vdev0buffer/virtio0/virtio0.rpmsg-openamp-demo-channel.-1.0 /sys/bus/rpmsg/devices/virtio0.rpmsg-openamp-demo-channel.-1.0

Echo test start --2023-9-18 By Leo

Master>probe rpmsg_char

1Open rpmsg dev [/sys/bus/rpmsg/devices/virtio0.rpmsg-openamp-demo-channel.-1.0]!
Failed to write virtio0.rpmsg-openamp-demo-channel.-1.0 to /sys/bus/rpmsg/drivers/rpmsg_chrdev/bind, No such device

root@xilinx-zcu106-2020_2:/mnt/nfs# ls /sys/bus/rpmsg/drivers/rpmsg_chrdev/ -l
total 0
–w------- 1 root root 4096 Sep 19 01:00 bind
lrwxrwxrwx 1 root root 0 Sep 19 01:08 module -> …/…/…/…/module/rpmsg_char
–w------- 1 root root 4096 Sep 19 01:00 uevent
–w------- 1 root root 4096 Sep 19 01:08 unbind
lrwxrwxrwx 1 root root 0 Sep 19 01:08 virtio0.rpmsg-openamp-demo-channel.-1.0 -> …/…/…/…/devices/platform/ff9a0000.zynqmp-rpu/r5@0/r5@0#vdev0buffer/virtio0/virtio0.rpmsg-openamp-demo-channel.-1.0

下一步讲如何修改image_echo_test。
未完待续