Makefile

COMMON_REPO := ../common_xilinx/

#Common Includes
include $(COMMON_REPO)/utility/boards.mk
include $(COMMON_REPO)/libs/xcl2/xcl2.mk
include $(COMMON_REPO)/libs/opencl/opencl.mk






# Host Application

#helloworld_SRCS=./src/host.cpp $(oclHelper_SRCS) $(xcl2_SRCS)
#helloworld_HDRS=$(xcl2_HDRS)

#helloworld_SRCS=$(wildcard ../common/src/AOCLUtils/*.cpp wrapcl/src/*.cpp host/src/*.cpp) \
#		$(oclHelper_SRCS) \
#		$(xcl2_SRCS)

#helloworld_HDRS=$(wildcard ./host/inc/*.h) \
#		$(xcl2_HDRS)

hostbin_SRCS=$(wildcard ../common/src/AOCLUtils/*.cpp wrapcl/src/*.cpp host/src/*.cpp) \
		$(oclHelper_SRCS) \
		$(xcl2_SRCS)

hostbin_HDRS=$(wildcard ./host/inc/*.h) \
		$(xcl2_HDRS)

# -U__STRICT_ANSI__: to avoid: ‘cl_float3 {aka union cl_float4}’ has no member named ‘y’


#helloworld_CXXFLAGS=-I../common/inc/ \
#		    -I./host/inc/ \
#		    -I. \
#		    -I./wrapcl/inc/ \
#		    $(opencl_CXXFLAGS) $(xcl2_CXXFLAGS) $(oclHelper_CXXFLAGS) \
#		    -std=c++0x \
#		    -U__STRICT_ANSI__ \
#		    $(ENABLE_KERNELS) \
#		    $(REP) \
#		    $(FIPO_FLAG)

hostbin_CXXFLAGS=-I../common/inc/ \
		    -I./host/inc/ \
		    -I. \
		    -I./wrapcl/inc/ \
		    $(opencl_CXXFLAGS) $(xcl2_CXXFLAGS) $(oclHelper_CXXFLAGS) \
		    -std=c++0x \
		    -U__STRICT_ANSI__ \
		    $(ENABLE_KERNELS) \
		    $(REP) \
		    $(FIPO_FLAG)
		    
#helloworld_LDFLAGS=$(opencl_LDFLAGS)
#hostbin_LDFLAGS=$(opencl_LDFLAGS)
hostbin_LDFLAGS=$(opencl_LDFLAGS) -lrt

#EXES=helloworld
EXES=hostbin

# Kernel
#vector_addition_SRCS=./src/vector_addition.cl
#vector_addition_SRCS=./device/Krnl_GA.cl
Krnl_GA_SRCS=./device/Krnl_GA.cl

#-------------------------------
# AWS only: apply the following 
# kernel compile & link configuration options

# Otherwise error during VPL
# "dr.bd.tcl" seems to be using a configuration for VCU1525
# instead for AWS VU9P

# Problem reported here:
# https://forums.xilinx.com/t5/SDAccel/Building-ERROR-VPL-17-55-set-property-expects-at-least-one/td-p/893221

# AWS reference guide
# https://github.com/aws/aws-fpga/blob/master/SDAccel/docs/On_Premises_Development_Steps.md
# aws-fpga/SDAccel/examples/aws/kernel_3ddr_bandwidth/Makefile

# https://www.xilinx.com/support/documentation/sw_manuals/xilinx2017_2/ug1023-sdaccel-user-guide.pdf
# Page 33 (could not find in the v2017.4)
# --max_memory_ports [all | <kernel_name>]
# Optional
# Sets the maximum memory port property 
# for all kernels or a given kernel

# Choosing the "bandwidth" kernel for maximum memory port property 
# krnl_GA_CLFLAGS=--max_memory_ports bandwidth

# Choosing the all kernels in design for maximum memory port property 
Krnl_GA_CLFLAGS=--max_memory_ports all

# https://www.xilinx.com/support/documentation/sw_manuals/xilinx2017_4/ug1023-sdaccel-user-guide.pdf
# Page 44
# --sp <kernel_inst_name>.<interface_name>:<bank>
# Valid DDR4 bank names are 
# bank0, bank1, bank2, and bank3 
# for platforms with 4 DDR banks.

#krnl_GA_LDCLFLAGS+= --sp bandwidth_1.m_axi_gmem0:bank0 \
#		    --sp bandwidth_1.m_axi_gmem1:bank1 \
#		    --sp bandwidth_1.m_axi_gmem2:bank2	

Krnl_GA_LDCLFLAGS+=--sp Krnl_GA_1.m_axi_gmem0:bank0 \
		   --sp Krnl_Conform_1.m_axi_gmem1:bank1 \
		   --sp Krnl_InterE_1.m_axi_gmem2:bank2 \
		   --sp Krnl_IntraE_1.m_axi_gmem3:bank3	


##
# https://www.xilinx.com/support/documentation/sw_manuals/xilinx2017_4/ug1281-sdaccel-debugging-guide.pdf
# Page 32
# Instrumentation 1: Kernel Hangs due to AXI Violations
##

# Add Lightweight AXI Protocol Checker (lapc)
# <protocol:< compute_unit_name >:< interface_name >>
LAPC:=none

ifneq ($(LAPC),none)
# Number of axi protocol checkers requested must be between 1 and 63.
#Krnl_GA_LDCLFLAGS+=--dk protocol:all:all

Krnl_GA_LDCLFLAGS+=--dk protocol:Krnl_GA_1:m_axi_gmem0 \
		   --dk protocol:Krnl_Conform_1:m_axi_gmem1 \
                   --dk protocol:Krnl_InterE_1:m_axi_gmem2 \
	           --dk protocol:Krnl_IntraE_1:m_axi_gmem3
endif

#-------------------------------



#XOS=vector_addition
XOS=Krnl_GA

#vector_addition_XOS=vector_addition
Krnl_GA_XOS=Krnl_GA

#XCLBINS=vector_addition
XCLBINS=Krnl_GA

# check
#check_EXE=helloworld
check_EXE=hostbin
#check_XCLBINS=vector_addition
check_XCLBINS=Krnl_GA

CHECKS=check

include $(COMMON_REPO)/utility/rules.mk

# Note that the definition of "newline" contains two blank lines.
# Using $(newline) will expand into only one newline.
define newline


endef

# Docking parameters 
# that directly determine the execution time
# Following values are used only for emulation
# to have a short but still a representative test
PDB  := 1stp
PSIZE:= 150
NRUN := 10
NEV  := 800000
NGEN := 27000
GFPOP:= 0

# Switching to the other SDAccel 2017.4 default platform
# As it is available in Xelera "saruman" server
#DEVICES:= xilinx:kcu1500:dynamic (in ../common_xilinx/utility/boards.mk)

# Boards in SDAccel installation path: /opt/cad/xilinx/SDx/2017.4/platforms/

# If several versions are present, e.g.: "vcu1525_5_0" and "vcu1525_5_1"
# Pay attention to which should be used
#BOARD:= xilinx:kcu1500:dynamic
#BOARD:= xilinx:vcu1525:dynamic

# Boards in external paths

# Cloned from https://github.com/aws/aws-fpga
# This corresponds to AWS Shell v04261818 
# https://github.com/aws/aws-fpga/blob/master/hdk/docs/AWS_Shell_RELEASE_NOTES.md
# IMPORTANT: source scripts to get the full DSA
# Because by default only the "sw" component is provided in GitHub
#BOARD:= /home/wimi/lvs/ocl_xilinx/aws-fpga/SDAccel/aws_platform/xilinx_aws-vu9p-f1-04261818_dynamic_5_0/xilinx_aws-vu9p-f1-04261818_dynamic_5_0.xpfm
BOARD:= /home/wimi/lvs/DSA_SDACCEL_AWS_VU9P/xilinx_aws-vu9p-f1-04261818_dynamic_5_0/xilinx_aws-vu9p-f1-04261818_dynamic_5_0.xpfm

# Copied from Xelera (saruman) SDAccel installation path
#BOARD:= /home/wimi/lvs/DSA_SDACCEL_AWS_VU9P/xilinx_aws-vu9p-f1_dynamic_5_0/xilinx_aws-vu9p-f1_dynamic_5_0.xpfm
#BOARD:= /home/wimi/lvs/DSA_SDACCEL_VCU1525_5_1/xilinx_vcu1525_dynamic_5_1/xilinx_vcu1525_dynamic_5_1.xpfm
#BOARD:= /opt/cad/xilinx/SDx/2017.1/platforms/xilinx_adm-pcie-7v3_1ddr_3_0/xilinx_adm-pcie-7v3_1ddr_3_0.xpfm

NDEVICES_EMU:=1


# Compile and generate report
kerrpt-sw:
	make clean
	@echo $(newline)
	make all REPORT=estimate TARGETS=sw_emu DEVICES=$(BOARD)
	@echo $(newline)
	cp ./xclbin/$(XCLBINS).sw_emu.*.xclbin .
	@echo $(newline)
	mv ./$(XCLBINS).sw_emu.*.xclbin ./$(XCLBINS).xclbin
	@echo $(newline)
	@echo "==============================================================================="
	@echo "INFO: sw emulation file created under: \"./xclbin/$(XCLBINS).sw_emu.*.xclbin\" "
	@echo "INFO: ... was copied and renamed as \"./$(XCLBINS).xclbin\"                    "
	@echo $(newline)
	@echo "INFO: a \"./system_estimate.xtxt\" report was generated!                       "
	@echo "==============================================================================="
	@echo $(newline)


# Compile for sw and hw emulation
kerswemu:
	make all TARGETS=sw_emu DEVICES=$(BOARD)
	cp ./xclbin/$(XCLBINS).sw_emu.*.xclbin .
	mv ./$(XCLBINS).sw_emu.*.xclbin ./$(XCLBINS).xclbin
	@echo $(newline)
	@echo "==============================================================================="
	@echo "INFO: sw emulation file created under: \"./xclbin/$(XCLBINS).sw_emu.*.xclbin\" "
	@echo "INFO: ... was copied and renamed as \"./$(XCLBINS).xclbin\"    	              "
	@echo "==============================================================================="
	@echo $(newline)


kerhwemu:
	make all TARGETS=hw_emu	DEVICES=$(BOARD)
	cp ./xclbin/$(XCLBINS).hw_emu.*.xclbin .
	mv ./$(XCLBINS).hw_emu.*.xclbin ./$(XCLBINS).xclbin
	@echo $(newline)
	@echo "==============================================================================="
	@echo "INFO: hw emulation file created under: \"./xclbin/$(XCLBINS).hw_emu.*.xclbin\" "
	@echo "INFO: ... was copied and renamed as \"./$(XCLBINS).xclbin\"                    "
	@echo "==============================================================================="
	@echo $(newline)


# Run emulation
# Compilation (host & device) if not done previously
# The .dlg file is written in ./
# "exe" and "xclbin" are rules from "rules.mk"
# SDAccel 2017.4: User Guide UG1023, "Running Software and Hardware Emulation in XOCC Flow"
#emconfigutil --platform xilinx_vcu1525_dynamic_5_0 --nd 1
swemu: kerswemu
	emconfigutil --platform $(BOARD) --nd $(NDEVICES_EMU)
	XCL_EMULATION_MODE=sw_emu ./$(EXES) \
	-ffile input/$(PDB)/derived/$(PDB)_protein.maps.fld \
	-lfile input/$(PDB)/derived/$(PDB)_ligand.pdbqt \
	-psize $(PSIZE) -nrun $(NRUN) -nev $(NEV) -ngen $(NGEN) -gfpop 1
	@echo $(newline)
	@echo "===================================================================================="
	@echo "INFO: swemu log file is under: \"./docking.dlg\"                                    "
	@echo $(newline)
	@echo "INFO: profile and timeline trace report files are formatted for SDx GUI             "
	@echo "INFO: ... if reports not generated, verify that \"sdaccel.ini\" exists!             "
	@echo $(newline)
	sdx_analyze profile sdaccel_profile_summary.csv
	@echo $(newline)
	sdx_analyze trace sdaccel_timeline_trace.csv
	@echo $(newline)
	@echo "INFO: (1) open sdx, (2) select \"File->Open File\", (3) browse to *.xprf or *.wdb  "
	@echo "===================================================================================="
	@echo $(newline)

# Debug on software using Xilinx GDB
# Analyze hangs with the following gdb extensions:
# xprint queue [<cl_command_queue>]
# xprint event <cl_event>
# xprint mem [<cl_mem>]
# xprint kernel
# xprint all
swemudebug:
	xgdb --args \
	./$(EXES) \
	-ffile input/$(PDB)/derived/$(PDB)_protein.maps.fld \
	-lfile input/$(PDB)/derived/$(PDB)_ligand.pdbqt \
	-psize $(PSIZE) -nrun $(NRUN) -nev $(NEV) -ngen $(NGEN) -gfpop 1


hwemu: kerhwemu
	emconfigutil --platform $(BOARD) --nd $(NDEVICES_EMU)
	XCL_EMULATION_MODE=hw_emu ./$(EXES) \
	-ffile input/$(PDB)/derived/$(PDB)_protein.maps.fld \
	-lfile input/$(PDB)/derived/$(PDB)_ligand.pdbqt \
	-psize $(PSIZE) -nrun $(NRUN) -nev $(NEV) -ngen $(NGEN) -gfpop 1
	@echo $(newline)
	@echo "==========================================================================="
	@echo "INFO: hwemu log file is under: \"./docking.dlg\"                           "
	@echo $(newline)
	@echo "INFO: profile and timeline trace report files are formatted for SDx GUI    "
	@echo "INFO: ... if reports not generated, verify that \"sdaccel.ini\" exists!    "
	@echo $(newline)
	sdx_analyze profile sdaccel_profile_summary.csv
	@echo $(newline)
	sdx_analyze trace sdaccel_timeline_trace.csv
	@echo "==========================================================================="
	@echo $(newline)


# Build hw (FPGA bitstream)
# NOT integrated yet in a multistep compilation
hw: 	
	make all TARGETS=hw DEVICES=$(BOARD)
	@echo $(newline)
	@echo "========================================================================="
	@echo "INFO: FPGA bitstream is under: \"./xclbin/$(XCLBINS).hw.*.xclbin\"       " 
	@echo "========================================================================="
	@echo $(newline)


# Build instrumented hw (FPGA bitstream)
# NOT integrated yet in a multistep compilation
hw-instr: 	
	make all TARGETS=hw DEVICES=$(BOARD) LAPC=yes
	@echo $(newline)
	@echo "=============================================================================================="
	@echo "INFO: [instrumented] FPGA bitstream is under: \"./xclbin/$(XCLBINS).hw.*.xclbin\"             " 
	@echo "=============================================================================================="
	@echo $(newline)



# Execute on remote server
# Host-recompilation
# Copy of host to $(TARGET_DIR_HW)
# The working directory is changed
# So .dlg file is written in $(TARGET_DIR_HW)
eva: exe
	./$(EXES) \
	-ffile input/$(PDB)/derived/$(PDB)_protein.maps.fld \
	-lfile input/$(PDB)/derived/$(PDB)_ligand.pdbqt \
	-resnam docking_$(PDB)_$(NRUN) \
	-nrun $(NRUN) -gfpop $(GFPOP)
	@echo $(newline)
	@echo "==========================================================================="
	@echo "INFO: evaluation log file is under: \"./docking.dlg\"                      "
	@echo $(newline)
	@echo "INFO: profile and timeline trace report files are formatted for SDx GUI    "
	@echo "INFO: ... if reports not generated, verify that \"sdaccel.ini\" exists!    "
	@echo $(newline)
	sdx_analyze profile sdaccel_profile_summary.csv
	@echo $(newline)
	sdx_analyze trace sdaccel_timeline_trace.csv
	@echo "==========================================================================="
	@echo $(newline)


# Debug on hardware using Xilinx GDB
# Analyze hangs with the following gdb extensions:
# xprint queue [<cl_command_queue>]
# xprint event <cl_event>
# xprint mem [<cl_mem>]
# xprint kernel
# xprint all
evadebug:
	xgdb --args \
	./$(EXES) \
	-ffile input/$(PDB)/derived/$(PDB)_protein.maps.fld \
	-lfile input/$(PDB)/derived/$(PDB)_ligand.pdbqt \
	-resnam docking_debug_$(PDB)_$(NRUN) \
	-nrun $(NRUN) -gfpop $(GFPOP)


# Project name
PROJECT_NAME := ofdock_taskpar_xl

COPY_DIR := copy_$(PROJECT_NAME)
COPY_DIR_DOCK := $(PROJECT_NAME)

# Make sure this remote-server alias exists (in ~/.ssh/config)

## Xelera
#SERVER_BRIDGE := erebor:/scratch/
#SERVER_EXEC   := saruman:~/ocladock_sdx17_4

## AWS
SERVER_BRIDGE := eredmithrim:/home/lvs/ESA/AWS
SERVER_EXEC   := AWS

# Delete an existing folder if previously created
# Create a folder $(COPY_DIR) in the parent directory
# Copy binaries into $(COPY_DIR) folder, avoiding the (innecesary) compilation intermediate files
# Copy source files into $(COPY_DIR) as host is built in the remote $(SERVER)
# Create a zip file out of the $(COPY_DIR) folder
copy: clean-copy
	mkdir ../$(COPY_DIR)
	mkdir ../$(COPY_DIR)/$(COPY_DIR_DOCK)
	mkdir ../$(COPY_DIR)/$(COPY_DIR_DOCK)/xclbin
	cp ./xclbin/$(XCLBINS).hw.*.xclbin ../$(COPY_DIR)/$(COPY_DIR_DOCK)/xclbin/
	cp ./xclbin/$(XCLBINS).hw.*.xo ../$(COPY_DIR)/$(COPY_DIR_DOCK)/xclbin/
	cp ./xclbin/$(XCLBINS).hw.*.xclbin ../$(COPY_DIR)/$(COPY_DIR_DOCK)/
	mv ../$(COPY_DIR)/$(COPY_DIR_DOCK)/$(XCLBINS).hw.*.xclbin ../$(COPY_DIR)/$(COPY_DIR_DOCK)/$(XCLBINS).xclbin
	cp sdaccel.ini ../$(COPY_DIR)/$(COPY_DIR_DOCK)/
	cp Makefile ../$(COPY_DIR)/$(COPY_DIR_DOCK)/
	cp defines.h ../$(COPY_DIR)/$(COPY_DIR_DOCK)/
	cp defines_fixedpt.h ../$(COPY_DIR)/$(COPY_DIR_DOCK)/
	cp defines_fixedpt_64.h ../$(COPY_DIR)/$(COPY_DIR_DOCK)/
	cp -r device/ ../$(COPY_DIR)/$(COPY_DIR_DOCK)/
	cp -r host/ ../$(COPY_DIR)/$(COPY_DIR_DOCK)/
	#cp -r input/ ../$(COPY_DIR)/$(COPY_DIR_DOCK)/
	cp -r wrapcl/ ../$(COPY_DIR)/$(COPY_DIR_DOCK)/
	cp -r ../common ../$(COPY_DIR)/
	cp -r ../common_xilinx ../$(COPY_DIR)/
	cp ../conf_sdx174_saruman.sh ../$(COPY_DIR)/
	cd ../;	zip -r $(COPY_DIR).zip $(COPY_DIR)
	@echo $(newline)
	@echo "Transfer the \"$(COPY_DIR).zip\" file to $(SERVER_BRIDGE) server: "
	@echo " "
	cd ../ && scp $(COPY_DIR).zip $(SERVER_BRIDGE)
	@echo $(newline)
	@echo "=================================================================="
	@echo "Files listed were transferred to $(SERVER_BRIDGE)                 "
	@echo $(newline)
	@echo "Then, create AFI and transfer files manually                      "
	@echo "from $(SERVER_BRIDGE) to $(SERVER_EXEC)                           "
	@echo "=================================================================="
	@echo $(newline)


clean-copy:
	rm -rf ../$(COPY_DIR)
	rm -f ../$(COPY_DIR).zip
	@echo $(newline)
	@echo "=================================================================="
	@echo "Previous \"$(COPY_DIR).zip\" and its folder were deleted          "
	@echo "=================================================================="
	@echo $(newline)