performdocking.cpp 41.5 KB
Newer Older
lvs's avatar
lvs committed
1
2
3
4
5
#include "performdocking.h"

//// --------------------------------
//// Device memory buffers
//// --------------------------------
Leonardo Solis's avatar
Leonardo Solis committed
6

lvs's avatar
lvs committed
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

/*								                  // Nr elements	// Nr bytes
cl_mem mem_atom_charges_const;		// float [MAX_NUM_OF_ATOMS];			// 90	 = 90	//360
cl_mem mem_atom_types_const;		// char  [MAX_NUM_OF_ATOMS];			// 90	 = 90	//360
cl_mem mem_intraE_contributors_const;	// char  [3*MAX_INTRAE_CONTRIBUTORS];		// 3*8128=28384 //24384	
cl_mem mem_VWpars_AC_const;		// float [MAX_NUM_OF_ATYPES*MAX_NUM_OF_ATYPES];	// 14*14 = 196  //784
cl_mem mem_VWpars_BD_const;		// float [MAX_NUM_OF_ATYPES*MAX_NUM_OF_ATYPES]; // 14*14 = 196	//784
cl_mem mem_dspars_S_const;		// float [MAX_NUM_OF_ATYPES];			// 14    = 14   //56
cl_mem mem_dspars_V_const;		// float [MAX_NUM_OF_ATYPES];			// 14    = 14   //56
cl_mem mem_rotlist_const;		// int   [MAX_NUM_OF_ROTATIONS];		// 4096  = 4096 //16384
cl_mem mem_ref_coords_x_const;		// float [MAX_NUM_OF_ATOMS];			// 90    = 90   //360
cl_mem mem_ref_coords_y_const;		// float [MAX_NUM_OF_ATOMS];			// 90    = 90   //360
cl_mem mem_ref_coords_z_const;		// float [MAX_NUM_OF_ATOMS];			// 90	 = 90   //360
cl_mem mem_rotbonds_moving_vectors_const;// float [3*MAX_NUM_OF_ROTBONDS];		// 3*32  = 96   //384
cl_mem mem_rotbonds_unit_vectors_const;	// float [3*MAX_NUM_OF_ROTBONDS];		// 3*32  = 96   //384
cl_mem mem_ref_orientation_quats_const;	// float [4*MAX_NUM_OF_RUNS];			// 4*100 = 400  //1600
*/

//// --------------------------------
//// Docking
//// --------------------------------
28
29
30
31
32
33
34
int docking_with_fpga( const    Gridinfo*   mygrid,
	              /*const*/ float*      cpu_floatgrids,
                                Dockpars*   mypars,
		      const     Liganddata* myligand_init,
		      const     int*        argc,
		                char**      argv,
		                clock_t     clock_start_program)
lvs's avatar
lvs committed
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
/* The function performs the docking algorithm and generates the corresponding result files.
parameter mygrid:
		describes the grid
		filled with get_gridinfo()
parameter cpu_floatgrids:
		points to the memory region containing the grids
		filled with get_gridvalues_f()
parameter mypars:
		describes the docking parameters
		filled with get_commandpars()
parameter myligand_init:
		describes the ligands
		filled with get_liganddata()
parameters argc and argv:
		are the corresponding command line arguments parameter clock_start_program:
		contains the state of the clock tick counter at the beginning of the program
filled with clock() */
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
{
	// The get_xil_devices will return vector of Xilinx Devices 
	std::vector<cl::Device> devices = xcl::get_xil_devices();
	cl::Device device = devices[0];

	//Creating Context and Command Queue for selected Device 
	cl::Context context(device);
    	std::string device_name = device.getInfo<CL_DEVICE_NAME>(); 
	std::cout << "Found Device=" << device_name.c_str() << std::endl;

	#ifdef ENABLE_KRNL_GA
	cl::CommandQueue command_queue_ga			(context, device, CL_QUEUE_PROFILING_ENABLE);
	#endif
	#ifdef ENABLE_KRNL_CONFORM
	cl::CommandQueue command_queue_conform			(context, device, CL_QUEUE_PROFILING_ENABLE);
	#endif
	#ifdef ENABLE_KRNL_INTERE
	cl::CommandQueue command_queue_intere			(context, device, CL_QUEUE_PROFILING_ENABLE);
	#endif
	#ifdef ENABLE_KRNL_INTRAE
	cl::CommandQueue command_queue_intrae			(context, device, CL_QUEUE_PROFILING_ENABLE);
	#endif
	#ifdef ENABLE_KRNL_PRNG_BT_USHORT_FLOAT
	cl::CommandQueue command_queue_prng_bt_ushort_float	(context, device, CL_QUEUE_PROFILING_ENABLE);
	#endif
	#ifdef ENABLE_KRNL_PRNG_GG_UCHAR
	cl::CommandQueue command_queue_prng_gg_uchar		(context, device, CL_QUEUE_PROFILING_ENABLE);
	#endif
	#ifdef ENABLE_KRNL_PRNG_GG_FLOAT
	cl::CommandQueue command_queue_prng_gg_float		(context, device, CL_QUEUE_PROFILING_ENABLE);
	#endif
	#ifdef ENABLE_KRNL_IGL_ARBITER
	cl::CommandQueue command_queue_igl_arbiter		(context, device, CL_QUEUE_PROFILING_ENABLE);
	#endif

    	// import_binary() command will find the OpenCL binary file created using the 
    	// xocc compiler load into OpenCL Binary and return as Binaries
    	// OpenCL and it can contain many functions which can be executed on the
   	// device.
    	std::string binaryFile = xcl::find_binary_file(device_name,"Krnl_GA");
    	cl::Program::Binaries bins = xcl::import_binary_file(binaryFile);
    	devices.resize(1);
    	cl::Program program(context, devices, bins);

    	// This call will extract a kernel out of the program we loaded in the
    	// previous line. A kernel is an OpenCL function that is executed on the
    	// FPGA. This function is defined in the device/Krnl_GA.cl file.
	#ifdef ENABLE_KRNL_GA
	cl::Kernel kernel_ga			(program, "Krnl_GA");
	#endif
	#ifdef ENABLE_KRNL_CONFORM
	cl::Kernel kernel_conform		(program, "Krnl_Conform");
	#endif
	#ifdef ENABLE_KRNL_INTERE
	cl::Kernel kernel_intere		(program, "Krnl_InterE");
	#endif
	#ifdef ENABLE_KRNL_INTRAE
	cl::Kernel kernel_intrae		(program, "Krnl_IntraE");
	#endif
	#ifdef ENABLE_KRNL_PRNG_BT_USHORT_FLOAT
	cl::Kernel kernel_prng_bt_ushort_float	(program, "Krnl_Prng_BT_ushort_float");
	#endif
	#ifdef ENABLE_KRNL_PRNG_GG_UCHAR
	cl::Kernel kernel_prng_gg_uchar		(program, "Krnl_Prng_GG_uchar");
	#endif
	#ifdef ENABLE_KRNL_PRNG_GG_FLOAT
	cl::Kernel kernel_prng_gg_float		(program, "Krnl_Prng_GG_float");
	#endif
	#ifdef ENABLE_KRNL_IGL_ARBITER
	cl::Kernel kernel_igl_arbiter		(program, "Krnl_IGL_Arbiter");
	#endif
lvs's avatar
lvs committed
123

124
125
126
	clock_t clock_start_docking;
	clock_t	clock_stop_docking;
	clock_t clock_stop_program_before_clustering;
lvs's avatar
lvs committed
127

128
129
	Liganddata myligand_reference;

130
 	//allocating FPGA memory for floatgrids,
131
132
	size_t size_floatgrids_nbytes = sizeof(float) * (mygrid->num_of_atypes+2) *
					(mygrid->size_xyz[0]) * (mygrid->size_xyz[1]) * (mygrid->size_xyz[2]);
lvs's avatar
lvs committed
133

134
135
	size_t size_populations_nbytes = mypars->num_of_runs * mypars->pop_size * ACTUAL_GENOTYPE_LENGTH * sizeof(float);
	size_t size_populations_nelems = mypars->num_of_runs * mypars->pop_size * ACTUAL_GENOTYPE_LENGTH;
lvs's avatar
lvs committed
136

137
138
	size_t size_energies_nbytes = mypars->num_of_runs * mypars->pop_size * sizeof(float);
	size_t size_energies_nelems = mypars->num_of_runs * mypars->pop_size;
lvs's avatar
lvs committed
139

140
141
142
143
144
	//allocating and initializing CPU memory for initial population
	vector<float,aligned_allocator<float>> cpu_init_populations (size_populations_nelems, 0.0f);

	//allocating CPU memory for final population
	vector<float,aligned_allocator<float>> cpu_final_populations (size_populations_nelems);
lvs's avatar
lvs committed
145

146
147
148
149
150
151
152
153
	//allocating CPU memory for results
	vector<float,aligned_allocator<float>> cpu_energies (size_energies_nelems);

	//allocating CPU memory for resulting ligands
	vector<Ligandresult,aligned_allocator<Ligandresult>> cpu_result_ligands (mypars->num_of_runs);

	//allocating memory in CPU for reference orientation angles
	vector<float,aligned_allocator<float>> cpu_ref_ori_angles (mypars->num_of_runs*3);
lvs's avatar
lvs committed
154
155
156
157

	//generating initial populations and random orientation angles of reference ligand
	//(ligand will be moved to origo and scaled as well)
	myligand_reference = *myligand_init;
158
	gen_initpop_and_reflig(mypars, cpu_init_populations.data(), cpu_ref_ori_angles.data(), &myligand_reference, mygrid);
159
160
161
162
163
164
165
166
167
168
169

	//allocating memory in CPU for pseudorandom number generator seeds
	const unsigned int num_of_prng_blocks = 25;
	size_t size_prng_seeds_nelems = num_of_prng_blocks * mypars->num_of_runs;
	vector<unsigned int,aligned_allocator<unsigned int>> cpu_prng_seeds (size_prng_seeds_nelems);
	
	//initializing seed generator
	genseed(time(NULL));	

	//generating seeds (for each thread during GA)
	for (unsigned int i=0; i<size_prng_seeds_nelems; i++) {
lvs's avatar
lvs committed
170
171
		cpu_prng_seeds[i] = genseed(0u);
	}
172
173
174
175

	size_t size_evals_of_runs_nbytes = mypars->num_of_runs*sizeof(int);
	size_t size_evals_of_runs_nelems = mypars->num_of_runs;

lvs's avatar
lvs committed
176
	// allocating memory in CPU for evaluation counters
177
178
179
180
	vector<int,aligned_allocator<int>> cpu_evals_of_runs (size_evals_of_runs_nelems, 0);
	
	// allocating memory in CPU for generation counters
	vector<int,aligned_allocator<int>> cpu_gens_of_runs (size_evals_of_runs_nelems, 0);
lvs's avatar
lvs committed
181

182
	//preparing the constant data fields for the FPGA (calcenergy.cpp)
183
184
	// -----------------------------------------------------------------------------------------------------
	kernelconstant_static  KerConstStatic;
185
	if (prepare_conststatic_fields_for_fpga(&myligand_reference, mypars, cpu_ref_ori_angles.data(), &KerConstStatic) == 1)
lvs's avatar
lvs committed
186
187
		return 1;

188
189
190
191
192
	//preparing parameter struct
	Dockparameters dockpars;
	dockpars.num_of_atoms  			= ((unsigned char)  myligand_reference.num_of_atoms);
	dockpars.num_of_atypes 			= ((unsigned char)  myligand_reference.num_of_atypes);
	dockpars.num_of_intraE_contributors 	= ((unsigned int)   myligand_reference.num_of_intraE_contributors);
193
194
195
196
197
198
199
	dockpars.gridsize_x    			= ((unsigned char)  mygrid->size_xyz[0]);
	dockpars.gridsize_y    			= ((unsigned char)  mygrid->size_xyz[1]);
	dockpars.gridsize_z    			= ((unsigned char)  mygrid->size_xyz[2]);
	dockpars.g1	       			= dockpars.gridsize_x ;
	dockpars.g2	       			= dockpars.gridsize_x * dockpars.gridsize_y;
	dockpars.g3	       			= dockpars.gridsize_x * dockpars.gridsize_y * dockpars.gridsize_z;
	dockpars.grid_spacing  			= ((float) mygrid->spacing);
200
	dockpars.rotbondlist_length 		= ((unsigned int) myligand_reference.num_of_rotcyc);
201
	dockpars.coeff_elec    			= ((float) mypars->coeffs.scaled_AD4_coeff_elec);
202
	dockpars.coeff_desolv  			= ((float) mypars->coeffs.AD4_coeff_desolv);
203
204
205
	dockpars.num_of_energy_evals 		= (unsigned int) mypars->num_of_energy_evals;
	dockpars.num_of_generations  		= (unsigned int) mypars->num_of_generations;
	dockpars.pop_size      			= (unsigned int) mypars->pop_size;
206
	dockpars.num_of_genes  			= (unsigned char)(myligand_reference.num_of_rotbonds + 6);
207
208
209
210
211
212
213
214
215
216
217
218
219
220
	dockpars.tournament_rate 		= (mypars->tournament_rate)/100;
	dockpars.crossover_rate  		= (mypars->crossover_rate)/100;
	dockpars.mutation_rate   		= (mypars->mutation_rate)/100;
	dockpars.abs_max_dang    		= mypars->abs_max_dang;
	dockpars.abs_max_dmov    		= mypars->abs_max_dmov;
	dockpars.lsearch_rate    		= mypars->lsearch_rate;
	dockpars.num_of_lsentities 		= (unsigned int) (mypars->lsearch_rate/100.0*mypars->pop_size + 0.5);
	dockpars.rho_lower_bound   		= mypars->rho_lower_bound;
	dockpars.base_dmov_mul_sqrt3 		= mypars->base_dmov_mul_sqrt3;
	dockpars.base_dang_mul_sqrt3 		= mypars->base_dang_mul_sqrt3;
	dockpars.cons_limit        		= (unsigned int) mypars->cons_limit;
	dockpars.max_num_of_iters  		= (unsigned int) mypars->max_num_of_iters;
	dockpars.qasp 				= mypars->qasp;
	dockpars.smooth 			= mypars->smooth;
lvs's avatar
lvs committed
221
222
223
224
225
226
227
228
229
230

	// these variables hold multiplications between kernel-constants
	// better calculate them here and then pass them to Krnl_GA
	const float two_absmaxdmov = 2.0 * dockpars.abs_max_dmov;
	const float two_absmaxdang = 2.0 * dockpars.abs_max_dang;

	// these variables hold multiplications between kernel-constants
	// better calculate them here and then pass them to Krnl_InterE and Krnl_InterE2
	const unsigned int mul_tmp2 = dockpars.num_of_atypes * dockpars.g3;
	const unsigned int mul_tmp3 = (dockpars.num_of_atypes + 1) * dockpars.g3;
231

232
	// -----------------------------------------------------------------------------------------------------
233
	// Replaced by cl_mem extensions (custom connections to DDR banks)
234
	#if 0
235

236
237
238
239
240
241
        // These commands will allocate memory on the FPGA. The cl::Buffer objects can
        // be used to reference the memory locations on the device. The cl::Buffer
	// object cannot be referenced directly and must be passed to other OpenCL
	// functions.

	// Krnl_GA buffers
242
243
	cl::Buffer mem_dockpars_conformations_current_Initial
							(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY, 
244
							size_populations_nbytes,	cpu_init_populations.data());
245
246
247
	cl::Buffer mem_dockpars_conformations_current_Final
							(context, CL_MEM_USE_HOST_PTR | CL_MEM_WRITE_ONLY, 
							size_populations_nbytes,	cpu_final_populations.data());
248
	cl::Buffer mem_dockpars_energies_current     	(context, CL_MEM_USE_HOST_PTR | CL_MEM_WRITE_ONLY, 
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
							size_energies_nbytes,     	cpu_energies.data());
	cl::Buffer mem_evals_performed			(context, CL_MEM_USE_HOST_PTR | CL_MEM_WRITE_ONLY, 
							size_evals_of_runs_nbytes, 	cpu_evals_of_runs.data());
	cl::Buffer mem_gens_performed			(context, CL_MEM_USE_HOST_PTR | CL_MEM_WRITE_ONLY, 
							size_evals_of_runs_nbytes,	cpu_gens_of_runs.data());
	// Krnl_Conform buffers
	cl::Buffer mem_KerConstStatic_rotlist_const	(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							MAX_NUM_OF_ROTATIONS*sizeof(int),	&KerConstStatic.rotlist_const[0]);
	cl::Buffer mem_KerConstStatic_ref_coords_const	(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							MAX_NUM_OF_ATOMS*sizeof(cl_float3),	&KerConstStatic.ref_coords_const[0]);	
	cl::Buffer mem_KerConstStatic_rotbonds_moving_vectors_const
							(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,  
							MAX_NUM_OF_ROTBONDS*sizeof(cl_float3),	&KerConstStatic.rotbonds_moving_vectors_const[0]);
	cl::Buffer mem_KerConstStatic_rotbonds_unit_vectors_const
							(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							MAX_NUM_OF_ROTBONDS*sizeof(cl_float3),	&KerConstStatic.rotbonds_unit_vectors_const[0]);
	cl::Buffer mem_KerConstStatic_ref_orientation_quats_const
							(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							MAX_NUM_OF_RUNS*sizeof(cl_float4),   	&KerConstStatic.ref_orientation_quats_const[0]);
	
	// Krnl_InterE buffers
	cl::Buffer mem_dockpars_fgrids			(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							size_floatgrids_nbytes,		cpu_floatgrids);	
	cl::Buffer mem_KerConstStatic_InterE_atom_charges_const
							(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							MAX_NUM_OF_ATOMS*sizeof(float),	&KerConstStatic.atom_charges_const[0]);
	cl::Buffer mem_KerConstStatic_InterE_atom_types_const
							(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							MAX_NUM_OF_ATOMS*sizeof(char),	&KerConstStatic.atom_types_const[0]);
	
	// Krnl_IntraE buffers
	cl::Buffer mem_KerConstStatic_IntraE_atom_charges_const
							(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							MAX_NUM_OF_ATOMS*sizeof(float),	&KerConstStatic.atom_charges_const[0]);
	cl::Buffer mem_KerConstStatic_IntraE_atom_types_const
							(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							MAX_NUM_OF_ATOMS*sizeof(char), &KerConstStatic.atom_types_const[0]);
	cl::Buffer mem_KerConstStatic_intraE_contributors_const
							(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							MAX_INTRAE_CONTRIBUTORS*sizeof(cl_char3), &KerConstStatic.intraE_contributors_const[0]);
	cl::Buffer mem_KerConstStatic_reqm_const	(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							ATYPE_NUM*sizeof(float),	&KerConstStatic.reqm_const);	
	cl::Buffer mem_KerConstStatic_reqm_hbond_const	(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							ATYPE_NUM*sizeof(float),	&KerConstStatic.reqm_hbond_const);
	cl::Buffer mem_KerConstStatic_atom1_types_reqm_const
							(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							ATYPE_NUM*sizeof(unsigned int),	&KerConstStatic.atom1_types_reqm_const);
	cl::Buffer mem_KerConstStatic_atom2_types_reqm_const
							(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							ATYPE_NUM*sizeof(unsigned int),	&KerConstStatic.atom2_types_reqm_const);
	cl::Buffer mem_KerConstStatic_VWpars_AC_const	(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							MAX_NUM_OF_ATYPES*MAX_NUM_OF_ATYPES*sizeof(float),	&KerConstStatic.VWpars_AC_const[0]);
	cl::Buffer mem_KerConstStatic_VWpars_BD_const	(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							MAX_NUM_OF_ATYPES*MAX_NUM_OF_ATYPES*sizeof(float),	&KerConstStatic.VWpars_BD_const[0]);
	cl::Buffer mem_KerConstStatic_dspars_S_const	(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							MAX_NUM_OF_ATYPES*sizeof(float),			&KerConstStatic.dspars_S_const[0]);
	cl::Buffer mem_KerConstStatic_dspars_V_const	(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY,
							MAX_NUM_OF_ATYPES*sizeof(float),			&KerConstStatic.dspars_V_const[0]);
	#if !defined(SW_EMU)
	//allocating CPU memory for dummy data (one integer)
	vector<int,aligned_allocator<int>> cpu_dummy (1);	
310

311
312
	// IMPORTANT: enable this dummy global argument only for "hw" build.
	// https://forums.xilinx.com/t5/SDAccel/ERROR-KernelCheck-83-114-in-sdx-2017-4/td-p/818135
313
	cl::Buffer mem_dummy				(context, CL_MEM_USE_HOST_PTR | CL_MEM_READ_ONLY, sizeof(int), cpu_dummy.data());
314
	#endif
315

316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
	#endif
	// -----------------------------------------------------------------------------------------------------
        // For Allocating Buffer to specific Global Memory Bank, 
	// user has to use cl_mem_ext_ptr_t and provide the Banks

	// Declaring two extensions for both buffers

	// Krnl_GA cl_mem extensions
	cl_mem_ext_ptr_t memExt_dockpars_conformations_current_Initial;
	cl_mem_ext_ptr_t memExt_dockpars_conformations_current_Final;
	cl_mem_ext_ptr_t memExt_dockpars_energies_current;
	cl_mem_ext_ptr_t memExt_evals_performed;
	cl_mem_ext_ptr_t memExt_gens_performed;

	// Krnl_Conform cl_mem extensions
	cl_mem_ext_ptr_t memExt_KerConstStatic_rotlist_const;
	cl_mem_ext_ptr_t memExt_KerConstStatic_ref_coords_const;	
	cl_mem_ext_ptr_t memExt_KerConstStatic_rotbonds_moving_vectors_const;			
	cl_mem_ext_ptr_t memExt_KerConstStatic_rotbonds_unit_vectors_const;
	cl_mem_ext_ptr_t memExt_KerConstStatic_ref_orientation_quats_const;

	// Krnl_InterE cl_mem extensions
	cl_mem_ext_ptr_t memExt_dockpars_fgrids;	
	cl_mem_ext_ptr_t memExt_KerConstStatic_InterE_atom_charges_const;
	cl_mem_ext_ptr_t memExt_KerConstStatic_InterE_atom_types_const;

	// Krnl_IntraE cl_mem extensions
	cl_mem_ext_ptr_t memExt_KerConstStatic_IntraE_atom_charges_const;
	cl_mem_ext_ptr_t memExt_KerConstStatic_IntraE_atom_types_const;
	cl_mem_ext_ptr_t memExt_KerConstStatic_intraE_contributors_const;
	cl_mem_ext_ptr_t memExt_KerConstStatic_reqm_const;	
	cl_mem_ext_ptr_t memExt_KerConstStatic_reqm_hbond_const;
	cl_mem_ext_ptr_t memExt_KerConstStatic_atom1_types_reqm_const;
	cl_mem_ext_ptr_t memExt_KerConstStatic_atom2_types_reqm_const;
	cl_mem_ext_ptr_t memExt_KerConstStatic_VWpars_AC_const;
	cl_mem_ext_ptr_t memExt_KerConstStatic_VWpars_BD_const;
	cl_mem_ext_ptr_t memExt_KerConstStatic_dspars_S_const;
	cl_mem_ext_ptr_t memExt_KerConstStatic_dspars_V_const;

355
356
357
358
	#if !defined(SW_EMU)
	cl_mem_ext_ptr_t memExt_dummy;
	#endif

359
360
	// Specify Bank0 Memory for Krnl_GA & Krnl_Conform
	memExt_dockpars_conformations_current_Initial.flags = XCL_MEM_DDR_BANK0;
361
362
363
364
365
366

	memExt_dockpars_conformations_current_Final.flags   = /*XCL_MEM_DDR_BANK0*/XCL_MEM_DDR_BANK2;
	memExt_dockpars_energies_current.flags              = /*XCL_MEM_DDR_BANK0*/XCL_MEM_DDR_BANK2;
	memExt_evals_performed.flags                        = /*XCL_MEM_DDR_BANK0*/XCL_MEM_DDR_BANK2;
	memExt_gens_performed.flags                         = /*XCL_MEM_DDR_BANK0*/XCL_MEM_DDR_BANK2;

367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
	memExt_KerConstStatic_rotlist_const.flags	          = XCL_MEM_DDR_BANK0;
	memExt_KerConstStatic_ref_coords_const.flags		  = XCL_MEM_DDR_BANK0;	
	memExt_KerConstStatic_rotbonds_moving_vectors_const.flags = XCL_MEM_DDR_BANK0;			
	memExt_KerConstStatic_rotbonds_unit_vectors_const.flags   = XCL_MEM_DDR_BANK0;
	memExt_KerConstStatic_ref_orientation_quats_const.flags   = XCL_MEM_DDR_BANK0;

	// Specify Bank1 Memory for Krnl_InterE & Krnl_IntraE
	memExt_dockpars_fgrids.flags				= XCL_MEM_DDR_BANK1;	
	memExt_KerConstStatic_InterE_atom_charges_const.flags	= XCL_MEM_DDR_BANK1;
	memExt_KerConstStatic_InterE_atom_types_const.flags	= XCL_MEM_DDR_BANK1;
	memExt_KerConstStatic_IntraE_atom_charges_const.flags	= XCL_MEM_DDR_BANK1;
	memExt_KerConstStatic_IntraE_atom_types_const.flags	= XCL_MEM_DDR_BANK1;
	memExt_KerConstStatic_intraE_contributors_const.flags	= XCL_MEM_DDR_BANK1;
	memExt_KerConstStatic_reqm_const.flags			= XCL_MEM_DDR_BANK1;	
	memExt_KerConstStatic_reqm_hbond_const.flags		= XCL_MEM_DDR_BANK1;
	memExt_KerConstStatic_atom1_types_reqm_const.flags	= XCL_MEM_DDR_BANK1;
	memExt_KerConstStatic_atom2_types_reqm_const.flags	= XCL_MEM_DDR_BANK1;
	memExt_KerConstStatic_VWpars_AC_const.flags		= XCL_MEM_DDR_BANK1;
	memExt_KerConstStatic_VWpars_BD_const.flags		= XCL_MEM_DDR_BANK1;
	memExt_KerConstStatic_dspars_S_const.flags		= XCL_MEM_DDR_BANK1;
	memExt_KerConstStatic_dspars_V_const.flags		= XCL_MEM_DDR_BANK1;

389
390
391
392
	#if !defined(SW_EMU)
	memExt_dummy.flags					= XCL_MEM_DDR_BANK3;
	#endif

393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
	// Specifying the mapping from host arrays to Krnl_GA & Krnl_Conform
	memExt_dockpars_conformations_current_Initial.obj = cpu_init_populations.data();
	memExt_dockpars_conformations_current_Final.obj   = cpu_final_populations.data();
	memExt_dockpars_energies_current.obj              = cpu_energies.data();
	memExt_evals_performed.obj                        = cpu_evals_of_runs.data();
	memExt_gens_performed.obj                         = cpu_gens_of_runs.data();
	memExt_KerConstStatic_rotlist_const.obj                 = &KerConstStatic.rotlist_const[0];
	memExt_KerConstStatic_ref_coords_const.obj              = &KerConstStatic.ref_coords_const[0];	
	memExt_KerConstStatic_rotbonds_moving_vectors_const.obj = &KerConstStatic.rotbonds_moving_vectors_const[0];			
	memExt_KerConstStatic_rotbonds_unit_vectors_const.obj   = &KerConstStatic.rotbonds_unit_vectors_const[0];
	memExt_KerConstStatic_ref_orientation_quats_const.obj   = &KerConstStatic.ref_orientation_quats_const[0];

	// Specifying the mapping from host arrays to Krnl_InterE & Krnl_IntraE
	memExt_dockpars_fgrids.obj                              = cpu_floatgrids;	
	memExt_KerConstStatic_InterE_atom_charges_const.obj     = &KerConstStatic.atom_charges_const[0];
	memExt_KerConstStatic_InterE_atom_types_const.obj       = &KerConstStatic.atom_types_const[0];
	memExt_KerConstStatic_IntraE_atom_charges_const.obj     = &KerConstStatic.atom_charges_const[0];
	memExt_KerConstStatic_IntraE_atom_types_const.obj       = &KerConstStatic.atom_types_const[0];
	memExt_KerConstStatic_intraE_contributors_const.obj     = &KerConstStatic.intraE_contributors_const[0];
	memExt_KerConstStatic_reqm_const.obj                    = &KerConstStatic.reqm_const;	
	memExt_KerConstStatic_reqm_hbond_const.obj              = &KerConstStatic.reqm_hbond_const;
	memExt_KerConstStatic_atom1_types_reqm_const.obj        = &KerConstStatic.atom1_types_reqm_const;
	memExt_KerConstStatic_atom2_types_reqm_const.obj        = &KerConstStatic.atom2_types_reqm_const; 
	memExt_KerConstStatic_VWpars_AC_const.obj               = &KerConstStatic.VWpars_AC_const[0];
	memExt_KerConstStatic_VWpars_BD_const.obj               = &KerConstStatic.VWpars_BD_const[0];
	memExt_KerConstStatic_dspars_S_const.obj                = &KerConstStatic.dspars_S_const[0];
	memExt_KerConstStatic_dspars_V_const.obj                = &KerConstStatic.dspars_V_const[0];

421
422
423
424
425
426
	#if !defined(SW_EMU)
	//allocating CPU memory for dummy data (one integer)
	vector<int,aligned_allocator<int>> cpu_dummy (1);
	memExt_dummy.obj 					= cpu_dummy.data();
	#endif

427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
	// Setting param to zero
	memExt_dockpars_conformations_current_Initial.param = 0;
	memExt_dockpars_conformations_current_Final.param   = 0;
	memExt_dockpars_energies_current.param              = 0;
	memExt_evals_performed.param                        = 0;
	memExt_gens_performed.param                         = 0;
	memExt_KerConstStatic_rotlist_const.param                 = 0;
	memExt_KerConstStatic_ref_coords_const.param              = 0;	
	memExt_KerConstStatic_rotbonds_moving_vectors_const.param = 0;			
	memExt_KerConstStatic_rotbonds_unit_vectors_const.param   = 0;
	memExt_KerConstStatic_ref_orientation_quats_const.param   = 0;
	memExt_dockpars_fgrids.param                          = 0;	
	memExt_KerConstStatic_InterE_atom_charges_const.param = 0;
	memExt_KerConstStatic_InterE_atom_types_const.param   = 0;
	memExt_KerConstStatic_IntraE_atom_charges_const.param = 0;
	memExt_KerConstStatic_IntraE_atom_types_const.param   = 0;
	memExt_KerConstStatic_intraE_contributors_const.param = 0;
	memExt_KerConstStatic_reqm_const.param                = 0;	
	memExt_KerConstStatic_reqm_hbond_const.param          = 0;
	memExt_KerConstStatic_atom1_types_reqm_const.param    = 0;
	memExt_KerConstStatic_atom2_types_reqm_const.param    = 0; 
	memExt_KerConstStatic_VWpars_AC_const.param           = 0;
	memExt_KerConstStatic_VWpars_BD_const.param           = 0;
	memExt_KerConstStatic_dspars_S_const.param            = 0;
	memExt_KerConstStatic_dspars_V_const.param            = 0;

453
454
455
456
	#if !defined(SW_EMU)
	memExt_dummy.param				      = 0;
	#endif

457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
	// -----------------------------------------------------------------------------------------------------
	// Krnl_GA buffers
	cl::Buffer mem_dockpars_conformations_current_Initial
							(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX, 
							size_populations_nbytes,	&memExt_dockpars_conformations_current_Initial);
	cl::Buffer mem_dockpars_conformations_current_Final
							(context, CL_MEM_WRITE_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX, 
							size_populations_nbytes,	&memExt_dockpars_conformations_current_Final);
	cl::Buffer mem_dockpars_energies_current     	(context, CL_MEM_WRITE_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX, 
							size_energies_nbytes,     	&memExt_dockpars_energies_current);
	cl::Buffer mem_evals_performed			(context, CL_MEM_WRITE_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX, 
							size_evals_of_runs_nbytes, 	&memExt_evals_performed);
	cl::Buffer mem_gens_performed			(context, CL_MEM_WRITE_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX, 
							size_evals_of_runs_nbytes,	&memExt_gens_performed);
	// Krnl_Conform buffers
	cl::Buffer mem_KerConstStatic_rotlist_const	(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							MAX_NUM_OF_ROTATIONS*sizeof(int),	&memExt_KerConstStatic_rotlist_const);
	cl::Buffer mem_KerConstStatic_ref_coords_const	(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							MAX_NUM_OF_ATOMS*sizeof(cl_float3),	&memExt_KerConstStatic_ref_coords_const);	
	cl::Buffer mem_KerConstStatic_rotbonds_moving_vectors_const
							(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,  
							MAX_NUM_OF_ROTBONDS*sizeof(cl_float3),	&memExt_KerConstStatic_rotbonds_moving_vectors_const);
	cl::Buffer mem_KerConstStatic_rotbonds_unit_vectors_const
							(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							MAX_NUM_OF_ROTBONDS*sizeof(cl_float3),	&memExt_KerConstStatic_rotbonds_unit_vectors_const);
	cl::Buffer mem_KerConstStatic_ref_orientation_quats_const
							(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							MAX_NUM_OF_RUNS*sizeof(cl_float4),   	&memExt_KerConstStatic_ref_orientation_quats_const);
	
	// Krnl_InterE buffers
	cl::Buffer mem_dockpars_fgrids			(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							size_floatgrids_nbytes,		&memExt_dockpars_fgrids);	
	cl::Buffer mem_KerConstStatic_InterE_atom_charges_const
							(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							MAX_NUM_OF_ATOMS*sizeof(float),	&memExt_KerConstStatic_InterE_atom_charges_const);
	cl::Buffer mem_KerConstStatic_InterE_atom_types_const
							(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							MAX_NUM_OF_ATOMS*sizeof(char),	&memExt_KerConstStatic_InterE_atom_types_const);
	
	// Krnl_IntraE buffers
	cl::Buffer mem_KerConstStatic_IntraE_atom_charges_const
							(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							MAX_NUM_OF_ATOMS*sizeof(float),	&memExt_KerConstStatic_IntraE_atom_charges_const);
	cl::Buffer mem_KerConstStatic_IntraE_atom_types_const
							(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							MAX_NUM_OF_ATOMS*sizeof(char),  &memExt_KerConstStatic_IntraE_atom_types_const);
	cl::Buffer mem_KerConstStatic_intraE_contributors_const
							(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							MAX_INTRAE_CONTRIBUTORS*sizeof(cl_char3), &memExt_KerConstStatic_intraE_contributors_const);
	cl::Buffer mem_KerConstStatic_reqm_const	(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							ATYPE_NUM*sizeof(float),	&memExt_KerConstStatic_reqm_const);	
	cl::Buffer mem_KerConstStatic_reqm_hbond_const	(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							ATYPE_NUM*sizeof(float),	&memExt_KerConstStatic_reqm_hbond_const);
	cl::Buffer mem_KerConstStatic_atom1_types_reqm_const
							(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							ATYPE_NUM*sizeof(unsigned int),	&memExt_KerConstStatic_atom1_types_reqm_const);
	cl::Buffer mem_KerConstStatic_atom2_types_reqm_const
							(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							ATYPE_NUM*sizeof(unsigned int),	&memExt_KerConstStatic_atom2_types_reqm_const);
	cl::Buffer mem_KerConstStatic_VWpars_AC_const	(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							MAX_NUM_OF_ATYPES*MAX_NUM_OF_ATYPES*sizeof(float),	&memExt_KerConstStatic_VWpars_AC_const);
	cl::Buffer mem_KerConstStatic_VWpars_BD_const	(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							MAX_NUM_OF_ATYPES*MAX_NUM_OF_ATYPES*sizeof(float),	&memExt_KerConstStatic_VWpars_BD_const);
	cl::Buffer mem_KerConstStatic_dspars_S_const	(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							MAX_NUM_OF_ATYPES*sizeof(float),			&memExt_KerConstStatic_dspars_S_const);
	cl::Buffer mem_KerConstStatic_dspars_V_const	(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX,
							MAX_NUM_OF_ATYPES*sizeof(float),			&memExt_KerConstStatic_dspars_V_const);
	#if !defined(SW_EMU)
	// IMPORTANT: enable this dummy global argument only for "hw" build.
	// https://forums.xilinx.com/t5/SDAccel/ERROR-KernelCheck-83-114-in-sdx-2017-4/td-p/818135
527
528
	cl::Buffer mem_dummy				(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR | CL_MEM_EXT_PTR_XILINX, 
							sizeof(int), 						&memExt_dummy);
529
	#endif
530
	// -----------------------------------------------------------------------------------------------------
531
532
533
534
	//Separate Read/write Buffer vector is needed to migrate data between host/device
	std::vector<cl::Memory> inBufVec, outBufVec;

	// Krnl_GA
535
	inBufVec.push_back(mem_dockpars_conformations_current_Initial);
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
	// Krnl_Conform
	inBufVec.push_back(mem_KerConstStatic_rotlist_const);
	inBufVec.push_back(mem_KerConstStatic_ref_coords_const);
	inBufVec.push_back(mem_KerConstStatic_rotbonds_moving_vectors_const);
	inBufVec.push_back(mem_KerConstStatic_rotbonds_unit_vectors_const);
	inBufVec.push_back(mem_KerConstStatic_ref_orientation_quats_const);
	// Krnl_InterE
	inBufVec.push_back(mem_dockpars_fgrids);
	inBufVec.push_back(mem_KerConstStatic_InterE_atom_charges_const);
	inBufVec.push_back(mem_KerConstStatic_InterE_atom_types_const);
	// Krnl_IntraE
	inBufVec.push_back(mem_KerConstStatic_IntraE_atom_charges_const);
	inBufVec.push_back(mem_KerConstStatic_IntraE_atom_types_const);
	inBufVec.push_back(mem_KerConstStatic_intraE_contributors_const);
	inBufVec.push_back(mem_KerConstStatic_reqm_const);
	inBufVec.push_back(mem_KerConstStatic_reqm_hbond_const);
	inBufVec.push_back(mem_KerConstStatic_atom1_types_reqm_const);
	inBufVec.push_back(mem_KerConstStatic_atom2_types_reqm_const);
	inBufVec.push_back(mem_KerConstStatic_VWpars_AC_const);
	inBufVec.push_back(mem_KerConstStatic_VWpars_BD_const);
	inBufVec.push_back(mem_KerConstStatic_dspars_S_const);
	inBufVec.push_back(mem_KerConstStatic_dspars_V_const);

559
560
561
562
	#if !defined(SW_EMU)
	inBufVec.push_back(mem_dummy);
	#endif

563
	// Krnl_GA
564
565
        outBufVec.push_back(mem_dockpars_conformations_current_Final);
	outBufVec.push_back(mem_dockpars_energies_current);
566
567
	outBufVec.push_back(mem_evals_performed);
	outBufVec.push_back(mem_gens_performed);
Leonardo Solis's avatar
Leonardo Solis committed
568

569
570
571
572
573
574
575
	// -----------------------------------------------------------------------------------------------------
    	// These commands will load CPU-sources vectors from the host
   	// application and into cl::Buffer objects. 
	// The data will be be transferred from system memory 
	// over PCIe to the FPGA on-board DDR memory.
	command_queue_ga.enqueueMigrateMemObjects(inBufVec,0/* 0 means from host*/);
	// -----------------------------------------------------------------------------------------------------
lvs's avatar
lvs committed
576
577

	clock_start_docking = clock();
578
579
580
581

	int narg;
	#ifdef ENABLE_KRNL_GA
	narg = 0;
582
583
	kernel_ga.setArg(narg++, mem_dockpars_conformations_current_Initial);
	kernel_ga.setArg(narg++, mem_dockpars_conformations_current_Final);
584
585
586
587
588
589
590
591
592
593
594
595
596
	kernel_ga.setArg(narg++, mem_dockpars_energies_current);
	kernel_ga.setArg(narg++, mem_evals_performed);
	kernel_ga.setArg(narg++, mem_gens_performed);
	kernel_ga.setArg(narg++, dockpars.pop_size);
	kernel_ga.setArg(narg++, dockpars.num_of_energy_evals);
	kernel_ga.setArg(narg++, dockpars.num_of_generations);
	kernel_ga.setArg(narg++, dockpars.tournament_rate);
	kernel_ga.setArg(narg++, dockpars.mutation_rate);
	kernel_ga.setArg(narg++, dockpars.abs_max_dmov);
	kernel_ga.setArg(narg++, dockpars.abs_max_dang);
	kernel_ga.setArg(narg++, two_absmaxdmov);
	kernel_ga.setArg(narg++, two_absmaxdang);
	kernel_ga.setArg(narg++, dockpars.crossover_rate);
597
598
	kernel_ga.setArg(narg++, dockpars.num_of_genes);
	// Other kernel args are configured at every docking run
lvs's avatar
lvs committed
599
600
	#endif

601
602
603
604
605
606
607
608
609
610
	#ifdef ENABLE_KRNL_CONFORM
	narg = 0;
	kernel_conform.setArg(narg++, mem_KerConstStatic_rotlist_const);
	kernel_conform.setArg(narg++, mem_KerConstStatic_ref_coords_const);
	kernel_conform.setArg(narg++, mem_KerConstStatic_rotbonds_moving_vectors_const);
	kernel_conform.setArg(narg++, mem_KerConstStatic_rotbonds_unit_vectors_const);
	kernel_conform.setArg(narg++, dockpars.rotbondlist_length);
	kernel_conform.setArg(narg++, dockpars.num_of_atoms);
	kernel_conform.setArg(narg++, dockpars.num_of_genes);
	kernel_conform.setArg(narg++, mem_KerConstStatic_ref_orientation_quats_const);
611
	// Other kernel args are configured at every docking run
lvs's avatar
lvs committed
612
613
614
615
	#endif

	unsigned char gridsizex_minus1 = dockpars.gridsize_x - 1;
	unsigned char gridsizey_minus1 = dockpars.gridsize_y - 1;
lvs's avatar
lvs committed
616
	unsigned char gridsizez_minus1 = dockpars.gridsize_z - 1;
lvs's avatar
lvs committed
617
618
	float fgridsizex_minus1 = (float) gridsizex_minus1;
	float fgridsizey_minus1 = (float) gridsizey_minus1;
lvs's avatar
lvs committed
619
620
	float fgridsizez_minus1 = (float) gridsizez_minus1;

621
622
623
624
625
626
627
628
629
630
631
632
633
634
	#ifdef ENABLE_KRNL_INTERE
	narg = 0;
	kernel_intere.setArg(narg++, mem_dockpars_fgrids);
	kernel_intere.setArg(narg++, mem_KerConstStatic_InterE_atom_charges_const);
	kernel_intere.setArg(narg++, mem_KerConstStatic_InterE_atom_types_const);
	kernel_intere.setArg(narg++, dockpars.g1);
	kernel_intere.setArg(narg++, dockpars.g2);
	kernel_intere.setArg(narg++, dockpars.g3);
	kernel_intere.setArg(narg++, dockpars.num_of_atoms);
	kernel_intere.setArg(narg++, fgridsizex_minus1);
	kernel_intere.setArg(narg++, fgridsizey_minus1);
	kernel_intere.setArg(narg++, fgridsizez_minus1);
	kernel_intere.setArg(narg++, mul_tmp2);
	kernel_intere.setArg(narg++, mul_tmp3);
lvs's avatar
lvs committed
635
	#endif
lvs's avatar
lvs committed
636

637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
	#ifdef ENABLE_KRNL_INTRAE
	narg = 0;
	kernel_intrae.setArg(narg++, mem_KerConstStatic_IntraE_atom_charges_const);
	kernel_intrae.setArg(narg++, mem_KerConstStatic_IntraE_atom_types_const);
	kernel_intrae.setArg(narg++, mem_KerConstStatic_intraE_contributors_const);
	kernel_intrae.setArg(narg++, dockpars.smooth);
	kernel_intrae.setArg(narg++, mem_KerConstStatic_reqm_const);
	kernel_intrae.setArg(narg++, mem_KerConstStatic_reqm_hbond_const);
	kernel_intrae.setArg(narg++, mem_KerConstStatic_atom1_types_reqm_const);
	kernel_intrae.setArg(narg++, mem_KerConstStatic_atom2_types_reqm_const);
	kernel_intrae.setArg(narg++, mem_KerConstStatic_VWpars_AC_const);
	kernel_intrae.setArg(narg++, mem_KerConstStatic_VWpars_BD_const);
	kernel_intrae.setArg(narg++, mem_KerConstStatic_dspars_S_const);
	kernel_intrae.setArg(narg++, mem_KerConstStatic_dspars_V_const);
	kernel_intrae.setArg(narg++, dockpars.num_of_atoms);
	kernel_intrae.setArg(narg++, dockpars.num_of_intraE_contributors);
	kernel_intrae.setArg(narg++, dockpars.grid_spacing);
	kernel_intrae.setArg(narg++, dockpars.num_of_atypes);
	kernel_intrae.setArg(narg++, dockpars.coeff_elec);
	kernel_intrae.setArg(narg++, dockpars.qasp);
	kernel_intrae.setArg(narg++, dockpars.coeff_desolv);
	#endif

660
661
662
	#ifdef ENABLE_KRNL_PRNG_BT_USHORT_FLOAT
	// Other kernel args are configured at every docking run
	kernel_prng_bt_ushort_float.setArg(2, dockpars.pop_size);
663
	#if !defined(SW_EMU)
664
	kernel_prng_bt_ushort_float.setArg(3, mem_dummy);
665
	#endif
666
	#endif
667

668
669
670
	#ifdef ENABLE_KRNL_PRNG_GG_UCHAR
	// Other kernel args are configured at every docking run
	kernel_prng_gg_uchar.setArg(1, dockpars.num_of_genes);
671
	#if !defined(SW_EMU)
672
	kernel_prng_gg_uchar.setArg(2, mem_dummy);
673
	#endif
674
	#endif
lvs's avatar
lvs committed
675

676
677
678
	#ifdef ENABLE_KRNL_PRNG_GG_FLOAT
	// Other kernel args are configured at every docking run
	kernel_prng_gg_float.setArg(1, dockpars.num_of_genes);
Leonardo Solis's avatar
Leonardo Solis committed
679
	#if !defined(SW_EMU)
680
	kernel_prng_gg_float.setArg(2, mem_dummy);
Leonardo Solis's avatar
Leonardo Solis committed
681
	#endif
682
	#endif
683

684
	#ifdef ENABLE_KRNL_IGL_ARBITER
685
	#if !defined(SW_EMU)
686
687
	kernel_igl_arbiter.setArg(0, mem_dummy);
	#endif
688
	#endif
lvs's avatar
lvs committed
689

690
	printf("Docking runs to be executed: %lu\n", mypars->num_of_runs); 
lvs's avatar
lvs committed
691
	printf("Execution run: ");
lvs's avatar
lvs committed
692

693
	for (unsigned int run_cnt = 0; run_cnt < mypars->num_of_runs; run_cnt++) {
lvs's avatar
lvs committed
694
		printf(" %u", run_cnt+1); 
lvs's avatar
lvs committed
695
		fflush(stdout);
lvs's avatar
lvs committed
696

697
698
699
700
		#ifdef ENABLE_KRNL_GA
		unsigned short ushort_run_cnt  = (unsigned ushort) run_cnt;
		unsigned int   Host_Offset_Pop = run_cnt * dockpars.pop_size * ACTUAL_GENOTYPE_LENGTH;
		unsigned int   Host_Offset_Ene = run_cnt * dockpars.pop_size;
701
702
703
		kernel_ga.setArg(16, ushort_run_cnt);
		kernel_ga.setArg(17, Host_Offset_Pop);
		kernel_ga.setArg(18, Host_Offset_Ene);
lvs's avatar
lvs committed
704
		#endif
705

706
		#ifdef ENABLE_KRNL_CONFORM
Leonardo Solis's avatar
Leonardo Solis committed
707
		kernel_conform.setArg(8, ushort_run_cnt);
708
		#endif
709

710
		#ifdef ENABLE_KRNL_PRNG_BT_USHORT_FLOAT
711
712
		kernel_prng_bt_ushort_float.setArg(0, cpu_prng_seeds[num_of_prng_blocks * run_cnt]);
		kernel_prng_bt_ushort_float.setArg(1, cpu_prng_seeds[num_of_prng_blocks * run_cnt + 1]);
713
		#endif
714

715
716
717
		#ifdef ENABLE_KRNL_PRNG_GG_UCHAR
		kernel_prng_gg_uchar.setArg(0, cpu_prng_seeds[num_of_prng_blocks * run_cnt + 2]);
		#endif
718

719
		#ifdef ENABLE_KRNL_PRNG_GG_FLOAT
720
		kernel_prng_gg_float.setArg(0, cpu_prng_seeds[num_of_prng_blocks * run_cnt + 3]);
721
		#endif
lvs's avatar
lvs committed
722

723
		#ifdef ENABLE_KRNL_GA
724
		command_queue_ga.enqueueTask(kernel_ga);
725
		#endif
726
727
		#ifdef ENABLE_KRNL_CONFORM
		command_queue_conform.enqueueTask(kernel_conform);
728
		#endif
729
730
		#ifdef ENABLE_KRNL_INTERE
		command_queue_intere.enqueueTask(kernel_intere);
731
		#endif
732
733
		#ifdef ENABLE_KRNL_INTRAE
		command_queue_intrae.enqueueTask(kernel_intrae);
734
		#endif
735
736
		#ifdef ENABLE_KRNL_PRNG_BT_USHORT_FLOAT
		command_queue_prng_bt_ushort_float.enqueueTask(kernel_prng_bt_ushort_float);
737
		#endif
738
739
740
741
742
		#ifdef ENABLE_KRNL_PRNG_GG_UCHAR
		command_queue_prng_gg_uchar.enqueueTask(kernel_prng_gg_uchar);
		#endif
		#ifdef ENABLE_KRNL_PRNG_GG_FLOAT
		command_queue_prng_gg_float.enqueueTask(kernel_prng_gg_float);
743
		#endif
744
745
		#ifdef ENABLE_KRNL_IGL_ARBITER
		command_queue_igl_arbiter.enqueueTask(kernel_igl_arbiter);
746
		#endif
lvs's avatar
lvs committed
747

748
		#ifdef ENABLE_KRNL_GA	
749
		command_queue_ga.finish(); 
lvs's avatar
lvs committed
750
		#endif
751
		#ifdef ENABLE_KRNL_CONFORM	
752
		command_queue_conform.finish(); 
lvs's avatar
lvs committed
753
		#endif
754
		#ifdef ENABLE_KRNL_INTERE	
755
		command_queue_intere.finish(); 
lvs's avatar
lvs committed
756
		#endif
757
		#ifdef ENABLE_KRNL_INTRAE	
758
		command_queue_intrae.finish(); 
759
760
		#endif
		#ifdef ENABLE_KRNL_PRNG_BT_USHORT_FLOAT
761
		command_queue_prng_bt_ushort_float.finish();
762
763
		#endif
		#ifdef ENABLE_KRNL_PRNG_GG_UCHAR
764
		command_queue_prng_gg_uchar.finish();
lvs's avatar
lvs committed
765
		#endif
766
		#ifdef ENABLE_KRNL_PRNG_GG_FLOAT
767
768
		command_queue_prng_gg_float.finish(); 
		#endif
769
		#ifdef ENABLE_KRNL_IGL_ARBITER
770
		command_queue_igl_arbiter.finish();
lvs's avatar
lvs committed
771
		#endif
lvs's avatar
lvs committed
772

773
774
		clock_stop_docking = clock();
	} // End of for (unsigned int run_cnt = 0; run_cnt < mypars->num_of_runs; run_cnt++)
lvs's avatar
lvs committed
775

776
777
778
779
780
781
782
783
784
	printf("\n");
	fflush(stdout);

	// -----------------------------------------------------------------------------------------------------
    	// These commands will load CPU-sources vectors from the host
   	// application and into cl::Buffer objects. 
	// The data will be be transferred from system memory 
	// over PCIe to the FPGA on-board DDR memory.
	command_queue_ga.enqueueMigrateMemObjects(outBufVec,CL_MIGRATE_MEM_OBJECT_HOST);
785
786
787
788
789

	// Making sure all data was transferred back to host
	#ifdef ENABLE_KRNL_GA	
	command_queue_ga.finish(); 
	#endif
790
	// -----------------------------------------------------------------------------------------------------
lvs's avatar
lvs committed
791
792
793

	for (unsigned int run_cnt = 0; run_cnt < mypars->num_of_runs; run_cnt++) {

794
795
		arrange_result(cpu_final_populations.data() + run_cnt*mypars->pop_size*ACTUAL_GENOTYPE_LENGTH, 
			       cpu_energies.data()          + run_cnt*mypars->pop_size, 
796
797
798
			       mypars->pop_size);

		/*printf("cpu_evals_of_runs[%u]: %u\n", run_cnt, cpu_evals_of_runs[run_cnt]);*/
lvs's avatar
lvs committed
799

800
801
		make_resfiles(cpu_final_populations.data() + run_cnt*mypars->pop_size*ACTUAL_GENOTYPE_LENGTH, 
			      cpu_energies.data()          + run_cnt*mypars->pop_size, 
lvs's avatar
lvs committed
802
803
804
805
806
807
			      &myligand_reference,
			      myligand_init, 
			      mypars, 
   			      cpu_evals_of_runs[run_cnt], 
			      cpu_gens_of_runs[run_cnt], /*generation_cnt, */
			      mygrid, 
808
809
			      cpu_floatgrids,
			      cpu_ref_ori_angles.data() + 3*run_cnt, 
lvs's avatar
lvs committed
810
811
812
813
814
			      argc, 
			      argv, 
			      0,
			      run_cnt, 
                              &(cpu_result_ligands [run_cnt]));
815
	} // End of for (unsigned int run_cnt = 0; run_cnt < mypars->num_of_runs; run_cnt++) 
lvs's avatar
lvs committed
816

817
	/*
lvs's avatar
lvs committed
818
819
820
821
822
	for (int cnt_pop=0;cnt_pop<size_populations/sizeof(float);cnt_pop++)
		printf("total_num_pop: %u, cpu_final_populations[%u]: %f\n",(unsigned int)(size_populations/sizeof(float)),cnt_pop,cpu_final_populations[cnt_pop]);

	for (int cnt_pop=0;cnt_pop<size_energies/sizeof(float);cnt_pop++)
		printf("total_num_energies: %u, cpu_energies[%u]: %f\n",    (unsigned int)(size_energies/sizeof(float)),cnt_pop,cpu_energies[cnt_pop]);
823
	*/
lvs's avatar
lvs committed
824
825
826
827

	clock_stop_program_before_clustering = clock();


828
	clusanal_gendlg(cpu_result_ligands.data(), 
lvs's avatar
lvs committed
829
830
831
832
833
834
835
836
837
838
839
			mypars->num_of_runs,
			myligand_init, mypars,
   		        mygrid, 
			argc,
			argv,
			ELAPSEDSECS(clock_stop_docking, clock_start_docking)/mypars->num_of_runs,
			ELAPSEDSECS(clock_stop_program_before_clustering, clock_start_program));

	clock_stop_docking = clock();

	return 0;
840
}