ADD: scripts2 folder on how to generate jobs files (and merge cases) for the 2D SIM+ CMV story (cbf227c2) · Commits · Cem Emre Akbas / annotations_merging

scripts2/Fluo-N2DH-SIM+-01.cmd

0 → 100644

+21 −0

Original line number	Diff line number	Diff line
		python3 \
		/home/ulman/data/silverGT_SIMdata/scripts2/prepareGivenVideo_intoParts_zipped.py \
		"$1/Fiji.app/ImageJ-linux64" "$2" \
		Fluo-N2DH-SIM+-01 \
		0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,64 \
		"$1/gtReference/Fluo-N2DH-SIM+/01_GT" \
		3 \
		"$1/userResults/Fluo-N2DH-SIM+/fixed/BGU-IL_(2)/01_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/fixed/CVUT-CZ/01_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/fixed/MU-Lux-CZ/01_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/BGU-IL_(3)/01_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/BGU-IL_(4)/01_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/FR-Ro-GE/01_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/HD-Hau-GE/01_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/HD-Wol-GE/01_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/HIT-CN/01_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/KTH-SE_(1)/01_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/LEID-NL/01_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/PAST-FR/01_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/TUG-AT/01_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/UZH-CH/01_RES"

scripts2/Fluo-N2DH-SIM+-02.cmd

0 → 100644

+21 −0

Original line number	Diff line number	Diff line
		python3 \
		/home/ulman/data/silverGT_SIMdata/scripts2/prepareGivenVideo_intoParts_zipped.py \
		"$1/Fiji.app/ImageJ-linux64" "$2" \
		Fluo-N2DH-SIM+-02 \
		0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,149 \
		"$1/gtReference/Fluo-N2DH-SIM+/02_GT" \
		3 \
		"$1/userResults/Fluo-N2DH-SIM+/fixed/BGU-IL_(2)/02_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/fixed/CVUT-CZ/02_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/fixed/MU-Lux-CZ/02_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/BGU-IL_(3)/02_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/BGU-IL_(4)/02_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/FR-Ro-GE/02_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/HD-Hau-GE/02_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/HD-Wol-GE/02_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/HIT-CN/02_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/KTH-SE_(1)/02_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/LEID-NL/02_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/PAST-FR/02_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/TUG-AT/02_RES" \
		"$1/userResults/Fluo-N2DH-SIM+/UZH-CH/02_RES"

scripts2/howToPrepareJobs.txt

0 → 100644

+24 −0

Original line number	Diff line number	Diff line
		# temp folder where merge cases (.zip file jobs) and PBS scripts
		mkdir /temp/silverGT

		# create the .zip jobs files
		# 1st param: where _startingPackage.zip will be unpacked
		# 2nd param: where job .zip files shall be created
		. Fluo-N2DH-SIM+-01.cmd /tmp/silverGT_ulman /temp/silverGT
		. Fluo-N2DH-SIM+-02.cmd /tmp/silverGT_ulman /temp/silverGT

		# create the associated .sh jobs that shall be just executed as: qsub $jobFILE.sh
		cd /temp/silverGT
		mkdir jobs
		cd jobs
		echo "customize template for 01 video"
		for i in ../Fluo-N2DH-SIM+-01__job*zip; do /home/ulman/data/silverGT_SIMdata/scripts2/jobParallelScriptTEMPLATE.sh $i; done
		echo "customize template for 02 video"
		for i in ../Fluo-N2DH-SIM+-02__job*zip; do /home/ulman/data/silverGT_SIMdata/scripts2/jobParallelScriptTEMPLATE.sh $i; done

		# it4i site has to be prepared:
		# /scratch/temp/silverGT/ - here are the _startinPackage.zip files
		# /scratch/temp/silverGT/in/ - here are the job files
		# /scratch/temp/silverGT/out/ - here results will be cumulated
		#
		# upload the .zip files into it4i:/scratch/temp/silverGT/in/

scripts2/jobParallelProcessOneMerge.sh

0 → 100755

+17 −0

Original line number	Diff line number	Diff line
		#!/bin/bash

		# 11hrs and 40 mins is 700 mins
		# if current time is 700+ min later than the 'tasklist' file was created, we skip this job
		if [ $( find tasklist -cmin +700 \| wc -l ) -eq 1 ]; then
		touch "notComplete"
		exit 0
		fi

		# enter the job folder and execute it
		cd $1
		if [ ! -e log_2_outputs.txt ]; then
		date > log_2_outputs.txt
		head -n 3 /proc/meminfo >> log_2_outputs.txt
		source log_1_commands.txt $PWD >> log_2_outputs.txt 2> /dev/null
		rm mask*.tif
		fi

scripts2/jobParallelScriptTEMPLATE.sh

0 → 100755

+42 −0

Original line number	Diff line number	Diff line
		# give it a .zip with task batch, it will create a temporary file and fire qsub

		zipFile=`basename $1`
		jobName=`basename $zipFile .zip`
		jobFile=jobParallelScript_${jobName}.sh

		echo "#!/bin/bash" > $jobFile
		echo "#PBS -A OPEN-19-3" >> $jobFile
		echo "#PBS -q qprod" >> $jobFile
		echo "#PBS -l select=1:ncpus=24,walltime=5:30:00" >> $jobFile
		echo "#PBS -N $jobName" >> $jobFile
		echo "" >> $jobFile
		echo "# setup env" >> $jobFile
		echo "module add Python/3.5.2" >> $jobFile
		echo "module add parallel" >> $jobFile
		echo 'echo "starting @ "`date`' >> $jobFile
		echo "" >> $jobFile
		echo "# prepare input folder" >> $jobFile
		echo "cd /tmp; unzip /scratch/temp/silverGT/sim02_startingPackage.zip > /dev/null" >> $jobFile
		echo 'echo "package unzipping done @ "`date`' >> $jobFile
		echo "" >> $jobFile
		echo "# prepare tasks folder, and stay in this folder" >> $jobFile
		echo "cd /ramdisk/\$PBS_JOBID; unzip /scratch/temp/silverGT/in/$zipFile > /dev/null" >> $jobFile
		echo 'echo "jobs unzipping done @ "`date`' >> $jobFile
		echo "" >> $jobFile
		echo "# look what needs to be computed, and compute it..." >> $jobFile
		echo "rm -f tasklist notComplete" >> $jobFile
		echo "ls -d mer* > tasklist" >> $jobFile
		echo "parallel -a tasklist /tmp/silverGT_ulman/jobParallelProcessOneMerge.sh" >> $jobFile
		echo 'echo "parallel done @ "`date`' >> $jobFile
		echo "test -e notComplete && echo \"this job is not complete\" \|\| echo \"this job is complete\"" >> $jobFile
		echo "" >> $jobFile
		echo "# collect results (and suppress listing of visited files)" >> $jobFile
		echo "zip -r /tmp/$zipFile . > /dev/null" >> $jobFile
		echo "mv /tmp/$zipFile /scratch/temp/silverGT/out/$zipFile" >> $jobFile
		echo 'echo "jobs zipping and moving done @ "`date`' >> $jobFile
		echo "" >> $jobFile
		echo "# clean up the /tmp" >> $jobFile
		echo "rm -r /tmp/silverGT_ulman" >> $jobFile
		echo 'echo "clean up done; stopping @ "`date`' >> $jobFile

		#qsub $jobFile