Proactively monitor simulation time within NEURON

// CheckTime NEURON code for ending a simulation
// within the time limit for the supercomputer
// Written by Marianne Bezaire (marianne.bezaire@gmail.com)
// November 2013

// put this stopwatch call at the very beginning of your code:
simstart = startsw()

// set some parameters to control how often your program checks, how much time it has, etc:
JobHours = 2 // UNIT: HOURS. This is the number of hours that
			 // you setin the NSG task, the hard time limit
			 // of how long your code can run (up to 8 hours
			 // for the NSG currently)
StepBy = 100 // UNIT: MILLISECONDS. This is the number of ms
			 // that your code will advance before checking how
			 // much time is left. Every StepBy ms, it will check
			 // again to see if it has time to run another StepBy ms
			 // of simulation time
EstWriteTime = 600  // UNIT: SECONDS. This is the number of seconds
					// that you want your code to reserve for writing
					// results at the end of the simulation. You may
					// want to set this high at first and time how long
					// it usually takes you before setting it lower. I
					// usually leave it set to 10 - 20 minutes (600-1200 s)
					// for my code.

// all of your setup code goes here:
// ...
// ...
// then put this code before your run() statement
objref fihw
fihw = new FInitializeHandler(2, "midbal()")
walltime = startsw() // this times when each segment of your simulation starts,
					 // to track how long each StepBy ms takes to run
strdef cmd, cmdo
newtstop = tstop // if the code will stop early, this is the estimated new stop
				 // time, only computed to print out a warning message
warningflag=0 // Only compute the estimated new tstop once
proc midbal() {local wt, thisstep, simleft, compleft
	wt = startsw()
	if (t>0) {
		thisstep = wt - walltime // compute how long this StepBy ms segment took to run
		simleft = tstop - t // compute how much simulation time remains
		compleft = JobHours*3600 - (startsw() - loadstart) // compute how much real time is left to complete the simulation
		//print "t=",t,", tstop=",tstop,", simleft=",simleft,", jobtime=",JobHours*3600,", timeused=",startsw() - loadstart,", compleft=", compleft // print statement for debugging
		if (warningflag==0 && (simleft/StepBy*thisstep+EstWriteTime)>compleft && pc.id == 0) {  // only on one pc, and only if a warning hasn't been printed before, check whether
																								// there is enough time to run all of the remaining simulation segments AND write results
																								// If there is not enough time, print out a warning message to stdout that the simulation
																								// will likely stop early.
			newtstop = int((compleft-EstWriteTime)/thisstep)*StepBy + t // compute how much simulation time will likely fit into the real time left on the supercomputer, based on how
																		// long the previous StepBy ms segment took to complete.
			print "Not enough time to complete ", tstop,  " ms simulation, simulation will likely stop around ", newtstop, " ms"
			warningflag=1 // only write the warning statement once.
		}
		if (pc.id == 0) { printf("%g ms interval at t=%g ms was %g sn", StepBy, t, thisstep) } // Each StepBy ms simulation segment, print how much real time it took on Host 0.

		if ((thisstep+EstWriteTime)>compleft && t<tstop) { // If not enough time for another time step and writing results, end now
			{pc.barrier()} // wait till all processors get to this point
			if (pc.id == 0) { print "simulation stopped early at t=", t, " ms"} // print a notification to stdout that the simulation is ending
			tstop = t // set tstop to the current simulation time
			stoprun=1 // you must also set the stoprun flag to 1 to get the run to complete right now
		}
	}
	walltime = wt // reset the counter for the next StepBy ms simulation segment
	cvode.event(t+StepBy, "midbal(0)") // run this function again at the end of the next StepBy ms segment
}
// ....
run() // Then call your run statement at some point later
// ...

Leave a Reply

Your email address will not be published. Required fields are marked *