<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:Consolas;
panose-1:2 11 6 9 2 2 4 3 2 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Times New Roman",serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:blue;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:purple;
text-decoration:underline;}
pre
{mso-style-priority:99;
mso-style-link:"HTML Preformatted Char";
margin:0in;
margin-bottom:.0001pt;
font-size:10.0pt;
font-family:"Courier New";}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-priority:34;
margin-top:0in;
margin-right:0in;
margin-bottom:0in;
margin-left:.5in;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Times New Roman",serif;}
p.msonormal0, li.msonormal0, div.msonormal0
{mso-style-name:msonormal;
mso-margin-top-alt:auto;
margin-right:0in;
mso-margin-bottom-alt:auto;
margin-left:0in;
font-size:12.0pt;
font-family:"Times New Roman",serif;}
span.apple-converted-space
{mso-style-name:apple-converted-space;}
span.HTMLPreformattedChar
{mso-style-name:"HTML Preformatted Char";
mso-style-priority:99;
mso-style-link:"HTML Preformatted";
font-family:Consolas;}
span.EmailStyle21
{mso-style-type:personal-reply;
font-family:"Calibri",sans-serif;
color:#1F497D;}
span.keyword
{mso-style-name:keyword;}
span.comment
{mso-style-name:comment;}
span.comment-delimiter
{mso-style-name:comment-delimiter;}
span.variable-name
{mso-style-name:variable-name;}
span.string
{mso-style-name:string;}
span.builtin
{mso-style-name:builtin;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;}
@page WordSection1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:1058942466;
mso-list-type:hybrid;
mso-list-template-ids:-1441656350 67698703 67698713 67698715 67698703 67698713 67698715 67698703 67698713 67698715;}
@list l0:level1
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level2
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level3
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l0:level4
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level5
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level6
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l0:level7
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level8
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level9
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l1
{mso-list-id:1380592448;
mso-list-type:hybrid;
mso-list-template-ids:-2127915720 67698703 67698713 67698715 67698703 67698713 67698715 67698703 67698713 67698715;}
@list l1:level1
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l1:level2
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l1:level3
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l1:level4
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l1:level5
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l1:level6
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l1:level7
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l1:level8
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l1:level9
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l2
{mso-list-id:1734356235;
mso-list-type:hybrid;
mso-list-template-ids:954996694 67698703 67698713 67698715 67698703 67698713 67698715 67698703 67698713 67698715;}
@list l2:level1
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l2:level2
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l2:level3
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l2:level4
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l2:level5
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l2:level6
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l2:level7
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l2:level8
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l2:level9
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l3
{mso-list-id:2060282247;
mso-list-type:hybrid;
mso-list-template-ids:121426340 67698703 67698713 67698715 67698703 67698713 67698715 67698703 67698713 67698715;}
@list l3:level1
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l3:level2
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l3:level3
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l3:level4
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l3:level5
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l3:level6
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l3:level7
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l3:level8
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l3:level9
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
ol
{margin-bottom:0in;}
ul
{margin-bottom:0in;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang="EN-US" link="blue" vlink="purple">
<div class="WordSection1">
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Hi All,<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">I’m using the GATE docker container with HT condor, and want to pass along how I’m managing it. It works very well for a condor cluster with completely independent
nodes, with no need to share users, disks, or executables. The sequence I’m using goes like this:<o:p></o:p></span></p>
<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l3 level1 lfo4"><![if !supportLists]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><span style="mso-list:Ignore">1.<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Create GATE macros, using aliases for anything you want to vary between jobs, including the simulation start and stop times<o:p></o:p></span></p>
<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l3 level1 lfo4"><![if !supportLists]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><span style="mso-list:Ignore">2.<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Write a shell script that will run the gate job inside the container. This script will<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in;mso-list:l3 level2 lfo4">
<![if !supportLists]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><span style="mso-list:Ignore">a.<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Perform any configuration needed to run GATE<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in;mso-list:l3 level2 lfo4">
<![if !supportLists]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><span style="mso-list:Ignore">b.<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Run GATE with the –a option to set aliases<o:p></o:p></span></p>
<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l3 level1 lfo4"><![if !supportLists]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><span style="mso-list:Ignore">3.<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Write one or more condor job files (*.submit) that<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in;mso-list:l3 level2 lfo4">
<![if !supportLists]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><span style="mso-list:Ignore">a.<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Specifies the condor docker universe and docker image<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in;mso-list:l3 level2 lfo4">
<![if !supportLists]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><span style="mso-list:Ignore">b.<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Defines the output, log and error files for the job<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in;mso-list:l3 level2 lfo4">
<![if !supportLists]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><span style="mso-list:Ignore">c.<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Defines the files to be transferred into the container before running the gate job<o:p></o:p></span></p>
<p class="MsoListParagraph" style="margin-left:1.0in;text-indent:-.25in;mso-list:l3 level2 lfo4">
<![if !supportLists]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><span style="mso-list:Ignore">d.<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Defines the files to be transferred out of the container after running the gate job<o:p></o:p></span></p>
<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l3 level1 lfo4"><![if !supportLists]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><span style="mso-list:Ignore">4.<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Submit the jobs to condor<o:p></o:p></span></p>
<p class="MsoListParagraph" style="text-indent:-.25in;mso-list:l3 level1 lfo4"><![if !supportLists]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><span style="mso-list:Ignore">5.<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Condor handles transferring files in and out of the container, and running the jobs<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">I automate this with a python script that writes the shell scripts, the job files and submits condor jobs. Such python serve the purpose of the GATE job splitter,
gjs, plus allow for additional control if there is more than start and stop times that you’d like to vary among jobs, be it phantom geometry or activity or whatever you think up.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><u><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Example shell script that will run the gate job inside the container</span></u><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">
(split000.sh):<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">#!/usr/bin/bash --login<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">echo running $0 on `hostname` at `date`<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">cd `dirname $0`<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">echo pwd = `pwd`<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">source setup_gate_container_env.sh<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">Gate –a [SimulationStartTime,0.0][SimulationStopTime,1[OutputBaseName,simulation-split000] simulation.mac<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><u><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Example condor job file (split000.submit):<o:p></o:p></span></u></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">universe = docker<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">executable = split000.sh<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">docker_image = opengatecollaboration/gate<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">output = simulation-split000.out<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">error = simulation-split000.err<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">log = simulation-split000.log<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">should_transfer_files = YES<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">when_to_transfer_output = ON_EXIT<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">transfer_input_files = <comma-separated list of all files needed><o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">transfer_output_files = split000.root<o:p></o:p></span></p>
<div>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">queue<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><o:p> </o:p></span></p>
</div>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Transfer_input_files is a comma-separated list of everything needed to run the macro simulation.mac. It must include all macros that you might /control/execute,
the materials database file, and other input files, including setup_gate_container_env.sh which is sourced by the shell script.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Transfer_output_files is a comma-separated list of everything you’d like to have copied out of the container after the script has completed. Here it is just
a split000.root, but it could include other output files or visualization results.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">After submitting to condor with condor_submit split000.submit, the job runs and all the file copying in and out of container is handled by condor.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Now, I don’t create these scripts and submit files by hand – I use a python script that looks like the following (sumit-splits.py):<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">#!/usr/bin/env python<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">import glob<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">import os<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">import stat<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">import subprocess<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">import time<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"># Simulation start time [s]<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">start_time = 0.0<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"># Simulation stop time [s]<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">stop_time = 300.0<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"># Number of jobs to submit to condor<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">Nsplits = 600<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">split_duration = (stop_time - start_time)/Nsplits<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">basename = ' simulation'<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">GATEMacro = basename + '.mac'<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D">for i in range(Nsplits):<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> t1 = i*split_duration<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> t2 = t1 + split_duration<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> print "Writing scripts for split #", i, 'of', Nsplits<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> OutputBaseName = basename + '-split' + str(i).zfill(len(str(Nsplits)))<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> script_file = OutputBaseName + '.sh'<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> submit_file = OutputBaseName + '.submit'<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> log_file = OutputBaseName + '.log'<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> output_file = OutputBaseName + '.out'<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> error_file = OutputBaseName + '.err'<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> # Write the script that will be run in the docker container:<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> print "Writing", script_file<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f = open(script_file, 'w')<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("#!/usr/bin/bash --login\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("echo running $0 on `hostname` at `date`\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("cd `dirname $0`\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("echo pwd = `pwd`\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("source setup_gate_container_env.sh\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("Gate -a ")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("[SimulationStartTime," + str(t1) + "]")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("[SimulationStopTime," + str(t2) + "]")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("[OutputBaseName," + OutputBaseName + "]")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write(" " + GATEMacro + "\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.close()<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> # Set permissions so that condor can execute the script:<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> stats = os.stat(script_file)<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> os.chmod(script_file, stats.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> # Make a list of macros to be transfered for the condor jobs.<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> # Since condor's transfer_input_files cannot handle wild cards,<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> # we'll create a comma-separated list here:<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> input_files = []<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> input_files.append('GateMaterials.db')<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> input_files.append('setup_gate_container_env.sh')<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> for macro_file in glob.glob('*.mac'):<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> input_files.append(macro_file)<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> # Write the condor submit file:<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> print "Writing", submit_file<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f = open(submit_file, 'w')<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("universe = docker\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("executable = " + script_file + "\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("docker_image = opengatecollaboration/gate\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("output = " + output_file + "\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("error = " + error_file + "\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("log = " + log_file + "\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("should_transfer_files = YES\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("when_to_transfer_output = ON_EXIT\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("transfer_input_files = " + ','.join(input_files) + "\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("transfer_output_files = " + OutputBaseName + ".root\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.write("queue\n")<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> f.close()<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> # Submit the job to condor:<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> print "Submitting", submit_file<o:p></o:p></span></p>
<p class="MsoNormal" style="margin-left:.5in"><span style="font-size:9.0pt;font-family:Consolas;color:#1F497D"> subprocess.call(["condor_submit", submit_file])<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Hope some of you find this helpful.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D">Mike<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif;color:#1F497D"><o:p> </o:p></span></p>
</div>
<br>
<hr>
<font face="Arial" color="Gray" size="1">The information contained in this message may be confidential and legally protected under applicable law. The message is intended solely for the addressee(s). If you are not the intended recipient, you are hereby notified
that any use, forwarding, dissemination, or reproduction of this message is strictly prohibited and may be unlawful. If you are not the intended recipient, please contact the sender by return e-mail and destroy all copies of the original message.<br>
</font>
</body>
</html>