1 /**************************************************************************
2 * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
17 // AliMuonAccEffSubmitter : a class to help submit Acc x Eff simulations
18 // anchored to real runs for J/psi, upsilon, single muons, etc...
20 // This class is dealing with 3 different directories :
22 // - template directory ($ALICE_ROOT/PWG/muondep/AccEffTemplates) containing the
23 // basic template files to be used for a simuation. A template can contain
24 // some variables that will be replaced during during the copy from template
27 // - local directory, where the files from the template directory, are copied
28 // once the class has been configured properly (i.e. using the various Set, Use,
29 // etc... methods). Some other files (e.g. JDL ones) are generated from
30 // scratch and also copied into this directory.
31 // At this point one could(should) check the files, as they are the ones
32 // to be copied to the remote directory for the production
34 // - remote directory, the alien directory where the files will be copied
35 // (from the local directory) before the actual submission
37 // ==========================================================
41 // AliMuonAccEffSubmitter a; // (1)
42 // a.UseOCDBSnapshots(kFALSE);
43 // a.SetRemoteDir("/alice/cern.ch/user/l/laphecet/Analysis/LHC13d/simjpsi/pp503z0");
44 // a.ShouldOverwriteFiles(true);
45 // a.MakeNofEventsPropToTriggerCount("CMUL7-B-NOPF-MUON");
46 // a.SetVar("VAR_GENLIB_PARNAME","\"pp 5.03\"");
47 // a.SetRunList(195682);
49 // a.Run("test"); // will do everything but the submit
50 // a.Submit(false); // actual submission
52 // (1) note that for this to work in the Root prompt you need to load (for instance
53 // in your rootlogon.C) all the chain of libraries up to libPWGmuondep. As
54 // the time of this writing (March 2014), this means :
56 // gSystem->Load("libVMC");
57 // gSystem->Load("libTree");
58 // gSystem->Load("libProofPlayer");
59 // gSystem->Load("libPhysics");
60 // gSystem->Load("libMatrix");
61 // gSystem->Load("libMinuit");
62 // gSystem->Load("libXMLParser");
63 // gSystem->Load("libGui");
64 // gSystem->Load("libSTEERBase");
65 // gSystem->Load("libESD");
66 // gSystem->Load("libAOD");
67 // gSystem->Load("libANALYSIS");
68 // gSystem->Load("libRAWDatabase");
69 // gSystem->Load("libCDB");
70 // gSystem->Load("libSTEER");
71 // gSystem->Load("libANALYSISalice");
72 // gSystem->Load("libCORRFW");
73 // gSystem->Load("libPWGmuon");
74 // gSystem->Load("libMUONcore");
75 // gSystem->Load("libMUONmapping");
76 // gSystem->Load("libMUONcalib");
77 // gSystem->Load("libMUONgeometry");
78 // gSystem->Load("libMUONtrigger");
79 // gSystem->Load("libRAWDatabase");
80 // gSystem->Load("libMUONraw");
81 // gSystem->Load("libMUONbase");
82 // gSystem->Load("libMUONshuttle");
83 // gSystem->Load("libMUONrec");
84 // gSystem->Load("libPWGmuondep");
86 // author: Laurent Aphecetche (Subatech)
89 #include "AliMuonAccEffSubmitter.h"
91 #include "AliAnalysisTriggerScalers.h"
95 #include "TGridResult.h"
98 #include "TObjString.h"
110 ClassImp(AliMuonAccEffSubmitter)
112 //______________________________________________________________________________
113 AliMuonAccEffSubmitter::AliMuonAccEffSubmitter(const char* generator, Bool_t localOnly,
114 const char* generatorVersion)
115 : AliMuonGridSubmitter(AliMuonGridSubmitter::kAccEff,localOnly),
117 fFixedNofEvents(10000),
118 fMaxEventsPerChunk(5000),
120 fSplitMaxInputFileNumber(20),
123 fUseOCDBSnapshots(kFALSE),
125 fUseAODMerging(kFALSE)
129 // if generator contains "pythia8" and generatorVersion is given then
130 // the pythiaversion must represent the integer part XXX of the
131 // include directory $ALICE_ROOT/PYTHI8/pythiaXXX/include where the file
132 // Analysis.h is to be found.
134 // if generator contains "pythia6" then generatorVersion should be the
135 // X.YY part of libpythia6.X.YY.so
138 AddIncludePath("-I$ALICE_ROOT/STEER/STEER -I$ALICE_ROOT/PYTHIA6 -I$ALICE_ROOT/LHAPDF -I$ALICE_ROOT/PWG/muon -I$ALICE_ROOT/PWG/muondep -I$ALICE_ROOT/MUON -I$ALICE_ROOT/include -I$ALICE_ROOT/EVGEN -I$ALICE_ROOT/FASTSIM");
140 TString ocdbPath("raw://");
143 ocdbPath = "local://$ALICE_ROOT/OCDB";
146 SetOCDBPath(ocdbPath.Data());
148 SetLocalDirectory("Snapshot",LocalDir());
150 SetVar("VAR_OCDB_PATH",Form("\"%s\"",ocdbPath.Data()));
152 SetVar("VAR_GENPARAM_INCLUDE","AliGenMUONlib.h");
153 SetVar("VAR_GENPARAM_NPART","1");
154 SetVar("VAR_GENPARAM_GENLIB_TYPE","AliGenMUONlib::kJpsi");
155 SetVar("VAR_GENPARAM_GENLIB_PARNAME","\"pPb 5.03\"");
157 SetVar("VAR_GENCORRHF_QUARK","5");
158 SetVar("VAR_GENCORRHF_ENERGY","5");
160 // some default values for J/psi
161 SetVar("VAR_GENPARAMCUSTOM_PDGPARTICLECODE","443");
163 // default values below are from J/psi p+Pb (from muon_calo pass)
164 SetVar("VAR_GENPARAMCUSTOM_Y_P0","4.08E5");
165 SetVar("VAR_GENPARAMCUSTOM_Y_P1","7.1E4");
167 SetVar("VAR_GENPARAMCUSTOM_PT_P0","1.13E9");
168 SetVar("VAR_GENPARAMCUSTOM_PT_P1","18.05");
169 SetVar("VAR_GENPARAMCUSTOM_PT_P2","2.05");
170 SetVar("VAR_GENPARAMCUSTOM_PT_P3","3.34");
172 // some default values for single muons
173 SetVar("VAR_GENPARAMCUSTOMSINGLE_PTMIN","0.35");
175 SetVar("VAR_GENPARAMCUSTOMSINGLE_PT_P0","4.05962");
176 SetVar("VAR_GENPARAMCUSTOMSINGLE_PT_P1","1.0");
177 SetVar("VAR_GENPARAMCUSTOMSINGLE_PT_P2","2.46187");
178 SetVar("VAR_GENPARAMCUSTOMSINGLE_PT_P3","2.08644");
180 SetVar("VAR_GENPARAMCUSTOMSINGLE_Y_P0","0.729545");
181 SetVar("VAR_GENPARAMCUSTOMSINGLE_Y_P1","0.53837");
182 SetVar("VAR_GENPARAMCUSTOMSINGLE_Y_P2","0.141776");
183 SetVar("VAR_GENPARAMCUSTOMSINGLE_Y_P3","0.0130173");
185 // some default values for GenBox
187 SetVar("VAR_GENMUBOX_PTMIN","0");
188 SetVar("VAR_GENMUBOX_PTMAX","20");
189 SetVar("VAR_GENMUBOX_YMIN","-4.1");
190 SetVar("VAR_GENMUBOX_YMAX","-2.4");
192 SetVar("VAR_PYTHIA8_CMS_ENERGY","8000");
193 SetVar("VAR_PYTHIA6_CMS_ENERGY","8000");
195 SetVar("VAR_PURELY_LOCAL",Form("%d",localOnly));
197 SetVar("VAR_SIM_ALIGNDATA","\"alien://folder=/alice/simulation/2008/v4-15-Release/Ideal\"");
199 SetVar("VAR_REC_ALIGNDATA","\"alien://folder=/alice/simulation/2008/v4-15-Release/Residual\"");
201 SetVar("VAR_USE_ITS_RECO","0");
203 UseOCDBSnapshots(fUseOCDBSnapshots);
205 gSystem->Load("libEVGEN");
207 SetVar("VAR_TRIGGER_CONFIGURATION","");
209 SetVar("VAR_PYTHIA8_INCLUDES","");
210 SetVar("VAR_PYTHIA8_SETENV","");
212 SetVar("VAR_PYTHIA6_INCLUDES","");
213 SetVar("VAR_PYTHIA6_SETENV","");
215 if ( TString(generator).Contains("pythia8",TString::kIgnoreCase) )
217 fMaxEventsPerChunk = 500; // 5000 is not reasonable with Pythia8 (and ITS+MUON...)
219 fCompactMode = 2; // keep AOD as for the time being the filtering driven from AODtrain.C cannot
220 // add SPD tracklets to muon AODs.
222 SetVar("VAR_USE_ITS_RECO","1");
224 if (gSystem->AccessPathName(gSystem->ExpandPathName(Form("$ALICE_ROOT/PYTHIA8/pythia%s",generatorVersion))))
226 AliError(Form("Directory -I$ALICE_ROOT/PYTHIA8/pythia%s/include not found",generatorVersion));
230 AddIncludePath(Form("-I$ALICE_ROOT/PYTHIA8 -I$ALICE_ROOT/PYTHIA8/pythia%s/include",generatorVersion));
231 // SetVar("VAR_PYTHIA8_VERSION",Form("\"%d\"",pythia8version));
233 SetVar("VAR_PYTHIA8_INCLUDES",Form("gSystem->AddIncludePath(\"-I$ALICE_ROOT/PYTHIA6 -I$ALICE_ROOT/STEER/STEER -I$ALICE_ROOT/LHAPDF -I$ALICE_ROOT/PYTHIA8 -I$ALICE_ROOT/PYTHIA8/pythia%s/include\");\n",generatorVersion));
237 p8env += Form(" gSystem->Setenv(\"PYTHIA8DATA\", gSystem->ExpandPathName(\"$ALICE_ROOT/PYTHIA8/pythia%s/xmldoc\"));\n",generatorVersion);
239 p8env += " gSystem->Setenv(\"LHAPDF\",gSystem->ExpandPathName(\"$ALICE_ROOT/LHAPDF\"));\n";
241 p8env += " gSystem->Setenv(\"LHAPATH\",gSystem->ExpandPathName(\"$ALICE_ROOT/LHAPDF/PDFsets\"));\n";
243 SetVar("VAR_PYTHIA8_SETENV",p8env.Data());
245 gSystem->Load("libFASTSIM");
246 gSystem->Load("liblhapdf"); // Parton density functions
247 gSystem->Load("libEGPythia6"); // TGenerator interface
248 // gSystem->Load("libpythia6"); // Pythia 6.2
249 gSystem->Load("libAliPythia6"); // ALICE specific implementations
250 gSystem->Load("libpythia8");
251 gSystem->Load("libAliPythia8");
253 SetVar("VAR_PYTHIA8_SETUP_STRINGS","\"\"");
255 SetVar("VAR_TRIGGER_CONFIGURATION","p-p");
258 if ( TString(generator).Contains("pythia6",TString::kIgnoreCase) )
260 fMaxEventsPerChunk = 500; // 5000 is not reasonable with Pythia6 (and ITS+MUON...)
262 fCompactMode = 2; // keep AOD as for the time being the filtering driven from AODtrain.C cannot
263 // add SPD tracklets to muon AODs.
265 gSystem->Load("libFASTSIM");
266 gSystem->Load("liblhapdf"); // Parton density functions
267 gSystem->Load("libEGPythia6"); // TGenerator interface
268 gSystem->Load(Form("libpythia6.%s",generatorVersion));
269 gSystem->Load("libAliPythia6");
271 SetVar("VAR_PYTHIA6_INCLUDES","gSystem->AddIncludePath(\"-I$ALICE_ROOT/PYTHIA6 -I$ALICE_ROOT/STEER/STEER -I$ALICE_ROOT/STEER/STEERBase -I$ALICE_ROOT/LHAPDF -I$ALICE_ROOT/FASTSIM\");");
275 p6env += Form("gSystem->Load(\"libpythia6.%s\");",generatorVersion);
277 SetVar("VAR_PYTHIA6_SETENV",p6env.Data());
279 SetVar("VAR_USE_ITS_RECO","1");
281 SetVar("VAR_TRIGGER_CONFIGURATION","p-p");
284 SetGenerator(generator);
288 MakeNofEventsFixed(10);
292 MakeNofEventsPropToTriggerCount();
295 AddToTemplateFileList("CheckESD.C");
296 AddToTemplateFileList("CheckAOD.C");
297 AddToTemplateFileList("AODtrain.C");
298 AddToTemplateFileList("validation.sh");
300 AddToTemplateFileList("Config.C");
301 AddToTemplateFileList("rec.C");
302 AddToTemplateFileList("sim.C");
303 AddToTemplateFileList("simrun.C");
304 AddToTemplateFileList(RunJDLName().Data());
306 UseExternalConfig(fExternalConfig);
309 //______________________________________________________________________________
310 AliMuonAccEffSubmitter::~AliMuonAccEffSubmitter()
315 ///______________________________________________________________________________
316 Bool_t AliMuonAccEffSubmitter::Generate(const char* jdlname) const
318 if ( TString(jdlname).Contains("merge",TString::kIgnoreCase) )
320 return GenerateMergeJDL(jdlname);
324 return GenerateRunJDL(jdlname);
328 ///______________________________________________________________________________
329 Bool_t AliMuonAccEffSubmitter::GenerateMergeJDL(const char* name) const
331 /// Create the JDL for merging jobs
332 /// FIXME: not checked !
336 std::ostream* os = CreateJDLFile(name);
343 Bool_t final = TString(name).Contains("merge",TString::kIgnoreCase);
345 (*os) << "# Generated merging jdl (production mode)" << std::endl
346 << "# $1 = run number" << std::endl
347 << "# $2 = merging stage" << std::endl
348 << "# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip" << std::endl;
350 OutputToJDL(*os,"Packages",
351 GetMapValue("AliRoot"),
352 GetMapValue("Geant3"),
356 OutputToJDL(*os,"Executable","AOD_merge.sh");
358 OutputToJDL(*os,"Price","1");
362 OutputToJDL(*os,"Jobtag","comment: AliMuonAccEffSubmitter final merging");
366 OutputToJDL(*os,"Jobtag","comment: AliMuonAccEffSubmitter merging stage $2");
369 OutputToJDL(*os,"Workdirectorysize","5000MB");
371 OutputToJDL(*os,"Validationcommand",Form("%s/validation_merge.sh",RemoteDir().Data()));
373 OutputToJDL(*os,"TTL","14400");
375 OutputToJDL(*os,"OutputArchive",
376 "log_archive.zip:stderr,stdout@disk=1",
377 "root_archive.zip:AliAOD.root,AliAOD.Muons.root,AnalysisResults.root@disk=3"
380 OutputToJDL(*os,"Arguments",(final ? "2":"1")); // for AOD_merge.sh, 1 means intermediate merging stage, 2 means final merging
384 OutputToJDL(*os,"InputFile",Form("LF:%s/AODtrain.C",RemoteDir().Data()));
385 OutputToJDL(*os,"OutputDir",Form("%s/$1/Stage_$2/#alien_counter_03i#",RemoteDir().Data()));
386 OutputToJDL(*os,"InputDataCollection",Form("%s/$1/Stage_$2.xml,nodownload",RemoteDir().Data()));
387 OutputToJDL(*os,"split","se");
388 OutputToJDL(*os,"SplitMaxInputFileNumber",GetSplitMaxInputFileNumber());
389 OutputToJDL(*os,"InputDataListFormat","xml-single");
390 OutputToJDL(*os,"InputDataList","wn.xml");
394 OutputToJDL(*os,"InputFile",Form("LF:%s/AODtrain.C",RemoteDir().Data()),
395 Form("LF:%s/$1/wn.xml",RemoteDir().Data()));
396 OutputToJDL(*os,"OutputDir",Form("%s/$1",RemoteDir().Data()));
402 //______________________________________________________________________________
403 Bool_t AliMuonAccEffSubmitter::GenerateRunJDL(const char* name) const
405 /// Generate (locally) the JDL to perform the simulation+reco+aod filtering
406 /// (to be then copied to the grid and finally submitted)
410 std::ostream* os = CreateJDLFile(name);
417 OutputToJDL(*os,"Packages",
418 GetMapValue("AliRoot"),
419 GetMapValue("Geant3"),
423 OutputToJDL(*os,"Jobtag","comment: AliMuonAccEffSubmitter RUN $1");
425 OutputToJDL(*os,"split","production:1-$2");
427 OutputToJDL(*os,"Price","1");
429 OutputToJDL(*os,"OutputDir",Form("%s/$1/#alien_counter_03i#",RemoteDir().Data()));
431 OutputToJDL(*os,"Executable","/alice/bin/aliroot_new");
434 files.SetOwner(kTRUE);
435 TIter next(LocalFileList());
438 while ( ( file = static_cast<TObjString*>(next())) )
440 if ( !file->String().Contains(".jdl",TString::kIgnoreCase) ||
441 !file->String().Contains("OCDB_") )
443 files.Add(new TObjString(Form("LF:%s/%s",RemoteDir().Data(),file->String().Data())));
447 if ( fUseOCDBSnapshots )
449 files.Add(new TObjString(Form("LF:%s/OCDB/$1/OCDB_sim.root",RemoteDir().Data())));
450 files.Add(new TObjString(Form("LF:%s/OCDB/$1/OCDB_rec.root",RemoteDir().Data())));
453 OutputToJDL(*os,"InputFile",files);
455 if ( CompactMode() == 0 )
458 OutputToJDL(*os,"OutputArchive", "log_archive.zip:stderr,stdout,aod.log,checkaod.log,checkesd.log,rec.log,sim.log@disk=1",
459 "root_archive.zip:galice*.root,Kinematics*.root,TrackRefs*.root,AliESDs.root,AliAOD.root,AliAOD.Muons.root,Merged.QA.Data.root,Run*.root@disk=2");
461 else if ( CompactMode() == 1 )
463 // keep only muon AODs and QA
464 OutputToJDL(*os,"OutputArchive", "log_archive.zip:stderr,stdout,*.log@disk=1",
465 "root_archive.zip:galice*.root,AliAOD.Muons.root,Merged.QA.Data.root@disk=2");
467 else if ( CompactMode() == 2 )
469 // keep only AODs and QA
470 OutputToJDL(*os,"OutputArchive", "log_archive.zip:stderr,stdout,aod.log,checkaod.log,checkesd.log,rec.log,sim.log@disk=1",
471 "root_archive.zip:galice*.root,AliAOD.root,Merged.QA.Data.root@disk=2");
475 AliError(Form("Unknown CompactMode %d",CompactMode()));
480 OutputToJDL(*os,"splitarguments","simrun.C --run $1 --chunk #alien_counter# --event $3");
482 OutputToJDL(*os,"Workdirectorysize","5000MB");
484 OutputToJDL(*os,"JDLVariables","Packages","OutputDir");
486 OutputToJDL(*os,"Validationcommand",Form("%s/validation.sh",RemoteDir().Data()));
488 if ( GetVar("VAR_GENERATOR").Contains("pythia",TString::kIgnoreCase) )
490 OutputToJDL(*os,"TTL","36000");
494 OutputToJDL(*os,"TTL","14400");
500 //______________________________________________________________________________
501 Bool_t AliMuonAccEffSubmitter::MakeOCDBSnapshots()
503 /// Run sim.C and rec.C in a special mode to generate OCDB snapshots
504 /// Can only be done after the templates have been copied locally
506 if (!IsValid()) return kFALSE;
508 if (!fUseOCDBSnapshots) return kTRUE;
510 if (!NofRuns()) return kFALSE;
516 const std::vector<int>& runs = RunList();
518 for ( std::vector<int>::size_type i = 0; i < runs.size(); ++i )
520 Int_t runNumber = runs[i];
522 TString ocdbSim(Form("%s/OCDB/%d/OCDB_sim.root",SnapshotDir().Data(),runNumber));
523 TString ocdbRec(Form("%s/OCDB/%d/OCDB_rec.root",SnapshotDir().Data(),runNumber));
525 if ( !gSystem->AccessPathName(ocdbSim.Data()) &&
526 !gSystem->AccessPathName(ocdbRec.Data()) )
528 AliWarning(Form("Local OCDB snapshots already there for run %d. Will not redo them. If you want to force them, delete them by hand !",runNumber));
533 gSystem->Exec(Form("aliroot -b -q -x simrun.C --run %d --snapshot",runNumber));
535 if ( gSystem->AccessPathName(ocdbSim.Data()) )
537 AliError(Form("Could not create OCDB snapshot for simulation"));
541 if ( gSystem->AccessPathName(ocdbRec.Data()) )
543 AliError(Form("Could not create OCDB snapshot for reconstruction"));
548 AddToLocalFileList(ocdbSim);
549 AddToLocalFileList(ocdbRec);
555 //______________________________________________________________________________
556 Bool_t AliMuonAccEffSubmitter::Merge(Int_t stage, Bool_t dryRun)
558 /// Submit multiple merging jobs with the format "submit AOD_merge(_final).jdl run# (stage#)".
559 /// Also produce the xml collection before sending jobs
560 /// Initial AODs will be taken from fRemoteDir/[RUNNUMBER] while the merged
561 /// ones will be put into fMergedDir/AODs/[RUNNUMBER]
564 /// - inDir = "/alice/sim/2012/LHC12a10_bis" (where to find the data to merge)
565 /// = 0x0 --> inDir = homeDir/outDir/resDir
566 /// - outDir = "Sim/LHC11h/embedding/AODs" (where to store merged results)
567 /// - runList.txt must contains the list of run number
568 /// - stage=0 --> final merging / stage>0 --> intermediate merging i
571 if (!RemoteDirectoryExists(MergedDir().Data())) {
572 AliError(Form("directory %s does not exist", MergedDir().Data()));
576 gGrid->Cd(MergedDir().Data());
578 TString jdl = MergeJDLName(stage==0);
580 if (!RemoteFileExists(jdl.Data()))
582 AliError(Form("file %s does not exist in %s\n", jdl.Data(), RemoteDir().Data()));
586 const std::vector<int>& runs = RunList();
590 AliError("No run to work with");
596 gSystem->Exec("rm -f __failed__");
597 Bool_t failedRun = kFALSE;
599 for ( std::vector<int>::size_type i = 0; i < runs.size(); ++i )
602 AliInfo(Form("\n --- processing run %d ---\n", run));
604 TString runDir = Form("%s/%d", MergedDir().Data(), run);
606 if (!RemoteDirectoryExists(runDir.Data()))
608 AliInfo(Form(" - creating output directory %s\n", runDir.Data()));
609 gSystem->Exec(Form("alien_mkdir -p %s", runDir.Data()));
612 if (RemoteFileExists(Form("%s/root_archive.zip", runDir.Data())))
614 AliWarning(" ! final merging already done");
618 Int_t lastStage = GetLastStage(runDir.Data());
620 if (stage > 0 && stage != lastStage+1)
622 AliError(Form(" ! lastest merging stage = %d. Next must be stage %d or final stage\n", lastStage, lastStage+1));
626 TString wn = (stage > 0) ? Form("Stage_%d.xml", stage) : "wn.xml";
627 TString find = (lastStage == 0) ?
628 Form("alien_find -x %s %s/%d *root_archive.zip", wn.Data(), RemoteDir().Data(), run) :
629 Form("alien_find -x %s %s/%d/Stage_%d *root_archive.zip", wn.Data(), RemoteDir().Data(), run, lastStage);
630 gSystem->Exec(Form("%s 1> %s 2>/dev/null", find.Data(), wn.Data()));
631 gSystem->Exec(Form("grep -c /event %s > __nfiles__", wn.Data()));
632 ifstream f2("__nfiles__");
634 nFiles.ReadLine(f2,kTRUE);
636 gSystem->Exec("rm -f __nfiles__");
637 printf(" - number of files to merge = %d\n", nFiles.Atoi());
638 if (nFiles.Atoi() == 0) {
639 printf(" ! collection of files to merge is empty\n");
640 gSystem->Exec(Form("rm -f %s", wn.Data()));
642 } else if (stage > 0 && nFiles.Atoi() <= splitLevel && !reply.BeginsWith("y")) {
643 if (!reply.BeginsWith("n")) {
644 printf(" ! number of files to merge <= split level (%d). Continue? [Y/n] ", splitLevel);
646 reply.Gets(stdin,kTRUE);
649 if (reply.BeginsWith("n")) {
650 gSystem->Exec(Form("rm -f %s", wn.Data()));
657 TString dirwn = Form("%s/%s", runDir.Data(), wn.Data());
658 if (RemoteFileExists(dirwn.Data())) gGrid->Rm(dirwn.Data());
659 gSystem->Exec(Form("alien_cp file:%s alien://%s", wn.Data(), dirwn.Data()));
660 gSystem->Exec(Form("rm -f %s", wn.Data()));
664 if (stage > 0) query = Form("submit %s %d %d", jdl.Data(), run, stage);
665 else query = Form("submit %s %d", jdl.Data(), run);
666 printf(" - %s ...", query.Data());
675 Bool_t done = kFALSE;
676 TGridResult *res = gGrid->Command(query);
679 TString cjobId1 = res->GetKey(0,"jobId");
680 if (!cjobId1.IsDec())
688 AliInfo(Form(" DONE\n --> the job Id is: %s \n", cjobId1.Data()));
700 gSystem->Exec(Form("echo %d >> __failed__", run));
708 AliInfo("\n--------------------\n");
709 AliInfo("list of failed runs:\n");
710 gSystem->Exec("cat __failed__");
711 gSystem->Exec("rm -f __failed__");
718 //______________________________________________________________________________
719 void AliMuonAccEffSubmitter::Print(Option_t* opt) const
723 AliMuonGridSubmitter::Print(opt);
727 std::cout << std::endl << Form("-- For each run, will generate %5.2f times the number of real events for trigger %s",
728 fRatio,ReferenceTrigger().Data()) << std::endl;
732 std::cout << std::endl << Form("-- For each run, will generate %10d events",fFixedNofEvents) << std::endl;
735 std::cout << "-- MaxEventsPerChunk = " << fMaxEventsPerChunk << std::endl;
737 std::cout << "-- Will" << (fUseOCDBSnapshots ? "" : " NOT") << " use OCDB snaphosts" << std::endl;
740 //______________________________________________________________________________
741 Bool_t AliMuonAccEffSubmitter::Run(const char* mode)
743 /// mode can be one of (case insensitive)
745 /// LOCAL : copy the template files from the template directory to the local one
746 /// UPLOAD : copy the local files to the grid (requires LOCAL)
747 /// OCDB : make ocdb snapshots (requires LOCAL)
748 /// SUBMIT : submit the jobs (requires LOCAL + UPLOAD)
749 /// FULL : all of the above (requires all of the above)
751 /// TEST : as SUBMIT, but in dry mode (does not actually submit the jobs)
753 /// LOCALTEST : completely local test (including execution)
755 if (!IsValid()) return kFALSE;
760 if ( smode == "FULL")
762 return ( Run("LOCAL") && Run("OCDB") && Run("UPLOAD") && Run("SUBMIT") );
765 if ( smode == "LOCAL")
767 return CopyTemplateFilesToLocal();
770 if ( smode == "UPLOAD" )
772 return (CopyLocalFilesToRemote());
775 if ( smode == "OCDB" )
777 Bool_t ok = Run("LOCAL");
780 ok = MakeOCDBSnapshots();
785 if ( smode == "TEST" )
787 Bool_t ok = Run("LOCAL") && Run("OCDB") && Run("UPLOAD");
790 ok = (Submit(kTRUE)>0);
795 if ( smode == "FULL" )
797 Bool_t ok = Run("LOCAL") && Run("OCDB") && Run("UPLOAD");
800 ok = (Submit(kFALSE)>0);
805 if( smode == "SUBMIT" )
807 return (Submit(kFALSE)>0);
810 if ( smode == "LOCALTEST" )
812 Bool_t ok = Run("LOCAL");
823 //______________________________________________________________________________
824 Bool_t AliMuonAccEffSubmitter::SetGenerator(const char* generator)
826 // set the variable to select the generator macro in Config.C
828 gSystem->Load("libEVGEN");
832 TString generatorFile(Form("%s/%s.C",TemplateDir().Data(),generator));
834 Int_t nofMissingVariables(0);
836 // first check we indeed have such a macro
837 if (!gSystem->AccessPathName(generatorFile.Data()))
839 TObjArray* variables = GetVariables(generatorFile.Data());
841 TIter next(variables);
844 while ( ( var = static_cast<TObjString*>(next())) )
846 if ( !Vars()->GetValue(var->String()) )
848 ++nofMissingVariables;
849 AliError(Form("file %s expect the variable %s to be defined, but we've not defined it !",generatorFile.Data(),var->String().Data()));
855 if ( !nofMissingVariables )
857 if (CheckCompilation(generatorFile.Data()))
860 SetVar("VAR_GENERATOR",Form("%s",generator));
861 AddToTemplateFileList(Form("%s.C",generator));
872 AliError(Form("Can not work with the macro %s",generatorFile.Data()));
877 //______________________________________________________________________________
878 void AliMuonAccEffSubmitter::SetOCDBPath(const char* ocdbPath)
880 /// Sets the OCDB path to be used
882 SetMapKeyValue("OCDBPath",ocdbPath);
886 //______________________________________________________________________________
887 void AliMuonAccEffSubmitter::SetOCDBSnapshotDir(const char* dir)
889 // change the directory used for snapshot
891 if (gSystem->AccessPathName(Form("%s/OCDB",dir)))
893 AliError(Form("Snapshot top directory (%s) should contain an OCDB subdir with runnumbers in there",dir));
897 SetMapKeyValue("OCDBSnapshot",dir);
901 //______________________________________________________________________________
902 void AliMuonAccEffSubmitter::MakeNofEventsPropToTriggerCount(const char* trigger, Float_t ratio)
904 SetMapKeyValue("ReferenceTrigger",trigger);
908 //______________________________________________________________________________
909 void AliMuonAccEffSubmitter::MakeNofEventsFixed(Int_t nevents)
911 fFixedNofEvents = nevents;
913 SetMapKeyValue("ReferenceTrigger","");
916 //______________________________________________________________________________
917 Int_t AliMuonAccEffSubmitter::LocalTest()
919 /// Generate a local macro (simrun.sh) to execute locally a full scale test
920 /// Can only be used with a fixed number of events (and runnumber is fixed to zero)
924 AliError("Can only work in local test with a fixed number of events");
928 if ( fFixedNofEvents <= 0 )
930 AliError("Please fix the number of input events using MakeNofEventsFixed()");
934 const std::vector<int>& runs = RunList();
938 AliError("No run to work with");
942 std::cout << "Generating script to execute : ./simrun.sh" << std::endl;
944 std::ofstream out("simrun.sh");
946 out << "#!/bin/bash" << std::endl;
947 // root.exe -b -q simrun.C --run <x> --chunk <y> --event <n>
948 out << "root.exe -b -q simrun.C --run "<< runs[0] <<" --event " << fFixedNofEvents << std::endl;
950 gSystem->Exec("chmod +x simrun.sh");
952 std::cout << "Cleaning up left-over files from previous simulation/reconstructions" << std::endl;
954 gSystem->Exec("rm -rf TrackRefs.root *.SDigits*.root Kinematics.root *.Hits.root geometry.root gphysi.dat Run*.tag.root HLT*.root *.ps *.Digits.root *.RecPoints.root galice.root *QA*.root Trigger.root *.log AliESD* AliAOD* *.d *.so *.stat");
956 std::cout << "Executing the script : ./simrun.sh" << std::endl;
959 gSystem->Exec("./simrun.sh");
966 void OutputRunList(const char* filename, const std::vector<int>& runlist)
968 /// output a runlist to ASCII file
970 std::ofstream out(filename);
972 for ( std::vector<int>::size_type j = 0; j < runlist.size(); ++j )
974 out << runlist[j] << std::endl;
979 //______________________________________________________________________________
980 Int_t AliMuonAccEffSubmitter::SplitRunList(const char* inputList, int maxJobs)
982 /// In order to be able to submit, split a given runlist into chunks that will
983 /// fit within maxJobs (1500 for a typical user)
985 std::vector<int> runs;
987 AliAnalysisTriggerScalers tmp(inputList);
988 runs = tmp.GetRunList();
990 AliAnalysisTriggerScalers* ts(0x0);
991 std::vector<int> currentRunList;
998 for (std::vector<int>::size_type i=0; i < runs.size(); ++i)
1000 Int_t runNumber = runs[i];
1002 Int_t nEvtRun(fFixedNofEvents);
1008 AliInfo(Form("Creating AliAnalysisTriggerScalers from OCDB=%s",OCDBPath().Data()));
1009 ts = new AliAnalysisTriggerScalers(runs,OCDBPath().Data());
1012 AliAnalysisTriggerScalerItem* trigger = ts->GetTriggerScaler(runNumber, "L2A", ReferenceTrigger().Data());
1016 AliError(Form("Could not get trigger %s for run %09d",ReferenceTrigger().Data(),runNumber));
1019 nEvtRun = TMath::Nint(fRatio * trigger->Value());
1024 while (nEvtRun/nChunk+0.5 > MaxEventsPerChunk())
1029 Int_t nEvtChunk = TMath::Nint(nEvtRun/nChunk + 0.5);
1033 nTotalJobs += nChunk;
1035 nEvts += nChunk*nEvtChunk;
1037 if ( nJobs > maxJobs )
1041 OutputRunList(Form("%s.%d",inputList,nFiles),currentRunList);
1043 currentRunList.clear();
1047 currentRunList.push_back(runNumber);
1051 if ( !currentRunList.empty() )
1054 OutputRunList(Form("%s.%d",inputList,nFiles),currentRunList);
1060 std::cout << Form("input run list was split into %d files. Total number of jobs %d. Total number of events %d",
1061 nFiles,nTotalJobs,nEvts) << std::endl;
1067 //______________________________________________________________________________
1068 Int_t AliMuonAccEffSubmitter::Submit(Bool_t dryRun)
1070 /// Submit multiple production jobs with the format "submit jdl 000run#.xml 000run#".
1072 /// Return the number of submitted (master) jobs
1075 /// - outputDir = "/alice/cern.ch/user/p/ppillot/Sim/LHC10h/JPsiPbPb276/AlignRawVtxRaw/ESDs"
1076 /// - runList must contains the list of run number
1077 /// - trigger is the (fully qualified) trigger name used to compute the base number of events
1078 /// - mult is the factor to apply to the number of trigger to get the number of events to be generated
1079 /// (# generated events = # triggers x mult
1081 if (!IsValid()) return 0;
1085 gGrid->Cd(RemoteDir());
1087 if (!RemoteFileExists(RunJDLName()))
1089 AliError(Form("file %s does not exist in %s", RunJDLName().Data(), RemoteDir().Data()));
1095 AliError("No run list set. Use SetRunList");
1098 const std::vector<int>& runs = RunList();
1102 AliError("No run to work with");
1106 // cout << "total number of selected MB events = " << totEvt << endl;
1107 // cout << "required number of generated events = " << nGenEvents << endl;
1108 // cout << "number of generated events per MB event = " << ratio << endl;
1111 std::cout << "run\tchunks\tevents" << std::endl;
1112 std::cout << "----------------------" << std::endl;
1117 AliAnalysisTriggerScalers* ts(0x0);
1119 for (std::vector<int>::size_type i=0; i < runs.size(); ++i)
1121 Int_t runNumber = runs[i];
1123 Int_t nEvtRun(fFixedNofEvents);
1129 AliInfo(Form("Creating AliAnalysisTriggerScalers from OCDB=%s",OCDBPath().Data()));
1130 ts = new AliAnalysisTriggerScalers(runs,OCDBPath().Data());
1133 AliAnalysisTriggerScalerItem* trigger = ts->GetTriggerScaler(runNumber, "L2A", ReferenceTrigger().Data());
1137 AliError(Form("Could not get trigger %s for run %09d",ReferenceTrigger().Data(),runNumber));
1140 nEvtRun = TMath::Nint(fRatio * trigger->Value());
1145 while (nEvtRun/nChunk+0.5 > MaxEventsPerChunk())
1150 Int_t nEvtChunk = TMath::Nint(nEvtRun/nChunk + 0.5);
1154 nEvts += nChunk*nEvtChunk;
1156 std::cout << runNumber << "\t" << nChunk << "\t" << nEvtChunk << std::endl;
1158 TString query(Form("submit %s %d %d %d", RunJDLName().Data(), runNumber, nChunk, nEvtChunk));
1160 std::cout << query.Data() << " ..." << std::flush;
1162 TGridResult* res = 0x0;
1166 res = gGrid->Command(query);
1171 TString cjobId1 = res->GetKey(0,"jobId");
1173 if (!cjobId1.Length())
1175 std::cout << " FAILED" << std::endl << std::endl;
1181 std::cout << "DONE" << std::endl;
1182 std::cout << Form(" --> the job Id is: %s",cjobId1.Data()) << std::endl << std::endl;
1187 std::cout << " FAILED" << std::endl << std::endl;
1193 std::cout << std::endl
1194 << "total number of jobs = " << nJobs << std::endl
1195 << "total number of generated events = " << nEvts << std::endl
1203 //______________________________________________________________________________
1204 void AliMuonAccEffSubmitter::UpdateLocalFileList(Bool_t clearSnapshots)
1206 /// Update the list of local files
1208 AliMuonGridSubmitter::UpdateLocalFileList();
1210 if (!NofRuns()) return;
1212 if ( clearSnapshots )
1214 TIter next(LocalFileList());
1217 while ( ( file = static_cast<TObjString*>(next())) )
1219 if ( file->String().Contains("OCDB_") )
1221 LocalFileList()->Remove(file);
1224 LocalFileList()->Compress();
1227 const char* type[] = { "sim","rec" };
1229 const std::vector<int>& runs = RunList();
1231 for ( std::vector<int>::size_type i = 0; i < runs.size(); ++i )
1233 Int_t runNumber = runs[i];
1235 for ( Int_t t = 0; t < 2; ++t )
1237 TString snapshot(Form("%s/OCDB/%d/OCDB_%s.root",SnapshotDir().Data(),runNumber,type[t]));
1239 if ( !gSystem->AccessPathName(snapshot.Data()) )
1241 AddToLocalFileList(snapshot);
1247 //______________________________________________________________________________
1248 void AliMuonAccEffSubmitter::UseOCDBSnapshots(Bool_t flag)
1250 /// Whether or not to use OCDB snapshots
1251 /// Using OCDB snapshots will speed-up both the sim and reco initialization
1252 /// phases on each worker node, but takes time to produce...
1253 /// So using them is not always a win-win...
1255 fUseOCDBSnapshots = flag;
1258 SetVar("VAR_OCDB_SNAPSHOT","kTRUE");
1260 // for some reason must include ITS objects in the snapshot
1261 // (to be able to instantiante the vertexer later on ?)
1263 SetVar("VAR_USE_ITS_RECO","1");
1267 SetVar("VAR_OCDB_SNAPSHOT","kFALSE");
1270 UpdateLocalFileList();
1273 //______________________________________________________________________________
1274 void AliMuonAccEffSubmitter::UseAODMerging(Bool_t flag)
1276 /// whether or not we should generate JDL for merging AODs
1278 fUseAODMerging = flag;
1280 AddToTemplateFileList(MergeJDLName(kFALSE).Data());
1281 AddToTemplateFileList(MergeJDLName(kTRUE).Data());
1282 AddToTemplateFileList("AOD_merge.sh");
1283 AddToTemplateFileList("validation_merge.sh");
1286 //______________________________________________________________________________
1287 void AliMuonAccEffSubmitter::UseExternalConfig(const char* externalConfigFullFilePath)
1289 // use an external config (or the default Config.C if externalConfigFullFilePath="")
1291 fExternalConfig = externalConfigFullFilePath;
1292 if ( fExternalConfig.Length() > 0 )
1294 AddToTemplateFileList(fExternalConfig);
1298 AddToTemplateFileList("Config.C");