1 /**************************************************************************
2 * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
17 // AliMuonGridSubmitter : a class to help submit jobs for several runs
19 // This class is dealing with 3 different directories :
21 // - template directory ($ALICE_ROOT/PWG/muondep/XXXTemplates) containing the
22 // basic template files to be used for a particular job type (XXX). A template can contain
23 // some variables that will be replaced during during the copy from template
26 // - local directory, where the files from the template directory, are copied
27 // once the class has been configured properly (i.e. using the various Set, Use,
28 // etc... methods). Some other files (e.g. JDL ones) are generated from
29 // scratch and also copied into this directory.
30 // At this point one could(should) check the files, as they are the ones
31 // to be copied to the remote directory for the production
33 // - remote directory, the alien directory where the files will be copied
34 // (from the local directory) before the actual submission
36 // author: Laurent Aphecetche (Subatech
39 #include "AliMuonGridSubmitter.h"
44 #include "TGridResult.h"
47 #include "TObjString.h"
52 #include "AliAnalysisTriggerScalers.h"
53 #include "Riostream.h"
55 //______________________________________________________________________________
56 AliMuonGridSubmitter::AliMuonGridSubmitter(AliMuonGridSubmitter::EJobType jobType, Bool_t localOnly)
61 fShouldOverwriteFiles(kFALSE),
62 fTemplateFileList(0x0),
68 if (!gGrid && !localOnly)
70 TGrid::Connect("alien://");
73 AliError("cannot connect to grid");
77 SetPackages("VO_ALICE@AliRoot::v5-04-Rev-20");
79 TString basedir = gSystem->ExpandPathName("$ALICE_ROOT/PWG/muondep");
82 dir.Form("%s/%sTemplates",basedir.Data(),JobTypeName(jobType).Data());
84 if (!SetTemplateDir(dir.Data()))
86 AliError(Form("Could not find %s directory. Please check.",dir.Data()));
89 SetLocalDir(gSystem->pwd());
92 //______________________________________________________________________________
93 AliMuonGridSubmitter::~AliMuonGridSubmitter()
96 delete fTemplateFileList;
97 delete fLocalFileList;
102 ///______________________________________________________________________________
103 void AliMuonGridSubmitter::AddIncludePath(const char* pathList) const
105 TObjArray* paths = TString(pathList).Tokenize(" ");
108 TString includePath = gSystem->GetIncludePath();
110 while ( ( p = static_cast<TObjString*>(next()) ) )
112 if ( !includePath.Contains(p->String()) )
114 gSystem->AddIncludePath(p->String().Data());
122 //______________________________________________________________________________
123 void AliMuonGridSubmitter::AddToTemplateFileList(const char* filename)
125 // add a file to the list of templates
126 // and update the local file list if needed
128 TObjArray* a = TemplateFileList();
130 if ( !a->FindObject(filename) )
132 a->Add(new TObjString(filename));
133 UpdateLocalFileList();
137 //______________________________________________________________________________
138 void AliMuonGridSubmitter::AddToLocalFileList(const char* filename)
140 // add a file to the list of local files
142 TObjArray* a = LocalFileList();
144 if ( !a->FindObject(filename) )
146 a->Add(new TObjString(filename));
150 //______________________________________________________________________________
151 Bool_t AliMuonGridSubmitter::CheckCompilation(const char* file) const
153 /// Check whether file can be compiled or not
154 /// FIXME: use gSystem->TempFileName for tmpfile !
158 TString sfile(gSystem->BaseName(file));
159 TString tmpfile(Form("tmpfile_%s",sfile.Data()));
161 gSystem->Exec(Form("cp %s %s",file,tmpfile.Data()));
163 ReplaceVars(tmpfile.Data());
165 gSystem->AddIncludePath("-I$ALICE_ROOT/include");
166 gSystem->AddIncludePath("-I$ALICE_ROOT/EVGEN");
168 if (gROOT->LoadMacro(Form("%s++",tmpfile.Data())))
170 AliError(Form("macro %s can not be compiled. Please check.",file));
174 gSystem->Exec(Form("rm %s",tmpfile.Data()));
180 //______________________________________________________________________________
181 Bool_t AliMuonGridSubmitter::CheckLocal() const
183 /// Check whether all required local files are there
184 TIter next(LocalFileList());
187 while ( ( file = static_cast<TObjString*>(next())) )
189 if ( gSystem->AccessPathName(file->String().Data()) )
198 //______________________________________________________________________________
199 Bool_t AliMuonGridSubmitter::CheckRemote() const
201 /// Check whether all required remote files are there
202 AliWarning("implement me");
206 //______________________________________________________________________________
207 void AliMuonGridSubmitter::CleanLocal(const char* excludeList) const
209 /// Clean (remove) local generated files
210 /// exclude contains a list of comma separated pattern of files
213 TIter next(LocalFileList());
215 TObjArray* excludeArray = TString(excludeList).Tokenize(",");
217 while ( ( file = static_cast<TObjString*>(next())) )
219 Bool_t shouldExclude(kFALSE);
221 TIter nextExclude(excludeArray);
224 while ( ( s = static_cast<TObjString*>(nextExclude())) && !shouldExclude )
226 if ( file->String().Contains(s->String()) ) shouldExclude=kTRUE;
231 gSystem->Unlink(file->String().Data());
238 //______________________________________________________________________________
239 Bool_t AliMuonGridSubmitter::CopyFile(const char* localFile)
241 /// copy a local file to remote destination
244 if ( gSystem->IsAbsoluteFileName(localFile) )
250 local = Form("%s/%s",LocalDir().Data(),gSystem->ExpandPathName(localFile));
253 if (gSystem->AccessPathName(local.Data()))
255 AliErrorClass(Form("Local file %s does not exist",local.Data()));
261 remote += RemoteDir().Data();
264 if ( gSystem->IsAbsoluteFileName(localFile) )
266 TString tmp(localFile);
267 tmp.ReplaceAll(GetMapValue("Snapshot"),"");
268 tmp.ReplaceAll(GetMapValue("Local"),"");
276 TString dirName = gSystem->DirName(remote.Data());
280 if (!RemoteDirectoryExists(dirName.Data()))
282 ok = gGrid->Mkdir(dirName.Data(),"-p");
287 AliDebugClass(1,Form("cp %s alien://%s",local.Data(),remote.Data()));
288 return TFile::Cp(local.Data(),Form("alien://%s",remote.Data()));
296 //______________________________________________________________________________
297 Bool_t AliMuonGridSubmitter::CheckRemoteDir() const
299 /// Check we have a grid connection and that the remote dir exists
301 if (RemoteDir().IsNull())
303 AliError("you must provide the grid location where to copy the files");
307 if (!RemoteDirectoryExists(RemoteDir()))
309 AliError(Form("directory %s does not exist", RemoteDir().Data()));
316 //______________________________________________________________________________
317 Bool_t AliMuonGridSubmitter::CopyLocalFilesToRemote()
319 /// copy all files necessary to run the simulation into remote directory
321 TIter next(LocalFileList());
326 while ( ( ftc = static_cast<TObjString*>(next())) )
328 allok = allok && CopyFile(ftc->String());
333 //______________________________________________________________________________
334 Bool_t AliMuonGridSubmitter::CopyTemplateFilesToLocal()
336 // copy (or generate) local files from the template ones
338 if (!IsValid()) return kFALSE;
342 TIter next(TemplateFileList());
346 Bool_t potentialProblem(kFALSE);
348 while ( ( file = static_cast<TObjString*>(next())) )
350 if ( file->String().Contains("OCDB") )
352 /// OCDB snapshots are not in template
356 if ( !ShouldOverwriteFiles() && !gSystem->AccessPathName(file->String().Data()) )
358 AliError(Form("Local file %s already exists. Remove it first if you want to update overwrite it",file->String().Data()));
359 potentialProblem = kTRUE;
363 TString stemplate(Form("%s/%s",TemplateDir().Data(),file->String().Data()));
364 TString slocal(Form("%s/%s",LocalDir().Data(),file->String().Data()));
366 AliDebug(1,Form("Copying %s to %s",stemplate.Data(),slocal.Data()));
368 Int_t c = gSystem->CopyFile(stemplate.Data(),slocal.Data(),ShouldOverwriteFiles());
372 if ( stemplate.Contains(".jdl",TString::kIgnoreCase) )
374 ok = Generate(file->String().Data());
378 AliError(Form("Error %d copying file %s",c,stemplate.Data()));
387 if ( HasVars(slocal.Data()) )
389 if (!ReplaceVars(slocal.Data()))
391 AliError("pb in ReplaceVars");
400 if ( potentialProblem )
402 AliWarning("At least one local file could not be overwritten. Cross-check that the local files are OK before we try to upload them to the Grid !");
408 //______________________________________________________________________________
409 std::ostream* AliMuonGridSubmitter::CreateJDLFile(const char* name) const
411 /// Create a (local and empty) JDL file
415 TString jdl(Form("%s/%s",LocalDir().Data(),name));
417 if ( !ShouldOverwriteFiles() && !gSystem->AccessPathName(jdl.Data()) )
419 AliErrorClass(Form("File %s already exists. Remove it if you want to overwrite it",jdl.Data()));
423 std::ofstream* os = new std::ofstream(gSystem->ExpandPathName(jdl.Data()));
427 AliErrorClass(Form("Cannot create file %s",jdl.Data()));
435 //______________________________________________________________________________
436 Int_t AliMuonGridSubmitter::GetLastStage(const char* remoteDir)
438 /// Get the last staging phase already performed
440 TGridResult* r = gGrid->Ls(remoteDir);
443 Int_t offset = TString("Stage_").Length();
445 while ( r->GetFileName(i) )
447 TString file(r->GetFileName(i));
448 if (file.BeginsWith("Stage_") && !file.Contains("xml") )
450 Int_t n = TString(file(offset,file.Length()-offset)).Atoi();
451 lastStage = TMath::Max(lastStage,n);
459 //______________________________________________________________________________
460 TString AliMuonGridSubmitter::GetMapValue(const char* key) const
462 // convenience method to access internal map of TObjStrings
463 if (!fInternalMap) return "";
465 TObject* o = fInternalMap->GetValue(key);
469 return static_cast<TObjString*>(o)->String();
475 //______________________________________________________________________________
476 TObjArray* AliMuonGridSubmitter::GetVariables(const char* file) const
478 /// Find the variables in the file
480 std::ifstream in(file);
482 TObjArray* variables(0x0);
484 while ( in.getline(line,1023,'\n') )
487 while (sline.Contains("VAR_") && !sline.BeginsWith("//") )
489 Int_t i1 = sline.Index("VAR_");
492 while ( ( i2 < sline.Length() ) && ( isalnum(sline[i2]) || sline[i2]=='_' ) ) ++i2;
496 variables = new TObjArray;
497 variables->SetOwner(kTRUE);
500 TString var = sline(i1,i2-i1);
501 if ( !variables->FindObject(var) )
503 variables->Add(new TObjString(var));
505 sline.ReplaceAll(var,"");
514 //______________________________________________________________________________
515 Bool_t AliMuonGridSubmitter::HasVars(const char* file) const
517 /// Whether or not the file contains variables that have to
520 std::ifstream in(file);
522 while ( in.getline(line,1023,'\n') )
525 if (sline.Contains("VAR_") && !sline.BeginsWith("//") )
533 //______________________________________________________________________________
534 TMap* AliMuonGridSubmitter::InternalMap() const
538 fInternalMap = new TMap;
539 fInternalMap->SetOwnerKeyValue(kTRUE,kTRUE);
544 //______________________________________________________________________________
545 TString AliMuonGridSubmitter::JobTypeName(AliMuonGridSubmitter::EJobType jobType) const
547 if ( jobType == kAccEff )
551 else if ( jobType == kQAMerge)
558 //______________________________________________________________________________
559 TObjArray* AliMuonGridSubmitter::LocalFileList() const
561 /// Return (after createing and filling it if needed)
562 /// the internal file list with paths from the local directory
566 fLocalFileList = static_cast<TObjArray*>(TemplateFileList()->Clone());
569 return fLocalFileList;
572 //______________________________________________________________________________
573 UInt_t AliMuonGridSubmitter::NofRuns() const
575 // number of runs we're dealing with
576 return fRunList.size();
579 //______________________________________________________________________________
580 TObjArray* AliMuonGridSubmitter::OrderKeys(const TMap& map) const
582 /// return an array where the map's keys are sorted alphabetically
583 /// the returned array should be deleted by the client
585 TObjArray* keyArray = new TObjArray;
586 keyArray->SetOwner(kTRUE);
590 while ( ( key = static_cast<TObjString*>(next())) )
592 keyArray->Add(new TObjString(key->String()));
600 //______________________________________________________________________________
601 void AliMuonGridSubmitter::OutputToJDL(std::ostream& out, const char* key,
602 const TObjArray& values) const
604 /// output to ostream of key,{values} pair
608 Int_t n = values.GetEntries();
612 out << "{" << std::endl;
616 while ( ( v = static_cast<TObjString*>(next())) )
618 if ( v->String().Length() == 0 ) continue;
621 out << "\t\"" << v->String().Data() << "\"";
629 TString& v1 = static_cast<TObjString*>(values.At(0))->String();
631 if ( v1.IsDigit() && !(TString(key).Contains("SplitMax")) && !(TString(key).Contains("TTL")) )
637 out << "\"" << v1.Data() << "\"";
640 out << ";" << std::endl;
643 //______________________________________________________________________________
644 void AliMuonGridSubmitter::OutputToJDL(std::ostream& out, const char* key, const char* v1,
645 const char* v2, const char* v3, const char* v4,
646 const char* v5, const char* v6, const char* v7,
647 const char* v8, const char* v9) const
649 /// output to ostream
652 values.SetOwner(kTRUE);
654 values.Add(new TObjString(v1));
655 if ( strlen(v2) > 0 ) values.Add(new TObjString(v2));
656 if ( strlen(v3) > 0 ) values.Add(new TObjString(v3));
657 if ( strlen(v4) > 0 ) values.Add(new TObjString(v4));
658 if ( strlen(v5) > 0 ) values.Add(new TObjString(v5));
659 if ( strlen(v6) > 0 ) values.Add(new TObjString(v6));
660 if ( strlen(v7) > 0 ) values.Add(new TObjString(v7));
661 if ( strlen(v8) > 0 ) values.Add(new TObjString(v8));
662 if ( strlen(v9) > 0 ) values.Add(new TObjString(v9));
664 OutputToJDL(out,key,values);
668 //______________________________________________________________________________
669 void AliMuonGridSubmitter::Print(Option_t* /*opt*/) const
675 std::cout << std::string(80,'*') << std::endl;
676 std::cout << "INVALID OBJECT. CHECK BELOW THE CONFIGURATION." << std::endl;
677 std::cout << std::string(80,'*') << std::endl;
680 std::cout << "-- Internals : " << std::endl;
682 TObjArray* im = OrderKeys(*fInternalMap);
687 while ( ( key = static_cast<TObjString*>(next()) ) )
689 TString value = static_cast<TObjString*>(fInternalMap->GetValue(key->String()))->String();
691 std::cout << key->String() << " : " << value.Data() << std::endl;
698 std::cout << NofRuns() << " run";
699 if ( NofRuns() > 1 ) std::cout << "s";
701 for ( std::vector<int>::size_type i = 0; i < fRunList.size(); ++i )
703 std::cout << fRunList[i] << " ";
705 std::cout << std::endl;
708 std::cout << std::endl << "-- Variables : " << std::endl;
710 TObjArray* iv = OrderKeys(*fVars);
713 while ( ( key = static_cast<TObjString*>(nextVar())) )
715 TObjString* value = static_cast<TObjString*>(fVars->GetValue(key->String()));
716 std::cout << "Variable " << key->String() << " will be replaced by " << value->String() << std::endl;
721 std::cout << std::endl << "-- Files to be uploaded:" << std::endl;
722 TIter nextFile(LocalFileList());
724 while ( ( sfile = static_cast<TObjString*>(nextFile())) )
726 std::cout << sfile->String().Data() << std::endl;
731 //______________________________________________________________________________
732 Bool_t AliMuonGridSubmitter::RemoteDirectoryExists(const char *dirname) const
734 // Returns true if directory exists. Can be also a path.
735 if (!gGrid) return kFALSE;
736 // Check if dirname is a path
737 TString dirstripped = dirname;
738 dirstripped = dirstripped.Strip();
739 dirstripped = dirstripped.Strip(TString::kTrailing, '/');
740 TString dir = gSystem->BaseName(dirstripped);
742 TString path = gSystem->DirName(dirstripped);
743 TGridResult *res = gGrid->Ls(path, "-F");
744 if (!res) return kFALSE;
748 while ((map=dynamic_cast<TMap*>(next()))) {
749 obj = map->GetValue("name");
751 if (dir == obj->GetName()) {
760 //______________________________________________________________________________
761 Bool_t AliMuonGridSubmitter::RemoteFileExists(const char *lfn)
763 // Returns true if file exists.
764 if (!gGrid) return kFALSE;
765 TGridResult *res = gGrid->Ls(lfn);
766 if (!res) return kFALSE;
767 TMap *map = dynamic_cast<TMap*>(res->At(0));
772 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
773 if (!objs || !objs->GetString().Length()) {
781 //______________________________________________________________________________
782 Bool_t AliMuonGridSubmitter::ReplaceVars(const char* file) const
784 /// Replace the variables (i.e. things starting by VAR_) found in file
788 std::ifstream in(file);
791 lines.SetOwner(kTRUE);
797 while ( in.getline(line,1023,'\n') )
800 while (sline.Contains("VAR_") && !sline.BeginsWith("//") )
808 while ( ( key = static_cast<TObjString*>(next())) )
810 AliDebug(1,Form("Does %s contains %s ?",sline.Data(),key->String().Data()));
812 if ( sline.Contains(key->String()) )
816 TObjString* value = static_cast<TObjString*>(fVars->GetValue(key->String()));
817 AliDebug(1,Form("Replacing %s by %s in %s",key->String().Data(),value->String().Data(),sline.Data()));
818 sline.ReplaceAll(key->String(),value->String());
826 AliError(Form("Could not find a replacement for variable %s",sline.Data()));
831 lines.Add(new TObjString(sline));
838 if ( nreplaced != nvars )
840 AliError(Form("nvars=%d nreplaced=%d",nvars,nreplaced));
843 std::ofstream out(file);
844 TIter nextLine(&lines);
846 while ( ( s = static_cast<TObjString*>(nextLine()) ) )
848 out << s->String().Data() << std::endl;
853 AliDebug(1,Form("replaced %d vars",nvars));
858 //______________________________________________________________________________
859 const std::vector<int>& AliMuonGridSubmitter::RunList() const
861 /// Return a reference to our runlist
865 //______________________________________________________________________________
866 void AliMuonGridSubmitter::SetPackages(const char* aliroot,
871 /// Set the packages to be used by the jobs
872 /// If root and geant3 are given (default is to let alien get the correct
873 /// values for the requested aliroot version), then they must
874 /// correspond to a valid combination, see http://alimonitor.cern.ch/packages/
876 SetMapKeyValue("AliRoot",aliroot);
877 SetMapKeyValue("Root",root);
878 SetMapKeyValue("Geant3",geant3);
879 SetMapKeyValue("API",api);
882 //______________________________________________________________________________
883 TString AliMuonGridSubmitter::GetRemoteDir(const char* dir, Bool_t create)
885 /// Set the target remote directory (on the grid)
887 if (!RemoteDirectoryExists(dir))
891 AliErrorClass(Form("Remote directory %s does not exist", dir));
896 AliInfoClass(Form("Remote directory %s does not exist. Trying to create it...",dir));
899 AliErrorClass("cannot connect to grid");
902 if ( !gGrid->Mkdir(dir,"-p") )
904 AliErrorClass(Form("Could not create remote dir. Sorry."));
912 //______________________________________________________________________________
913 Bool_t AliMuonGridSubmitter::SetLocalDirectory(const char* type, const char* path)
915 if (gSystem->AccessPathName(path)==kFALSE)
917 SetMapKeyValue(type,path);
923 //______________________________________________________________________________
924 void AliMuonGridSubmitter::SetMapKeyValue(const char* key, const char* value)
926 TObjString skey(key);
927 InternalMap()->Remove(&skey);
928 InternalMap()->Add(new TObjString(key),new TObjString(value));
931 //______________________________________________________________________________
932 Bool_t AliMuonGridSubmitter::SetRemoteDirectory(const char* type, const char* path)
934 // Set the merged directory to be used
935 TString v = GetRemoteDir(path,kTRUE);
936 SetMapKeyValue(type,v);
937 return (v.Length()>0);
940 //______________________________________________________________________________
941 void AliMuonGridSubmitter::SetRunList(const char* runlist)
943 // set the runlist from a text file
944 AliAnalysisTriggerScalers ts(runlist);
945 fRunList = ts.GetRunList();
948 //______________________________________________________________________________
949 void AliMuonGridSubmitter::SetRunList(int runNumber)
951 // set the runlist from a text file
953 fRunList.push_back(runNumber);
956 //______________________________________________________________________________
957 TString AliMuonGridSubmitter::GetVar(const char* key) const
959 TObjString* o = static_cast<TObjString*>(Vars()->GetValue(key));
967 //______________________________________________________________________________
968 Bool_t AliMuonGridSubmitter::SetVar(const char* varname, const char* value)
972 /// Pay attention to how string variables should be given here : you have
973 /// to espace the quotation marks :
975 /// SetVar("VAR_PYTHIA8_SETUP_STRINGS","\"SoftQCD:doubleDiffractive=off\"");
979 if (!s.BeginsWith("VAR_"))
981 AliError("Variable name should start with VAR_");
987 fVars->SetOwnerKeyValue(kTRUE,kTRUE);
990 TObject* o = new TObjString(s);
993 fVars->Add(o,new TObjString(value));
998 //______________________________________________________________________________
999 TObjArray* AliMuonGridSubmitter::TemplateFileList() const
1001 /// Return (after createing if needed)
1003 if (!fTemplateFileList)
1005 fTemplateFileList = new TObjArray;
1006 fTemplateFileList->SetOwner(kTRUE);
1008 return fTemplateFileList;
1011 //______________________________________________________________________________
1012 void AliMuonGridSubmitter::UpdateLocalFileList()
1014 // insure that local file list contains at least all of the template files
1015 TIter next(TemplateFileList());
1018 while ( ( s = static_cast<TObjString*>(next())) )
1020 AddToLocalFileList(s->String().Data());
1024 //______________________________________________________________________________
1025 TMap* AliMuonGridSubmitter::Vars() const
1030 fVars->SetOwnerKeyValue(kTRUE,kTRUE);