1 /**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
16 // Author: Mihaela Gheata, 01/09/2008
18 //==============================================================================
19 // AliAnalysisAlien - AliEn utility class. Provides interface for creating
20 // a personalized JDL, finding and creating a dataset.
21 //==============================================================================
23 #include "Riostream.h"
27 #include "TObjString.h"
28 #include "TObjArray.h"
30 #include "TGridResult.h"
31 #include "TGridCollection.h"
33 #include "TFileMerger.h"
34 #include "AliAnalysisManager.h"
35 #include "AliVEventHandler.h"
36 #include "AliAnalysisDataContainer.h"
37 #include "AliAnalysisAlien.h"
39 ClassImp(AliAnalysisAlien)
41 //______________________________________________________________________________
42 AliAnalysisAlien::AliAnalysisAlien()
47 fSplitMaxInputFileNumber(0),
49 fMasterResubmitThreshold(0),
81 //______________________________________________________________________________
82 AliAnalysisAlien::AliAnalysisAlien(const char *name)
83 :AliAnalysisGrid(name),
87 fSplitMaxInputFileNumber(0),
89 fMasterResubmitThreshold(0),
121 //______________________________________________________________________________
122 AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
123 :AliAnalysisGrid(other),
125 fPrice(other.fPrice),
127 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
128 fMaxInitFailed(other.fMaxInitFailed),
129 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
130 fNtestFiles(other.fNtestFiles),
131 fRunNumbers(other.fRunNumbers),
132 fExecutable(other.fExecutable),
133 fArguments(other.fArguments),
134 fAnalysisMacro(other.fAnalysisMacro),
135 fAnalysisSource(other.fAnalysisSource),
136 fAdditionalLibs(other.fAdditionalLibs),
137 fSplitMode(other.fSplitMode),
138 fAPIVersion(other.fAPIVersion),
139 fROOTVersion(other.fROOTVersion),
140 fAliROOTVersion(other.fAliROOTVersion),
142 fGridWorkingDir(other.fGridWorkingDir),
143 fGridDataDir(other.fGridDataDir),
144 fDataPattern(other.fDataPattern),
145 fGridOutputDir(other.fGridOutputDir),
146 fOutputArchive(other.fOutputArchive),
147 fOutputFiles(other.fOutputFiles),
148 fInputFormat(other.fInputFormat),
149 fDatasetName(other.fDatasetName),
150 fJDLName(other.fJDLName),
151 fMergeExcludes(other.fMergeExcludes),
152 fIncludePath(other.fIncludePath),
153 fCloseSE(other.fCloseSE),
158 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
159 if (other.fInputFiles) {
160 fInputFiles = new TObjArray();
161 TIter next(other.fInputFiles);
163 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
164 fInputFiles->SetOwner();
166 if (other.fPackages) {
167 fPackages = new TObjArray();
168 TIter next(other.fPackages);
170 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
171 fPackages->SetOwner();
175 //______________________________________________________________________________
176 AliAnalysisAlien::~AliAnalysisAlien()
179 if (fGridJDL) delete fGridJDL;
180 if (fInputFiles) delete fInputFiles;
181 if (fPackages) delete fPackages;
184 //______________________________________________________________________________
185 AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
188 if (this != &other) {
189 AliAnalysisGrid::operator=(other);
190 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
191 fPrice = other.fPrice;
193 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
194 fMaxInitFailed = other.fMaxInitFailed;
195 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
196 fNtestFiles = other.fNtestFiles;
197 fRunNumbers = other.fRunNumbers;
198 fExecutable = other.fExecutable;
199 fArguments = other.fArguments;
200 fAnalysisMacro = other.fAnalysisMacro;
201 fAnalysisSource = other.fAnalysisSource;
202 fAdditionalLibs = other.fAdditionalLibs;
203 fSplitMode = other.fSplitMode;
204 fAPIVersion = other.fAPIVersion;
205 fROOTVersion = other.fROOTVersion;
206 fAliROOTVersion = other.fAliROOTVersion;
208 fGridWorkingDir = other.fGridWorkingDir;
209 fGridDataDir = other.fGridDataDir;
210 fDataPattern = other.fDataPattern;
211 fGridOutputDir = other.fGridOutputDir;
212 fOutputArchive = other.fOutputArchive;
213 fOutputFiles = other.fOutputFiles;
214 fInputFormat = other.fInputFormat;
215 fDatasetName = other.fDatasetName;
216 fJDLName = other.fJDLName;
217 fMergeExcludes = other.fMergeExcludes;
218 fIncludePath = other.fIncludePath;
219 fCloseSE = other.fCloseSE;
220 if (other.fInputFiles) {
221 fInputFiles = new TObjArray();
222 TIter next(other.fInputFiles);
224 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
225 fInputFiles->SetOwner();
227 if (other.fPackages) {
228 fPackages = new TObjArray();
229 TIter next(other.fPackages);
231 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
232 fPackages->SetOwner();
238 //______________________________________________________________________________
239 void AliAnalysisAlien::AddIncludePath(const char *path)
241 // Add include path in the remote analysis macro.
243 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
244 else fIncludePath += Form("-I%s ", path);
247 //______________________________________________________________________________
248 void AliAnalysisAlien::AddRunNumber(Int_t run)
250 // Add a run number to the list of runs to be processed.
251 if (fRunNumbers.Length()) fRunNumbers += " ";
252 fRunNumbers += Form("%d", run);
255 //______________________________________________________________________________
256 void AliAnalysisAlien::AddDataFile(const char *lfn)
258 // Adds a data file to the input to be analysed. The file should be a valid LFN
259 // or point to an existing file in the alien workdir.
260 if (!fInputFiles) fInputFiles = new TObjArray();
261 fInputFiles->Add(new TObjString(lfn));
264 //______________________________________________________________________________
265 Bool_t AliAnalysisAlien::Connect()
267 // Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
268 if (gGrid && gGrid->IsConnected()) return kTRUE;
269 if (!gSystem->Getenv("alien_API_USER")) {
270 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
271 gSystem->Getenv("UID"));
275 Info("Connect", "Trying to connect to AliEn ...");
276 TGrid::Connect("alien://");
278 if (!gGrid || !gGrid->IsConnected()) {
279 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
282 fUser = gGrid->GetUser();
283 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
287 //______________________________________________________________________________
288 void AliAnalysisAlien::CdWork()
290 // Check validity of alien workspace. Create directory if possible.
292 Error("CdWork", "Alien connection required");
295 TString homedir = gGrid->GetHomeDirectory();
296 TString workdir = homedir + fGridWorkingDir;
297 if (!gGrid->Cd(workdir)) {
299 if (gGrid->Mkdir(workdir)) {
300 gGrid->Cd(fGridWorkingDir);
301 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
303 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
304 workdir.Data(), homedir.Data());
305 fGridWorkingDir = "";
310 //______________________________________________________________________________
311 Bool_t AliAnalysisAlien::CheckInputData()
313 // Check validity of input data. If necessary, create xml files.
314 if (!fInputFiles && !fRunNumbers.Length()) {
315 Error("CheckInputData", "You have to specify either a set of run numbers or some existing grid files. Use AddRunNumber()/AddDataFile().");
318 // Process declared files
319 Bool_t is_collection = kFALSE;
320 Bool_t is_xml = kFALSE;
321 Bool_t use_tags = kFALSE;
322 Bool_t checked = kFALSE;
325 TString workdir = gGrid->GetHomeDirectory();
326 workdir += fGridWorkingDir;
329 TIter next(fInputFiles);
330 while ((objstr=(TObjString*)next())) {
333 file += objstr->GetString();
334 // Store full lfn path
335 if (FileExists(file)) objstr->SetString(file);
337 file = objstr->GetName();
338 if (!FileExists(objstr->GetName())) {
339 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
340 objstr->GetName(), workdir.Data());
344 Bool_t iscoll, isxml, usetags;
345 CheckDataType(file, iscoll, isxml, usetags);
348 is_collection = iscoll;
351 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
353 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
354 Error("CheckInputData", "Some conflict was found in the types of inputs");
360 // Process requested run numbers
361 if (!fRunNumbers.Length()) return kTRUE;
362 // Check validity of alien data directory
363 if (!fGridDataDir.Length()) {
364 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
367 if (!gGrid->Cd(fGridDataDir)) {
368 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
372 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
376 if (checked && !is_xml) {
377 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
380 // Check validity of run number(s)
386 use_tags = fDataPattern.Contains("tag");
387 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
389 if (use_tags != fDataPattern.Contains("tag")) {
390 Error("CheckInputData", "Cannot mix input files using/not using tags");
393 if (fRunNumbers.Length()) {
394 arr = fRunNumbers.Tokenize(" ");
396 while ((os=(TObjString*)next())) {
397 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
398 if (!gGrid->Cd(path)) {
399 Error("CheckInputData", "Run number %s not found in path: %s", os->GetString().Data(), path.Data());
402 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
403 TString msg = "\n##### file: ";
405 msg += " type: xml_collection;";
406 if (use_tags) msg += " using_tags: Yes";
407 else msg += " using_tags: No";
408 Info("CheckDataType", msg.Data());
416 //______________________________________________________________________________
417 Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
419 // Create dataset for the grid data directory + run number.
420 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
422 Error("CreateDataset", "Cannot create dataset with no grid connection");
428 TString workdir = gGrid->GetHomeDirectory();
429 workdir += fGridWorkingDir;
431 // Compose the 'find' command arguments
433 TString options = "-x collection ";
434 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
435 TString conditions = "";
439 if (!fRunNumbers.Length()) return kTRUE;
441 TObjArray *arr = fRunNumbers.Tokenize(" ");
444 while ((os=(TObjString*)next())) {
445 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
446 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
447 else file = Form("%s.xml", os->GetString().Data());
448 if (FileExists(file) && !TestBit(AliAnalysisGrid::kTest)) {
449 Info("CreateDataset", "\n##### Removing previous dataset %s", file.Data());
456 // conditions = Form(" > %s", file.Data());
457 command += conditions;
458 TGridResult *res = gGrid->Command(command);
460 // Write standard output to file
461 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
462 if (TestBit(AliAnalysisGrid::kTest)) break;
463 // Copy xml file to alien space
464 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
465 if (!FileExists(file)) {
466 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
475 //______________________________________________________________________________
476 Bool_t AliAnalysisAlien::CreateJDL()
478 // Generate a JDL file according to current settings. The name of the file is
479 // specified by fJDLName.
480 Bool_t error = kFALSE;
483 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
484 Bool_t generate = kTRUE;
485 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
487 Error("CreateJDL", "Alien connection required");
490 // Check validity of alien workspace
492 TString workdir = gGrid->GetHomeDirectory();
493 workdir += fGridWorkingDir;
497 Error("CreateJDL()", "Define some input files for your analysis.");
500 // Compose list of input files
501 // Check if output files were defined
502 if (!fOutputFiles.Length()) {
503 Error("CreateJDL", "You must define at least one output file");
506 // Check if an output directory was defined and valid
507 if (!fGridOutputDir.Length()) {
508 Error("CreateJDL", "You must define AliEn output directory");
511 if (!gGrid->Cd(fGridOutputDir)) {
512 if (gGrid->Mkdir(fGridOutputDir)) {
513 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
515 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
521 // Exit if any error up to now
522 if (error) return kFALSE;
524 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
525 fGridJDL->SetExecutable(fExecutable);
526 // fGridJDL->SetTTL((UInt_t)fTTL);
527 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
528 if (fMaxInitFailed > 0)
529 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
530 if (fSplitMaxInputFileNumber > 0)
531 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
532 if (fSplitMode.Length())
533 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
534 // fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
535 if (fAliROOTVersion.Length())
536 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
537 if (fROOTVersion.Length())
538 fGridJDL->AddToPackages("ROOT", fROOTVersion);
539 if (fAPIVersion.Length())
540 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
541 fGridJDL->SetInputDataListFormat(fInputFormat);
542 fGridJDL->SetInputDataList("wn.xml");
544 TIter next(fInputFiles);
545 while ((os=(TObjString*)next()))
546 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
548 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
549 fGridJDL->AddToInputSandbox(Form("LF:%s/analysis.root", workdir.Data()));
550 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
551 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
552 if (fAdditionalLibs.Length()) {
553 arr = fAdditionalLibs.Tokenize(" ");
555 while ((os=(TObjString*)next())) {
556 if (os->GetString().Contains(".so")) continue;
557 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
562 TIter next(fPackages);
565 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
567 if (fOutputArchive.Length()) {
568 arr = fOutputArchive.Tokenize(" ");
570 while ((os=(TObjString*)next()))
571 if (!os->GetString().Contains("@") && fCloseSE.Length())
572 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
574 fGridJDL->AddToOutputArchive(os->GetString());
577 fGridJDL->SetOutputDirectory(Form("%s/%s/#alien_counter_03i#", workdir.Data(), fGridOutputDir.Data()));
578 arr = fOutputFiles.Tokenize(" ");
580 while ((os=(TObjString*)next())) {
581 if (!os->GetString().Contains("@") && fCloseSE.Length())
582 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
584 fGridJDL->AddToOutputSandbox(os->GetString());
587 // fGridJDL->SetPrice((UInt_t)fPrice);
588 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
589 fGridJDL->SetValidationCommand(Form("%s/validate.sh", workdir.Data()));
590 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
591 // Generate the JDL as a string
592 TString sjdl = fGridJDL->Generate();
594 index = sjdl.Index("Executable");
595 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
596 index = sjdl.Index("Split ");
597 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
598 index = sjdl.Index("SplitMaxInputFileNumber");
599 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
600 index = sjdl.Index("InputDataCollection");
601 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
602 index = sjdl.Index("InputFile");
603 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
604 index = sjdl.Index("InputDataList ");
605 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
606 index = sjdl.Index("InputDataListFormat");
607 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
608 index = sjdl.Index("Price");
609 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
610 index = sjdl.Index("Requirements");
611 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
612 index = sjdl.Index("Packages");
613 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
614 index = sjdl.Index("User");
615 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
616 index = sjdl.Index("TTL");
617 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
618 index = sjdl.Index("OutputFile");
619 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
620 index = sjdl.Index("OutputDir");
621 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
622 index = sjdl.Index("OutputArchive");
623 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
624 index = sjdl.Index("MaxInitFailed");
625 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
626 index = sjdl.Index("MasterResubmitThreshold");
627 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
628 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
629 index = sjdl.Index("Validationcommand");
630 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
631 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
632 sjdl.ReplaceAll("(member", "\n (member");
633 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
634 sjdl.ReplaceAll("{", "{\n ");
635 sjdl.ReplaceAll("};", "\n};");
636 sjdl.ReplaceAll("{\n \n", "{\n");
637 sjdl.ReplaceAll("\n\n", "\n");
638 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
639 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
640 sjdl.Prepend("JobTag = \"Automatically generated analysis JDL\";\n");
641 index = sjdl.Index("JDLVariables");
642 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
645 out.open(fJDLName.Data(), ios::out);
647 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
652 // Copy jdl to grid workspace
654 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
656 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn working space", fJDLName.Data());
657 if (FileExists(fJDLName)) gGrid->Rm(fJDLName);
658 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s/%s", workdir.Data(), fJDLName.Data()));
659 if (fAdditionalLibs.Length()) {
660 arr = fAdditionalLibs.Tokenize(" ");
663 while ((os=(TObjString*)next())) {
664 if (os->GetString().Contains(".so")) continue;
665 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
666 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
667 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
672 TIter next(fPackages);
674 while ((obj=next())) {
675 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
676 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
683 //______________________________________________________________________________
684 Bool_t AliAnalysisAlien::FileExists(const char *lfn) const
686 // Returns true if file exists.
688 Error("FileExists", "No connection to grid");
691 TGridResult *res = gGrid->Ls(lfn);
692 if (!res) return kFALSE;
693 TMap *map = dynamic_cast<TMap*>(res->At(0));
698 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
699 if (!objs || !objs->GetString().Length()) {
707 //______________________________________________________________________________
708 void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
710 // Check input data type.
711 is_collection = kFALSE;
715 Error("CheckDataType", "No connection to grid");
718 is_collection = IsCollection(lfn);
719 TString msg = "\n##### file: ";
722 msg += " type: raw_collection;";
723 // special treatment for collections
725 // check for tag files in the collection
726 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
728 msg += " using_tags: No (unknown)";
729 Info("CheckDataType", msg.Data());
732 const char* typeStr = res->GetKey(0, "origLFN");
733 if (!typeStr || !strlen(typeStr)) {
734 msg += " using_tags: No (unknown)";
735 Info("CheckDataType", msg.Data());
738 TString file = typeStr;
739 use_tags = file.Contains(".tag");
740 if (use_tags) msg += " using_tags: Yes";
741 else msg += " using_tags: No";
742 Info("CheckDataType", msg.Data());
747 is_xml = slfn.Contains(".xml");
749 // Open xml collection and check if there are tag files inside
750 msg += " type: xml_collection;";
751 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
753 msg += " using_tags: No (unknown)";
754 Info("CheckDataType", msg.Data());
757 TMap *map = coll->Next();
759 msg += " using_tags: No (unknown)";
760 Info("CheckDataType", msg.Data());
763 map = (TMap*)map->GetValue("");
765 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
766 use_tags = file.Contains(".tag");
768 if (use_tags) msg += " using_tags: Yes";
769 else msg += " using_tags: No";
770 Info("CheckDataType", msg.Data());
773 use_tags = slfn.Contains(".tag");
774 if (slfn.Contains(".root")) msg += " type: root file;";
775 else msg += " type: unhnown file;";
776 if (use_tags) msg += " using_tags: Yes";
777 else msg += " using_tags: No";
778 Info("CheckDataType", msg.Data());
781 //______________________________________________________________________________
782 void AliAnalysisAlien::EnablePackage(const char *package)
784 // Enables a par file supposed to exist in the current directory.
785 TString pkg(package);
786 pkg.ReplaceAll(".par", "");
788 if (gSystem->AccessPathName(pkg)) {
789 Error("EnablePackage", "Package %s not found", pkg.Data());
792 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
793 Info("EnablePackage", "AliEn plugin will use .par packages");
794 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
796 fPackages = new TObjArray();
797 fPackages->SetOwner();
799 fPackages->Add(new TObjString(pkg));
802 //______________________________________________________________________________
803 Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
805 // Returns true if file is a collection. Functionality duplicated from
806 // TAlien::Type() because we don't want to directly depend on TAlien.
808 Error("IsCollection", "No connection to grid");
811 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
812 if (!res) return kFALSE;
813 const char* typeStr = res->GetKey(0, "type");
814 if (!typeStr || !strlen(typeStr)) return kFALSE;
815 if (!strcmp(typeStr, "collection")) return kTRUE;
820 //______________________________________________________________________________
821 void AliAnalysisAlien::SetDefaults()
823 // Set default values for everything. What cannot be filled will be left empty.
824 if (fGridJDL) delete fGridJDL;
825 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
828 fSplitMaxInputFileNumber = 100;
830 fMasterResubmitThreshold = 0;
833 fExecutable = "analysis.sh";
835 fAnalysisMacro = "myAnalysis.C";
836 fAnalysisSource = "";
837 fAdditionalLibs = "";
841 fAliROOTVersion = "";
842 fUser = ""; // Your alien user name
843 fGridWorkingDir = "";
844 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
845 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
846 fGridOutputDir = "output";
847 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
848 fOutputFiles = ""; // Like "AliAODs.root histos.root"
849 fInputFormat = "xml-single";
850 fJDLName = "analysis.jdl";
854 //______________________________________________________________________________
855 Bool_t AliAnalysisAlien::MergeOutputs()
857 // Merge analysis outputs existing in the AliEn space.
858 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
859 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
861 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
864 // Get the output path
865 TString output = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
866 if (!gGrid->Cd(output)) output = Form("/%s/%s", gGrid->GetHomeDirectory(), fGridOutputDir.Data());
867 if (!gGrid->Cd(output)) {
868 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
871 if (!fOutputFiles.Length()) {
872 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
875 TObjArray *list = fOutputFiles.Tokenize(" ");
880 Bool_t merged = kTRUE;
881 while((str=(TObjString*)next())) {
882 output_file = str->GetString();
883 Int_t index = output_file.Index("@");
884 if (index > 0) output_file.Remove(index);
885 if (fMergeExcludes.Length() &&
886 fMergeExcludes.Contains(output_file.Data())) continue;
887 command = Form("find %s/ *%s", output.Data(), output_file.Data());
888 printf("command: %s\n", command.Data());
889 TGridResult *res = gGrid->Command(command);
894 while ((map=(TMap*)nextmap())) {
895 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
896 if (!objs || !objs->GetString().Length()) {
901 fm = new TFileMerger(kFALSE);
902 fm->SetFastMethod(kTRUE);
903 fm->OutputFile(output_file);
905 fm->AddFile(objs->GetString());
907 if (!fm || !fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
908 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
914 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
917 Info("MergeOutputs", "\n##### Merged %d output files <%s>", fm->GetMergeList()->GetSize(), output_file.Data());
923 Error("MergeOutputs", "Terminate() will NOT be executed");
928 //______________________________________________________________________________
929 void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
931 // Use the output files connected to output containers from the analysis manager
932 // rather than the files defined by SetOutputFiles
933 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
934 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
936 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
939 //______________________________________________________________________________
940 void AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
942 // Start remote grid analysis.
944 // Check if output files have to be taken from the analysis manager
945 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
946 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
947 if (!mgr || !mgr->IsInitialized()) {
948 Error("StartAnalysis", "You need an initialized analysis manager for this");
952 TIter next(mgr->GetOutputs());
953 AliAnalysisDataContainer *output;
954 while ((output=(AliAnalysisDataContainer*)next())) {
955 const char *filename = output->GetFileName();
956 if (!(strcmp(filename, "default"))) {
957 if (!mgr->GetOutputEventHandler()) continue;
958 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
960 if (fOutputFiles.Length()) fOutputFiles += " ";
961 fOutputFiles += filename;
964 // if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
965 if (TestBit(AliAnalysisGrid::kOffline)) {
966 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
967 \n there nor any job run. You can revise the JDL and analysis \
968 \n macro then run the same in \"submit\" mode.");
969 } else if (TestBit(AliAnalysisGrid::kTest)) {
970 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
972 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
973 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
974 \n space and job submitted.");
975 } else if (TestBit(AliAnalysisGrid::kMerge)) {
976 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
979 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
983 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
986 if (!CheckInputData()) {
987 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
990 CreateDataset(fDataPattern);
992 WriteAnalysisMacro();
994 WriteValidationScript();
995 if (!CreateJDL()) return;
996 if (TestBit(AliAnalysisGrid::kOffline)) return;
997 if (TestBit(AliAnalysisGrid::kTest)) {
998 // Locally testing the analysis
999 Info("StartAnalysis", "\n_______________________________________________________________________ \
1000 \n Running analysis script in a daughter shell as on a worker node \
1001 \n_______________________________________________________________________");
1002 TObjArray *list = fOutputFiles.Tokenize(" ");
1005 TString output_file;
1006 while((str=(TObjString*)next())) {
1007 output_file = str->GetString();
1008 Int_t index = output_file.Index("@");
1009 if (index > 0) output_file.Remove(index);
1010 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
1013 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
1014 gSystem->Exec("bash validate.sh");
1015 // gSystem->Exec("cat stdout");
1020 TGridResult *res = gGrid->Command(Form("submit %s", fJDLName.Data()));
1023 const char *cjobId = res->GetKey(0,"jobId");
1025 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1028 Info("StartAnalysis", "\n_______________________________________________________________________ \
1029 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1030 \n_______________________________________________________________________",
1031 fJDLName.Data(), cjobId);
1036 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1037 \n You may exit at any time and terminate the job later using the option <terminate> \
1038 \n ##################################################################################", jobID.Data());
1040 gSystem->Exec("aliensh");
1043 //______________________________________________________________________________
1044 void AliAnalysisAlien::WriteAnalysisFile()
1046 // Write current analysis manager into the file analysis.root
1047 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1048 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1049 if (!mgr || !mgr->IsInitialized()) {
1050 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1053 // Check analysis type
1055 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1056 handler = (TObject*)mgr->GetInputEventHandler();
1058 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1059 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1061 TDirectory *cdir = gDirectory;
1062 TFile *file = TFile::Open("analysis.root", "RECREATE");
1067 if (cdir) cdir->cd();
1068 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <analysis.root>\n", mgr->GetName());
1070 Bool_t copy = kTRUE;
1071 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1074 TString workdir = gGrid->GetHomeDirectory();
1075 workdir += fGridWorkingDir;
1076 Info("CreateJDL", "\n##### Copying file <analysis.root> containing your initialized analysis manager to your alien workspace");
1077 if (FileExists("analysis.root")) gGrid->Rm("analysis.root");
1078 TFile::Cp("file:analysis.root", Form("alien://%s/analysis.root", workdir.Data()));
1082 //______________________________________________________________________________
1083 void AliAnalysisAlien::WriteAnalysisMacro()
1085 // Write the analysis macro that will steer the analysis in grid mode.
1086 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1088 out.open(fAnalysisMacro.Data(), ios::out);
1090 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1093 TString func = fAnalysisMacro;
1094 TString type = "ESD";
1095 TString comment = "// Analysis using ";
1096 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1097 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1101 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1102 else comment += " data";
1103 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1104 func.ReplaceAll(".C", "");
1105 out << "void " << func.Data() << "()" << endl;
1107 out << comment.Data() << endl;
1108 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
1109 out << "// load base root libraries" << endl;
1110 out << " gSystem->Load(\"libTree\");" << endl;
1111 out << " gSystem->Load(\"libGeom\");" << endl;
1112 out << " gSystem->Load(\"libVMC\");" << endl;
1113 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
1114 out << "// Load analysis framework libraries" << endl;
1116 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1117 out << " gSystem->Load(\"libESD\");" << endl;
1118 out << " gSystem->Load(\"libAOD\");" << endl;
1119 out << " gSystem->Load(\"libANALYSIS\");" << endl;
1120 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1121 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1123 TIter next(fPackages);
1126 Bool_t hasSTEERBase = kFALSE;
1127 Bool_t hasESD = kFALSE;
1128 Bool_t hasAOD = kFALSE;
1129 Bool_t hasANALYSIS = kFALSE;
1130 Bool_t hasANALYSISalice = kFALSE;
1131 Bool_t hasCORRFW = kFALSE;
1132 while ((obj=next())) {
1133 pkgname = obj->GetName();
1134 if (pkgname.Contains("STEERBase")) hasSTEERBase = kTRUE;
1135 if (pkgname.Contains("ESD")) hasESD = kTRUE;
1136 if (pkgname.Contains("AOD")) hasAOD = kTRUE;
1137 if (pkgname.Contains("ANALYSIS") && !pkgname.Contains("ANALYSISalice")) hasANALYSIS = kTRUE;
1138 if (pkgname.Contains("ANALYSISalice")) hasANALYSISalice = kTRUE;
1139 if (pkgname.Contains("CORRFW")) hasCORRFW = kTRUE;
1141 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1142 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1143 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1144 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1145 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1146 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1147 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1148 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1149 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1150 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1151 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1152 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1153 out << "// Compile other par packages" << endl;
1155 while ((obj=next())) {
1156 pkgname = obj->GetName();
1157 if (pkgname.Contains("STEERBase") ||
1158 pkgname.Contains("ESD") ||
1159 pkgname.Contains("AOD") ||
1160 pkgname.Contains("ANALYSIS") ||
1161 pkgname.Contains("CORRFW")) continue;
1162 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
1165 out << "// include path" << endl;
1166 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1167 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
1168 if (fAdditionalLibs.Length()) {
1169 out << "// Add aditional AliRoot libraries" << endl;
1170 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1173 while((str=(TObjString*)next())) {
1174 if (str->GetString().Contains(".so"))
1175 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1177 if (list) delete list;
1180 out << "// analysis source to be compiled at runtime (if any)" << endl;
1181 if (fAnalysisSource.Length()) {
1182 TObjArray *list = fAnalysisSource.Tokenize(" ");
1185 while((str=(TObjString*)next())) {
1186 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1188 if (list) delete list;
1191 out << "// connect to AliEn and make the chain" << endl;
1192 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1193 if (IsUsingTags()) {
1194 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1196 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1198 out << "// read the analysis manager from file" << endl;
1199 out << " TFile *file = TFile::Open(\"analysis.root\");" << endl;
1200 out << " if (!file) return;" << endl;
1201 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1202 out << " AliAnalysisManager *mgr = 0;" << endl;
1203 out << " TKey *key;" << endl;
1204 out << " while ((key=(TKey*)nextkey())) {" << endl;
1205 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1206 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1207 out << " };" << endl;
1208 out << " if (!mgr) {" << endl;
1209 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file analysis.root\");" << endl;
1210 out << " return;" << endl;
1211 out << " }" << endl << endl;
1212 out << " mgr->PrintStatus();" << endl;
1213 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
1214 out << "}" << endl << endl;
1215 if (IsUsingTags()) {
1216 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1218 out << "// Create a chain using tags from the xml file." << endl;
1219 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1220 out << " if (!coll) {" << endl;
1221 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1222 out << " return NULL;" << endl;
1223 out << " }" << endl;
1224 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1225 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1226 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1227 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1228 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1229 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1230 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1231 out << " // Check if the cuts configuration file was provided" << endl;
1232 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1233 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1234 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1235 out << " }" << endl;
1236 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1237 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1238 out << " chain->ls();" << endl;
1239 out << " return chain;" << endl;
1240 out << "}" << endl << endl;
1241 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1242 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1243 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1244 msg += " AliLHCTagCuts *lhcCuts,\n";
1245 msg += " AliDetectorTagCuts *detCuts,\n";
1246 msg += " AliEventTagCuts *evCuts)";
1247 Info("WriteAnalysisMacro", msg.Data());
1250 out <<"//________________________________________________________________________________" << endl;
1251 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1253 out << "// Create a chain using url's from xml file" << endl;
1254 out << " TString treename = type;" << endl;
1255 out << " treename.ToLower();" << endl;
1256 out << " treename += \"Tree\";" << endl;
1257 out << " printf(\"***************************************\\n\");" << endl;
1258 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1259 out << " printf(\"***************************************\\n\");" << endl;
1260 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1261 out << " if (!coll) {" << endl;
1262 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1263 out << " return NULL;" << endl;
1264 out << " }" << endl;
1265 out << " TChain *chain = new TChain(treename);" << endl;
1266 out << " coll->Reset();" << endl;
1267 out << " while (coll->Next()) chain->Add(coll->GetTURL(\"\"));" << endl;
1268 out << " if (!chain->GetNtrees()) {" << endl;
1269 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1270 out << " return NULL;" << endl;
1271 out << " }" << endl;
1272 out << " return chain;" << endl;
1273 out << "}" << endl << endl;
1276 out <<"//________________________________________________________________________________" << endl;
1277 out << "Bool_t SetupPar(const char *package) {" << endl;
1278 out << "// Compile the package and set it up." << endl;
1279 out << " TString pkgdir = package;" << endl;
1280 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
1281 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
1282 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1283 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1284 out << " // Check for BUILD.sh and execute" << endl;
1285 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1286 out << " printf(\"*******************************\\n\");" << endl;
1287 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1288 out << " printf(\"*******************************\\n\");" << endl;
1289 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
1290 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
1291 out << " gSystem->ChangeDirectory(cdir);" << endl;
1292 out << " return kFALSE;" << endl;
1293 out << " }" << endl;
1294 out << " } else {" << endl;
1295 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
1296 out << " gSystem->ChangeDirectory(cdir);" << endl;
1297 out << " return kFALSE;" << endl;
1298 out << " }" << endl;
1299 out << " // Check for SETUP.C and execute" << endl;
1300 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1301 out << " printf(\"*******************************\\n\");" << endl;
1302 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1303 out << " printf(\"*******************************\\n\");" << endl;
1304 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1305 out << " } else {" << endl;
1306 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
1307 out << " gSystem->ChangeDirectory(cdir);" << endl;
1308 out << " return kFALSE;" << endl;
1309 out << " }" << endl;
1310 out << " // Restore original workdir" << endl;
1311 out << " gSystem->ChangeDirectory(cdir);" << endl;
1312 out << " return kTRUE;" << endl;
1315 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1317 Bool_t copy = kTRUE;
1318 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1321 TString workdir = gGrid->GetHomeDirectory();
1322 workdir += fGridWorkingDir;
1323 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1324 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1325 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1326 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1327 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1329 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1330 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1334 //______________________________________________________________________________
1335 void AliAnalysisAlien::WriteExecutable()
1337 // Generate the alien executable script.
1338 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1340 out.open(fExecutable.Data(), ios::out);
1342 Error("CreateJDL", "Bad file name for executable: %s", fExecutable.Data());
1345 out << "#!/bin/bash" << endl;
1346 out << "export GCLIENT_SERVER_LIST=\"pcapiserv04.cern.ch:10000|pcapiserv05.cern.ch:10000|pcapiserv06.cern.ch:10000|pcapiserv07.cern.ch:10000\"" << endl;
1347 out << "echo \"=========================================\"" << endl;
1348 out << "echo \"############## PATH : ##############\"" << endl;
1349 out << "echo $PATH" << endl;
1350 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1351 out << "echo $LD_LIBRARY_PATH" << endl;
1352 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1353 out << "echo $ROOTSYS" << endl;
1354 out << "echo \"############## which root : ##############\"" << endl;
1355 out << "which root" << endl;
1356 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1357 out << "echo $ALICE_ROOT" << endl;
1358 out << "echo \"############## which aliroot : ##############\"" << endl;
1359 out << "which aliroot" << endl;
1360 out << "echo \"=========================================\"" << endl << endl;
1361 // if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
1362 out << "root -b -q ";
1363 out << fAnalysisMacro.Data() << endl << endl;
1364 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1366 Bool_t copy = kTRUE;
1367 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1370 TString workdir = gGrid->GetHomeDirectory();
1371 workdir += fGridWorkingDir;
1372 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
1373 if (FileExists(executable)) gGrid->Rm(executable);
1374 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
1375 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
1379 //______________________________________________________________________________
1380 void AliAnalysisAlien::WriteValidationScript()
1382 // Generate the alien validation script.
1383 // Generate the validation script
1386 Error("WriteValidationScript", "Alien connection required");
1389 TString out_stream = "";
1390 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
1391 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1393 out.open("validate.sh", ios::out);
1394 out << "#!/bin/bash" << endl;
1395 out << "##################################################" << endl;
1396 out << "validateout=`dirname $0`" << endl;
1397 out << "validatetime=`date`" << endl;
1398 out << "validated=\"0\";" << endl;
1399 out << "error=0" << endl;
1400 out << "if [ -z $validateout ]" << endl;
1401 out << "then" << endl;
1402 out << " validateout=\".\"" << endl;
1403 out << "fi" << endl << endl;
1404 out << "cd $validateout;" << endl;
1405 out << "validateworkdir=`pwd`;" << endl << endl;
1406 out << "echo \"*******************************************************\"" << out_stream << endl;
1407 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
1409 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
1410 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
1411 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
1412 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1413 out << "ls -la ./" << out_stream << endl;
1414 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
1415 out << "##################################################" << endl;
1416 TObjArray *arr = fOutputFiles.Tokenize(" ");
1418 TString output_file;
1419 while ((os=(TObjString*)next1())) {
1420 output_file = os->GetString();
1421 Int_t index = output_file.Index("@");
1422 if (index > 0) output_file.Remove(index);
1423 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
1424 out << " error=1" << endl;
1425 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
1426 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
1427 out << "fi" << endl;
1430 out << "if [ $error = 0 ] ; then" << endl;
1431 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
1432 out << "fi" << endl;
1434 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1435 out << "echo \"*******************************************************\"" << out_stream << endl;
1436 out << "cd -" << endl;
1437 out << "exit $error" << endl;
1439 Bool_t copy = kTRUE;
1440 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1443 TString workdir = gGrid->GetHomeDirectory();
1444 workdir += fGridWorkingDir;
1445 Info("CreateJDL", "\n##### Copying validation script <validate.sh> to your AliEn working space");
1446 if (FileExists("validate.sh")) gGrid->Rm("validate.sh");
1447 TFile::Cp("file:validate.sh", Form("alien://%s/validate.sh", workdir.Data()));