1 /**************************************************************************
2 * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
4 * Author: The ALICE Off-line Project. *
5 * Contributors are mentioned in the code where appropriate. *
7 * Permission to use, copy, modify and distribute this software and its *
8 * documentation strictly for non-commercial purposes is hereby granted *
9 * without fee, provided that the above copyright notice appears in all *
10 * copies and that both the copyright notice and this permission notice *
11 * appear in the supporting documentation. The authors make no claims *
12 * about the suitability of this software for any purpose. It is *
13 * provided "as is" without express or implied warranty. *
14 **************************************************************************/
16 // Author: Mihaela Gheata, 01/09/2008
18 //==============================================================================
19 // AliAnalysisAlien - AliEn utility class. Provides interface for creating
20 // a personalized JDL, finding and creating a dataset.
21 //==============================================================================
23 #include "Riostream.h"
27 #include "TObjString.h"
28 #include "TObjArray.h"
30 #include "TGridResult.h"
31 #include "TGridCollection.h"
33 #include "TFileMerger.h"
34 #include "AliAnalysisManager.h"
35 #include "AliVEventHandler.h"
36 #include "AliAnalysisDataContainer.h"
37 #include "AliAnalysisAlien.h"
39 ClassImp(AliAnalysisAlien)
41 //______________________________________________________________________________
42 AliAnalysisAlien::AliAnalysisAlien()
47 fSplitMaxInputFileNumber(0),
49 fMasterResubmitThreshold(0),
82 //______________________________________________________________________________
83 AliAnalysisAlien::AliAnalysisAlien(const char *name)
84 :AliAnalysisGrid(name),
88 fSplitMaxInputFileNumber(0),
90 fMasterResubmitThreshold(0),
123 //______________________________________________________________________________
124 AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
125 :AliAnalysisGrid(other),
127 fPrice(other.fPrice),
129 fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
130 fMaxInitFailed(other.fMaxInitFailed),
131 fMasterResubmitThreshold(other.fMasterResubmitThreshold),
132 fNtestFiles(other.fNtestFiles),
133 fRunNumbers(other.fRunNumbers),
134 fExecutable(other.fExecutable),
135 fArguments(other.fArguments),
136 fAnalysisMacro(other.fAnalysisMacro),
137 fAnalysisSource(other.fAnalysisSource),
138 fAdditionalLibs(other.fAdditionalLibs),
139 fSplitMode(other.fSplitMode),
140 fAPIVersion(other.fAPIVersion),
141 fROOTVersion(other.fROOTVersion),
142 fAliROOTVersion(other.fAliROOTVersion),
144 fGridWorkingDir(other.fGridWorkingDir),
145 fGridDataDir(other.fGridDataDir),
146 fDataPattern(other.fDataPattern),
147 fGridOutputDir(other.fGridOutputDir),
148 fOutputArchive(other.fOutputArchive),
149 fOutputFiles(other.fOutputFiles),
150 fInputFormat(other.fInputFormat),
151 fDatasetName(other.fDatasetName),
152 fJDLName(other.fJDLName),
153 fMergeExcludes(other.fMergeExcludes),
154 fIncludePath(other.fIncludePath),
155 fCloseSE(other.fCloseSE),
156 fFriendChainName(other.fFriendChainName),
161 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
162 if (other.fInputFiles) {
163 fInputFiles = new TObjArray();
164 TIter next(other.fInputFiles);
166 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
167 fInputFiles->SetOwner();
169 if (other.fPackages) {
170 fPackages = new TObjArray();
171 TIter next(other.fPackages);
173 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
174 fPackages->SetOwner();
178 //______________________________________________________________________________
179 AliAnalysisAlien::~AliAnalysisAlien()
182 if (fGridJDL) delete fGridJDL;
183 if (fInputFiles) delete fInputFiles;
184 if (fPackages) delete fPackages;
187 //______________________________________________________________________________
188 AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
191 if (this != &other) {
192 AliAnalysisGrid::operator=(other);
193 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
194 fPrice = other.fPrice;
196 fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
197 fMaxInitFailed = other.fMaxInitFailed;
198 fMasterResubmitThreshold = other.fMasterResubmitThreshold;
199 fNtestFiles = other.fNtestFiles;
200 fRunNumbers = other.fRunNumbers;
201 fExecutable = other.fExecutable;
202 fArguments = other.fArguments;
203 fAnalysisMacro = other.fAnalysisMacro;
204 fAnalysisSource = other.fAnalysisSource;
205 fAdditionalLibs = other.fAdditionalLibs;
206 fSplitMode = other.fSplitMode;
207 fAPIVersion = other.fAPIVersion;
208 fROOTVersion = other.fROOTVersion;
209 fAliROOTVersion = other.fAliROOTVersion;
211 fGridWorkingDir = other.fGridWorkingDir;
212 fGridDataDir = other.fGridDataDir;
213 fDataPattern = other.fDataPattern;
214 fGridOutputDir = other.fGridOutputDir;
215 fOutputArchive = other.fOutputArchive;
216 fOutputFiles = other.fOutputFiles;
217 fInputFormat = other.fInputFormat;
218 fDatasetName = other.fDatasetName;
219 fJDLName = other.fJDLName;
220 fMergeExcludes = other.fMergeExcludes;
221 fIncludePath = other.fIncludePath;
222 fCloseSE = other.fCloseSE;
223 fFriendChainName = other.fFriendChainName;
224 if (other.fInputFiles) {
225 fInputFiles = new TObjArray();
226 TIter next(other.fInputFiles);
228 while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
229 fInputFiles->SetOwner();
231 if (other.fPackages) {
232 fPackages = new TObjArray();
233 TIter next(other.fPackages);
235 while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
236 fPackages->SetOwner();
242 //______________________________________________________________________________
243 void AliAnalysisAlien::AddIncludePath(const char *path)
245 // Add include path in the remote analysis macro.
247 if (p.Contains("-I")) fIncludePath += Form("%s ", path);
248 else fIncludePath += Form("-I%s ", path);
251 //______________________________________________________________________________
252 void AliAnalysisAlien::AddRunNumber(Int_t run)
254 // Add a run number to the list of runs to be processed.
255 if (fRunNumbers.Length()) fRunNumbers += " ";
256 fRunNumbers += Form("%d", run);
259 //______________________________________________________________________________
260 void AliAnalysisAlien::AddDataFile(const char *lfn)
262 // Adds a data file to the input to be analysed. The file should be a valid LFN
263 // or point to an existing file in the alien workdir.
264 if (!fInputFiles) fInputFiles = new TObjArray();
265 fInputFiles->Add(new TObjString(lfn));
268 //______________________________________________________________________________
269 Bool_t AliAnalysisAlien::Connect()
271 // Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
272 if (gGrid && gGrid->IsConnected()) return kTRUE;
273 if (!gSystem->Getenv("alien_API_USER")) {
274 Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
275 gSystem->Getenv("UID"));
279 Info("Connect", "Trying to connect to AliEn ...");
280 TGrid::Connect("alien://");
282 if (!gGrid || !gGrid->IsConnected()) {
283 Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
286 fUser = gGrid->GetUser();
287 Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
291 //______________________________________________________________________________
292 void AliAnalysisAlien::CdWork()
294 // Check validity of alien workspace. Create directory if possible.
296 Error("CdWork", "Alien connection required");
299 TString homedir = gGrid->GetHomeDirectory();
300 TString workdir = homedir + fGridWorkingDir;
301 if (!gGrid->Cd(workdir)) {
303 if (gGrid->Mkdir(workdir)) {
304 gGrid->Cd(fGridWorkingDir);
305 Info("CreateJDL", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
307 Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
308 workdir.Data(), homedir.Data());
309 fGridWorkingDir = "";
314 //______________________________________________________________________________
315 Bool_t AliAnalysisAlien::CheckInputData()
317 // Check validity of input data. If necessary, create xml files.
318 if (!fInputFiles && !fRunNumbers.Length()) {
319 Error("CheckInputData", "You have to specify either a set of run numbers or some existing grid files. Use AddRunNumber()/AddDataFile().");
322 // Process declared files
323 Bool_t is_collection = kFALSE;
324 Bool_t is_xml = kFALSE;
325 Bool_t use_tags = kFALSE;
326 Bool_t checked = kFALSE;
329 TString workdir = gGrid->GetHomeDirectory();
330 workdir += fGridWorkingDir;
333 TIter next(fInputFiles);
334 while ((objstr=(TObjString*)next())) {
337 file += objstr->GetString();
338 // Store full lfn path
339 if (FileExists(file)) objstr->SetString(file);
341 file = objstr->GetName();
342 if (!FileExists(objstr->GetName())) {
343 Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
344 objstr->GetName(), workdir.Data());
348 Bool_t iscoll, isxml, usetags;
349 CheckDataType(file, iscoll, isxml, usetags);
352 is_collection = iscoll;
355 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
357 if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
358 Error("CheckInputData", "Some conflict was found in the types of inputs");
364 // Process requested run numbers
365 if (!fRunNumbers.Length()) return kTRUE;
366 // Check validity of alien data directory
367 if (!fGridDataDir.Length()) {
368 Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
371 if (!gGrid->Cd(fGridDataDir)) {
372 Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
376 Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
380 if (checked && !is_xml) {
381 Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
384 // Check validity of run number(s)
390 use_tags = fDataPattern.Contains("tag");
391 TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
393 if (use_tags != fDataPattern.Contains("tag")) {
394 Error("CheckInputData", "Cannot mix input files using/not using tags");
397 if (fRunNumbers.Length()) {
398 arr = fRunNumbers.Tokenize(" ");
400 while ((os=(TObjString*)next())) {
401 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
402 if (!gGrid->Cd(path)) {
403 Error("CheckInputData", "Run number %s not found in path: %s", os->GetString().Data(), path.Data());
406 path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
407 TString msg = "\n##### file: ";
409 msg += " type: xml_collection;";
410 if (use_tags) msg += " using_tags: Yes";
411 else msg += " using_tags: No";
412 Info("CheckDataType", msg.Data());
420 //______________________________________________________________________________
421 Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
423 // Create dataset for the grid data directory + run number.
424 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
426 Error("CreateDataset", "Cannot create dataset with no grid connection");
432 TString workdir = gGrid->GetHomeDirectory();
433 workdir += fGridWorkingDir;
435 // Compose the 'find' command arguments
437 TString options = "-x collection ";
438 if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
439 TString conditions = "";
443 if (!fRunNumbers.Length()) return kTRUE;
445 TObjArray *arr = fRunNumbers.Tokenize(" ");
448 while ((os=(TObjString*)next())) {
449 path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
450 if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
451 else file = Form("%s.xml", os->GetString().Data());
452 if (FileExists(file) && !TestBit(AliAnalysisGrid::kTest)) {
453 Info("CreateDataset", "\n##### Removing previous dataset %s", file.Data());
460 // conditions = Form(" > %s", file.Data());
461 command += conditions;
462 TGridResult *res = gGrid->Command(command);
464 // Write standard output to file
465 gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", file.Data()));
466 if (TestBit(AliAnalysisGrid::kTest)) break;
467 // Copy xml file to alien space
468 TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
469 if (!FileExists(file)) {
470 Error("CreateDataset", "Command %s did NOT succeed", command.Data());
479 //______________________________________________________________________________
480 Bool_t AliAnalysisAlien::CreateJDL()
482 // Generate a JDL file according to current settings. The name of the file is
483 // specified by fJDLName.
484 Bool_t error = kFALSE;
487 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
488 Bool_t generate = kTRUE;
489 if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
491 Error("CreateJDL", "Alien connection required");
494 // Check validity of alien workspace
496 TString workdir = gGrid->GetHomeDirectory();
497 workdir += fGridWorkingDir;
501 Error("CreateJDL()", "Define some input files for your analysis.");
504 // Compose list of input files
505 // Check if output files were defined
506 if (!fOutputFiles.Length()) {
507 Error("CreateJDL", "You must define at least one output file");
510 // Check if an output directory was defined and valid
511 if (!fGridOutputDir.Length()) {
512 Error("CreateJDL", "You must define AliEn output directory");
515 if (!gGrid->Cd(fGridOutputDir)) {
516 if (gGrid->Mkdir(fGridOutputDir)) {
517 Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
519 Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
525 // Exit if any error up to now
526 if (error) return kFALSE;
528 fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
529 fGridJDL->SetExecutable(fExecutable);
530 // fGridJDL->SetTTL((UInt_t)fTTL);
531 fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
532 if (fMaxInitFailed > 0)
533 fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
534 if (fSplitMaxInputFileNumber > 0)
535 fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
536 if (fSplitMode.Length())
537 fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
538 // fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
539 if (fAliROOTVersion.Length())
540 fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
541 if (fROOTVersion.Length())
542 fGridJDL->AddToPackages("ROOT", fROOTVersion);
543 if (fAPIVersion.Length())
544 fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
545 fGridJDL->SetInputDataListFormat(fInputFormat);
546 fGridJDL->SetInputDataList("wn.xml");
548 TIter next(fInputFiles);
549 while ((os=(TObjString*)next()))
550 fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
552 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
553 fGridJDL->AddToInputSandbox(Form("LF:%s/analysis.root", workdir.Data()));
554 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
555 fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
556 if (fAdditionalLibs.Length()) {
557 arr = fAdditionalLibs.Tokenize(" ");
559 while ((os=(TObjString*)next())) {
560 if (os->GetString().Contains(".so")) continue;
561 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
566 TIter next(fPackages);
569 fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
571 if (fOutputArchive.Length()) {
572 arr = fOutputArchive.Tokenize(" ");
574 while ((os=(TObjString*)next()))
575 if (!os->GetString().Contains("@") && fCloseSE.Length())
576 fGridJDL->AddToOutputArchive(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
578 fGridJDL->AddToOutputArchive(os->GetString());
581 fGridJDL->SetOutputDirectory(Form("%s/%s/#alien_counter_03i#", workdir.Data(), fGridOutputDir.Data()));
582 arr = fOutputFiles.Tokenize(" ");
584 while ((os=(TObjString*)next())) {
585 if (!os->GetString().Contains("@") && fCloseSE.Length())
586 fGridJDL->AddToOutputSandbox(Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
588 fGridJDL->AddToOutputSandbox(os->GetString());
591 // fGridJDL->SetPrice((UInt_t)fPrice);
592 fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
593 fGridJDL->SetValidationCommand(Form("%s/validate.sh", workdir.Data()));
594 if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
595 // Generate the JDL as a string
596 TString sjdl = fGridJDL->Generate();
598 index = sjdl.Index("Executable");
599 if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
600 index = sjdl.Index("Split ");
601 if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
602 index = sjdl.Index("SplitMaxInputFileNumber");
603 if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
604 index = sjdl.Index("InputDataCollection");
605 if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
606 index = sjdl.Index("InputFile");
607 if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
608 index = sjdl.Index("InputDataList ");
609 if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
610 index = sjdl.Index("InputDataListFormat");
611 if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
612 index = sjdl.Index("Price");
613 if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
614 index = sjdl.Index("Requirements");
615 if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
616 index = sjdl.Index("Packages");
617 if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
618 index = sjdl.Index("User =");
619 if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
620 index = sjdl.Index("TTL");
621 if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
622 index = sjdl.Index("OutputFile");
623 if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
624 index = sjdl.Index("OutputDir");
625 if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
626 index = sjdl.Index("OutputArchive");
627 if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
628 index = sjdl.Index("MaxInitFailed");
629 if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
630 index = sjdl.Index("MasterResubmitThreshold");
631 if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
632 sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
633 index = sjdl.Index("Validationcommand");
634 if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
635 sjdl.ReplaceAll("\"LF:", "\n \"LF:");
636 sjdl.ReplaceAll("(member", "\n (member");
637 sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
638 sjdl.ReplaceAll("{", "{\n ");
639 sjdl.ReplaceAll("};", "\n};");
640 sjdl.ReplaceAll("{\n \n", "{\n");
641 sjdl.ReplaceAll("\n\n", "\n");
642 sjdl.ReplaceAll("OutputDirectory", "OutputDir");
643 sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
644 sjdl.Prepend("JobTag = \"Automatically generated analysis JDL\";\n");
645 index = sjdl.Index("JDLVariables");
646 if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
649 out.open(fJDLName.Data(), ios::out);
651 Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
656 // Copy jdl to grid workspace
658 Info("CreateJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
660 Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn working space", fJDLName.Data());
661 if (FileExists(fJDLName)) gGrid->Rm(fJDLName);
662 TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s/%s", workdir.Data(), fJDLName.Data()));
663 if (fAdditionalLibs.Length()) {
664 arr = fAdditionalLibs.Tokenize(" ");
667 while ((os=(TObjString*)next())) {
668 if (os->GetString().Contains(".so")) continue;
669 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
670 if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
671 TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
676 TIter next(fPackages);
678 while ((obj=next())) {
679 Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
680 TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
687 //______________________________________________________________________________
688 Bool_t AliAnalysisAlien::FileExists(const char *lfn) const
690 // Returns true if file exists.
692 Error("FileExists", "No connection to grid");
695 TGridResult *res = gGrid->Ls(lfn);
696 if (!res) return kFALSE;
697 TMap *map = dynamic_cast<TMap*>(res->At(0));
702 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
703 if (!objs || !objs->GetString().Length()) {
711 //______________________________________________________________________________
712 void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
714 // Check input data type.
715 is_collection = kFALSE;
719 Error("CheckDataType", "No connection to grid");
722 is_collection = IsCollection(lfn);
723 TString msg = "\n##### file: ";
726 msg += " type: raw_collection;";
727 // special treatment for collections
729 // check for tag files in the collection
730 TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
732 msg += " using_tags: No (unknown)";
733 Info("CheckDataType", msg.Data());
736 const char* typeStr = res->GetKey(0, "origLFN");
737 if (!typeStr || !strlen(typeStr)) {
738 msg += " using_tags: No (unknown)";
739 Info("CheckDataType", msg.Data());
742 TString file = typeStr;
743 use_tags = file.Contains(".tag");
744 if (use_tags) msg += " using_tags: Yes";
745 else msg += " using_tags: No";
746 Info("CheckDataType", msg.Data());
751 is_xml = slfn.Contains(".xml");
753 // Open xml collection and check if there are tag files inside
754 msg += " type: xml_collection;";
755 TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
757 msg += " using_tags: No (unknown)";
758 Info("CheckDataType", msg.Data());
761 TMap *map = coll->Next();
763 msg += " using_tags: No (unknown)";
764 Info("CheckDataType", msg.Data());
767 map = (TMap*)map->GetValue("");
769 if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
770 use_tags = file.Contains(".tag");
772 if (use_tags) msg += " using_tags: Yes";
773 else msg += " using_tags: No";
774 Info("CheckDataType", msg.Data());
777 use_tags = slfn.Contains(".tag");
778 if (slfn.Contains(".root")) msg += " type: root file;";
779 else msg += " type: unhnown file;";
780 if (use_tags) msg += " using_tags: Yes";
781 else msg += " using_tags: No";
782 Info("CheckDataType", msg.Data());
785 //______________________________________________________________________________
786 void AliAnalysisAlien::EnablePackage(const char *package)
788 // Enables a par file supposed to exist in the current directory.
789 TString pkg(package);
790 pkg.ReplaceAll(".par", "");
792 if (gSystem->AccessPathName(pkg)) {
793 Error("EnablePackage", "Package %s not found", pkg.Data());
796 if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
797 Info("EnablePackage", "AliEn plugin will use .par packages");
798 TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
800 fPackages = new TObjArray();
801 fPackages->SetOwner();
803 fPackages->Add(new TObjString(pkg));
806 //______________________________________________________________________________
807 Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
809 // Returns true if file is a collection. Functionality duplicated from
810 // TAlien::Type() because we don't want to directly depend on TAlien.
812 Error("IsCollection", "No connection to grid");
815 TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
816 if (!res) return kFALSE;
817 const char* typeStr = res->GetKey(0, "type");
818 if (!typeStr || !strlen(typeStr)) return kFALSE;
819 if (!strcmp(typeStr, "collection")) return kTRUE;
824 //______________________________________________________________________________
825 void AliAnalysisAlien::SetDefaults()
827 // Set default values for everything. What cannot be filled will be left empty.
828 if (fGridJDL) delete fGridJDL;
829 fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
832 fSplitMaxInputFileNumber = 100;
834 fMasterResubmitThreshold = 0;
837 fExecutable = "analysis.sh";
839 fAnalysisMacro = "myAnalysis.C";
840 fAnalysisSource = "";
841 fAdditionalLibs = "";
845 fAliROOTVersion = "";
846 fUser = ""; // Your alien user name
847 fGridWorkingDir = "";
848 fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
849 fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
850 fFriendChainName = "";
851 fGridOutputDir = "output";
852 fOutputArchive = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
853 fOutputFiles = ""; // Like "AliAODs.root histos.root"
854 fInputFormat = "xml-single";
855 fJDLName = "analysis.jdl";
859 //______________________________________________________________________________
860 Bool_t AliAnalysisAlien::MergeOutputs()
862 // Merge analysis outputs existing in the AliEn space.
863 if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
864 if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
866 Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
869 // Get the output path
870 TString output = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
871 if (!gGrid->Cd(output)) output = Form("/%s/%s", gGrid->GetHomeDirectory(), fGridOutputDir.Data());
872 if (!gGrid->Cd(output)) {
873 Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
876 if (!fOutputFiles.Length()) {
877 Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
880 TObjArray *list = fOutputFiles.Tokenize(" ");
885 Bool_t merged = kTRUE;
886 while((str=(TObjString*)next())) {
887 output_file = str->GetString();
888 Int_t index = output_file.Index("@");
889 if (index > 0) output_file.Remove(index);
890 if (fMergeExcludes.Length() &&
891 fMergeExcludes.Contains(output_file.Data())) continue;
892 command = Form("find %s/ *%s", output.Data(), output_file.Data());
893 printf("command: %s\n", command.Data());
894 TGridResult *res = gGrid->Command(command);
899 while ((map=(TMap*)nextmap())) {
900 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
901 if (!objs || !objs->GetString().Length()) {
906 fm = new TFileMerger(kFALSE);
907 fm->SetFastMethod(kTRUE);
908 fm->OutputFile(output_file);
910 fm->AddFile(objs->GetString());
912 if (!fm || !fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
913 Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
919 Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
922 Info("MergeOutputs", "\n##### Merged %d output files <%s>", fm->GetMergeList()->GetSize(), output_file.Data());
928 Error("MergeOutputs", "Terminate() will NOT be executed");
933 //______________________________________________________________________________
934 void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
936 // Use the output files connected to output containers from the analysis manager
937 // rather than the files defined by SetOutputFiles
938 if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
939 Info("SetDefaultOutputs", "Plugin will use the output files taken from \
941 TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
944 //______________________________________________________________________________
945 void AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
947 // Start remote grid analysis.
949 // Check if output files have to be taken from the analysis manager
950 if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
951 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
952 if (!mgr || !mgr->IsInitialized()) {
953 Error("StartAnalysis", "You need an initialized analysis manager for this");
957 TIter next(mgr->GetOutputs());
958 AliAnalysisDataContainer *output;
959 while ((output=(AliAnalysisDataContainer*)next())) {
960 const char *filename = output->GetFileName();
961 if (!(strcmp(filename, "default"))) {
962 if (!mgr->GetOutputEventHandler()) continue;
963 filename = mgr->GetOutputEventHandler()->GetOutputFileName();
965 if (fOutputFiles.Length()) fOutputFiles += " ";
966 fOutputFiles += filename;
968 // Add extra files registered to the analysis manager
969 if (mgr->GetExtraFiles().Length()) {
970 if (fOutputFiles.Length()) fOutputFiles += " ";
971 fOutputFiles += mgr->GetExtraFiles();
974 // if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
975 if (TestBit(AliAnalysisGrid::kOffline)) {
976 Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
977 \n there nor any job run. You can revise the JDL and analysis \
978 \n macro then run the same in \"submit\" mode.");
979 } else if (TestBit(AliAnalysisGrid::kTest)) {
980 Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
982 } else if (TestBit(AliAnalysisGrid::kSubmit)) {
983 Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
984 \n space and job submitted.");
985 } else if (TestBit(AliAnalysisGrid::kMerge)) {
986 Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
989 Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
993 Error("StartAnalysis", "Cannot start grid analysis without grid connection");
996 if (!CheckInputData()) {
997 Error("StartAnalysis", "There was an error in preprocessing your requested input data");
1000 CreateDataset(fDataPattern);
1001 WriteAnalysisFile();
1002 WriteAnalysisMacro();
1004 WriteValidationScript();
1005 if (!CreateJDL()) return;
1006 if (TestBit(AliAnalysisGrid::kOffline)) return;
1007 if (TestBit(AliAnalysisGrid::kTest)) {
1008 // Locally testing the analysis
1009 Info("StartAnalysis", "\n_______________________________________________________________________ \
1010 \n Running analysis script in a daughter shell as on a worker node \
1011 \n_______________________________________________________________________");
1012 TObjArray *list = fOutputFiles.Tokenize(" ");
1015 TString output_file;
1016 while((str=(TObjString*)next())) {
1017 output_file = str->GetString();
1018 Int_t index = output_file.Index("@");
1019 if (index > 0) output_file.Remove(index);
1020 if (!gSystem->AccessPathName(output_file)) gSystem->Exec(Form("rm %s", output_file.Data()));
1023 gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
1024 gSystem->Exec("bash validate.sh");
1025 // gSystem->Exec("cat stdout");
1030 TGridResult *res = gGrid->Command(Form("submit %s", fJDLName.Data()));
1031 printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
1034 const char *cjobId = res->GetKey(0,"jobId");
1036 Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
1039 Info("StartAnalysis", "\n_______________________________________________________________________ \
1040 \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
1041 \n_______________________________________________________________________",
1042 fJDLName.Data(), cjobId);
1047 Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
1048 \n You may exit at any time and terminate the job later using the option <terminate> \
1049 \n ##################################################################################", jobID.Data());
1051 gSystem->Exec("aliensh");
1054 //______________________________________________________________________________
1055 void AliAnalysisAlien::WriteAnalysisFile()
1057 // Write current analysis manager into the file analysis.root
1058 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1059 AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1060 if (!mgr || !mgr->IsInitialized()) {
1061 Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
1064 // Check analysis type
1066 if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
1067 handler = (TObject*)mgr->GetInputEventHandler();
1069 if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
1070 if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
1072 TDirectory *cdir = gDirectory;
1073 TFile *file = TFile::Open("analysis.root", "RECREATE");
1078 if (cdir) cdir->cd();
1079 Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <analysis.root>\n", mgr->GetName());
1081 Bool_t copy = kTRUE;
1082 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1085 TString workdir = gGrid->GetHomeDirectory();
1086 workdir += fGridWorkingDir;
1087 Info("CreateJDL", "\n##### Copying file <analysis.root> containing your initialized analysis manager to your alien workspace");
1088 if (FileExists("analysis.root")) gGrid->Rm("analysis.root");
1089 TFile::Cp("file:analysis.root", Form("alien://%s/analysis.root", workdir.Data()));
1093 //______________________________________________________________________________
1094 void AliAnalysisAlien::WriteAnalysisMacro()
1096 // Write the analysis macro that will steer the analysis in grid mode.
1097 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1099 out.open(fAnalysisMacro.Data(), ios::out);
1101 Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
1104 TString func = fAnalysisMacro;
1105 TString type = "ESD";
1106 TString comment = "// Analysis using ";
1107 if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
1108 if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
1112 if (type!="AOD" && fFriendChainName!="") {
1113 Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
1116 if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
1117 else comment += " data";
1118 out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
1119 func.ReplaceAll(".C", "");
1120 out << "void " << func.Data() << "()" << endl;
1122 out << comment.Data() << endl;
1123 out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
1124 out << " TStopwatch timer;" << endl;
1125 out << " timer.Start();" << endl << endl;
1126 out << "// load base root libraries" << endl;
1127 out << " gSystem->Load(\"libTree\");" << endl;
1128 out << " gSystem->Load(\"libGeom\");" << endl;
1129 out << " gSystem->Load(\"libVMC\");" << endl;
1130 out << " gSystem->Load(\"libPhysics\");" << endl << endl;
1131 out << "// Load analysis framework libraries" << endl;
1133 out << " gSystem->Load(\"libSTEERBase\");" << endl;
1134 out << " gSystem->Load(\"libESD\");" << endl;
1135 out << " gSystem->Load(\"libAOD\");" << endl;
1136 out << " gSystem->Load(\"libANALYSIS\");" << endl;
1137 out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1138 out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1140 TIter next(fPackages);
1143 Bool_t hasSTEERBase = kFALSE;
1144 Bool_t hasESD = kFALSE;
1145 Bool_t hasAOD = kFALSE;
1146 Bool_t hasANALYSIS = kFALSE;
1147 Bool_t hasANALYSISalice = kFALSE;
1148 Bool_t hasCORRFW = kFALSE;
1149 while ((obj=next())) {
1150 pkgname = obj->GetName();
1151 if (pkgname == "STEERBase" ||
1152 pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
1153 if (pkgname == "ESD" ||
1154 pkgname == "ESD.par") hasESD = kTRUE;
1155 if (pkgname == "AOD" ||
1156 pkgname == "AOD.par") hasAOD = kTRUE;
1157 if (pkgname == "ANALYSIS" ||
1158 pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
1159 if (pkgname == "ANALYSISalice" ||
1160 pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
1161 if (pkgname == "CORRFW" ||
1162 pkgname == "CORRFW.par") hasCORRFW = kTRUE;
1164 if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
1165 else out << " if (!SetupPar(\"STEERBase\")) return;" << endl;
1166 if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
1167 else out << " if (!SetupPar(\"ESD\")) return;" << endl;
1168 if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
1169 else out << " if (!SetupPar(\"AOD\")) return;" << endl;
1170 if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
1171 else out << " if (!SetupPar(\"ANALYSIS\")) return;" << endl;
1172 if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
1173 else out << " if (!SetupPar(\"ANALYSISalice\")) return;" << endl;
1174 if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
1175 else out << " if (!SetupPar(\"CORRFW\")) return;" << endl << endl;
1176 out << "// Compile other par packages" << endl;
1178 while ((obj=next())) {
1179 pkgname = obj->GetName();
1180 if (pkgname == "STEERBase" ||
1181 pkgname == "STEERBase.par" ||
1183 pkgname == "ESD.par" ||
1185 pkgname == "AOD.par" ||
1186 pkgname == "ANALYSIS" ||
1187 pkgname == "ANALYSIS.par" ||
1188 pkgname == "ANALYSISalice" ||
1189 pkgname == "ANALYSISalice.par" ||
1190 pkgname == "CORRFW" ||
1191 pkgname == "CORRFW.par") continue;
1192 out << " if (!SetupPar(\"" << obj->GetName() << "\")) return;" << endl;
1195 out << "// include path" << endl;
1196 if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
1197 out << " gSystem->AddIncludePath(\"-I$ALICE_ROOT/include\");" << endl << endl;
1198 if (fAdditionalLibs.Length()) {
1199 out << "// Add aditional AliRoot libraries" << endl;
1200 TObjArray *list = fAdditionalLibs.Tokenize(" ");
1203 while((str=(TObjString*)next())) {
1204 if (str->GetString().Contains(".so"))
1205 out << " gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
1207 if (list) delete list;
1210 out << "// analysis source to be compiled at runtime (if any)" << endl;
1211 if (fAnalysisSource.Length()) {
1212 TObjArray *list = fAnalysisSource.Tokenize(" ");
1215 while((str=(TObjString*)next())) {
1216 out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
1218 if (list) delete list;
1221 out << "// connect to AliEn and make the chain" << endl;
1222 out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
1223 if (IsUsingTags()) {
1224 out << " TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
1226 out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
1228 out << "// read the analysis manager from file" << endl;
1229 out << " TFile *file = TFile::Open(\"analysis.root\");" << endl;
1230 out << " if (!file) return;" << endl;
1231 out << " TIter nextkey(file->GetListOfKeys());" << endl;
1232 out << " AliAnalysisManager *mgr = 0;" << endl;
1233 out << " TKey *key;" << endl;
1234 out << " while ((key=(TKey*)nextkey())) {" << endl;
1235 out << " if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
1236 out << " mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
1237 out << " };" << endl;
1238 out << " if (!mgr) {" << endl;
1239 out << " ::Error(\"" << func.Data() << "\", \"No analysis manager found in file analysis.root\");" << endl;
1240 out << " return;" << endl;
1241 out << " }" << endl << endl;
1242 out << " mgr->PrintStatus();" << endl;
1243 out << " mgr->StartAnalysis(\"localfile\", chain);" << endl;
1244 out << " timer.Stop();" << endl;
1245 out << " timer.Print();" << endl;
1246 out << "}" << endl << endl;
1247 if (IsUsingTags()) {
1248 out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
1250 out << "// Create a chain using tags from the xml file." << endl;
1251 out << " TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
1252 out << " if (!coll) {" << endl;
1253 out << " ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1254 out << " return NULL;" << endl;
1255 out << " }" << endl;
1256 out << " TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
1257 out << " AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
1258 out << " tagAna->ChainGridTags(tagResult);" << endl << endl;
1259 out << " AliRunTagCuts *runCuts = new AliRunTagCuts();" << endl;
1260 out << " AliLHCTagCuts *lhcCuts = new AliLHCTagCuts();" << endl;
1261 out << " AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
1262 out << " AliEventTagCuts *evCuts = new AliEventTagCuts();" << endl;
1263 out << " // Check if the cuts configuration file was provided" << endl;
1264 out << " if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
1265 out << " gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
1266 out << " ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1267 out << " }" << endl;
1268 if (fFriendChainName=="") {
1269 out << " TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
1271 out << " TString tmpColl=\"tmpCollection.xml\";" << endl;
1272 out << " tagAna->CreateXMLCollection(tmpColl.Data(),runCuts, lhcCuts, detCuts, evCuts);" << endl;
1273 out << " TChain *chain = CreateChain(tmpColl.Data(),type);" << endl;
1275 out << " if (!chain || !chain->GetNtrees()) return NULL;" << endl;
1276 out << " chain->ls();" << endl;
1277 out << " return chain;" << endl;
1278 out << "}" << endl << endl;
1279 if (gSystem->AccessPathName("ConfigureCuts.C")) {
1280 TString msg = "\n##### You may want to provide a macro ConfigureCuts.C with a method:\n";
1281 msg += " void ConfigureCuts(AliRunTagCuts *runCuts,\n";
1282 msg += " AliLHCTagCuts *lhcCuts,\n";
1283 msg += " AliDetectorTagCuts *detCuts,\n";
1284 msg += " AliEventTagCuts *evCuts)";
1285 Info("WriteAnalysisMacro", msg.Data());
1288 if (!IsUsingTags() || fFriendChainName!="") {
1289 out <<"//________________________________________________________________________________" << endl;
1290 out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
1292 out << "// Create a chain using url's from xml file" << endl;
1293 out << " TString treename = type;" << endl;
1294 out << " treename.ToLower();" << endl;
1295 out << " treename += \"Tree\";" << endl;
1296 out << " printf(\"***************************************\\n\");" << endl;
1297 out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
1298 out << " printf(\"***************************************\\n\");" << endl;
1299 out << " TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
1300 out << " if (!coll) {" << endl;
1301 out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
1302 out << " return NULL;" << endl;
1303 out << " }" << endl;
1304 out << " TChain *chain = new TChain(treename);" << endl;
1305 if(fFriendChainName!="") {
1306 out << " TChain *chainFriend = new TChain(treename);" << endl;
1308 out << " coll->Reset();" << endl;
1309 out << " while (coll->Next()) {" << endl;
1310 out << " chain->Add(coll->GetTURL(\"\"));" << endl;
1311 if(fFriendChainName!="") {
1312 out << " TString fileFriend=coll->GetTURL(\"\");" << endl;
1313 out << " fileFriend.ReplaceAll(\"AliAOD.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1314 out << " fileFriend.ReplaceAll(\"AliAODs.root\",\""<<fFriendChainName.Data()<<"\");" << endl;
1315 out << " chainFriend->Add(fileFriend.Data());" << endl;
1317 out << " }" << endl;
1318 out << " if (!chain->GetNtrees()) {" << endl;
1319 out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
1320 out << " return NULL;" << endl;
1321 out << " }" << endl;
1322 if(fFriendChainName!="") {
1323 out << " chain->AddFriend(chainFriend);" << endl;
1325 out << " return chain;" << endl;
1326 out << "}" << endl << endl;
1329 out <<"//________________________________________________________________________________" << endl;
1330 out << "Bool_t SetupPar(const char *package) {" << endl;
1331 out << "// Compile the package and set it up." << endl;
1332 out << " TString pkgdir = package;" << endl;
1333 out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
1334 out << " gSystem->Exec(Form(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
1335 out << " TString cdir = gSystem->WorkingDirectory();" << endl;
1336 out << " gSystem->ChangeDirectory(pkgdir);" << endl;
1337 out << " // Check for BUILD.sh and execute" << endl;
1338 out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
1339 out << " printf(\"*******************************\\n\");" << endl;
1340 out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
1341 out << " printf(\"*******************************\\n\");" << endl;
1342 out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
1343 out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
1344 out << " gSystem->ChangeDirectory(cdir);" << endl;
1345 out << " return kFALSE;" << endl;
1346 out << " }" << endl;
1347 out << " } else {" << endl;
1348 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
1349 out << " gSystem->ChangeDirectory(cdir);" << endl;
1350 out << " return kFALSE;" << endl;
1351 out << " }" << endl;
1352 out << " // Check for SETUP.C and execute" << endl;
1353 out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
1354 out << " printf(\"*******************************\\n\");" << endl;
1355 out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
1356 out << " printf(\"*******************************\\n\");" << endl;
1357 out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
1358 out << " } else {" << endl;
1359 out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
1360 out << " gSystem->ChangeDirectory(cdir);" << endl;
1361 out << " return kFALSE;" << endl;
1362 out << " }" << endl;
1363 out << " // Restore original workdir" << endl;
1364 out << " gSystem->ChangeDirectory(cdir);" << endl;
1365 out << " return kTRUE;" << endl;
1368 Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
1370 Bool_t copy = kTRUE;
1371 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1374 TString workdir = gGrid->GetHomeDirectory();
1375 workdir += fGridWorkingDir;
1376 if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
1377 if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
1378 if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
1379 Info("WriteAnalysisMacro", "\n##### Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
1380 TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
1382 Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
1383 TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
1387 //______________________________________________________________________________
1388 void AliAnalysisAlien::WriteExecutable()
1390 // Generate the alien executable script.
1391 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1393 out.open(fExecutable.Data(), ios::out);
1395 Error("CreateJDL", "Bad file name for executable: %s", fExecutable.Data());
1398 out << "#!/bin/bash" << endl;
1399 out << "export GCLIENT_SERVER_LIST=\"pcapiserv04.cern.ch:10000|pcapiserv05.cern.ch:10000|pcapiserv06.cern.ch:10000|pcapiserv07.cern.ch:10000\"" << endl;
1400 out << "echo \"=========================================\"" << endl;
1401 out << "echo \"############## PATH : ##############\"" << endl;
1402 out << "echo $PATH" << endl;
1403 out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
1404 out << "echo $LD_LIBRARY_PATH" << endl;
1405 out << "echo \"############## ROOTSYS : ##############\"" << endl;
1406 out << "echo $ROOTSYS" << endl;
1407 out << "echo \"############## which root : ##############\"" << endl;
1408 out << "which root" << endl;
1409 out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
1410 out << "echo $ALICE_ROOT" << endl;
1411 out << "echo \"############## which aliroot : ##############\"" << endl;
1412 out << "which aliroot" << endl;
1413 out << "echo \"=========================================\"" << endl << endl;
1414 // if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
1415 out << "root -b -q ";
1416 out << fAnalysisMacro.Data() << endl << endl;
1417 out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
1419 Bool_t copy = kTRUE;
1420 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1423 TString workdir = gGrid->GetHomeDirectory();
1424 workdir += fGridWorkingDir;
1425 TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
1426 if (FileExists(executable)) gGrid->Rm(executable);
1427 Info("CreateJDL", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
1428 TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
1432 //______________________________________________________________________________
1433 void AliAnalysisAlien::WriteValidationScript()
1435 // Generate the alien validation script.
1436 // Generate the validation script
1439 Error("WriteValidationScript", "Alien connection required");
1442 TString out_stream = "";
1443 if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
1444 if (!TestBit(AliAnalysisGrid::kSubmit)) {
1446 out.open("validate.sh", ios::out);
1447 out << "#!/bin/bash" << endl;
1448 out << "##################################################" << endl;
1449 out << "validateout=`dirname $0`" << endl;
1450 out << "validatetime=`date`" << endl;
1451 out << "validated=\"0\";" << endl;
1452 out << "error=0" << endl;
1453 out << "if [ -z $validateout ]" << endl;
1454 out << "then" << endl;
1455 out << " validateout=\".\"" << endl;
1456 out << "fi" << endl << endl;
1457 out << "cd $validateout;" << endl;
1458 out << "validateworkdir=`pwd`;" << endl << endl;
1459 out << "echo \"*******************************************************\"" << out_stream << endl;
1460 out << "echo \"* Automatically generated validation script *\"" << out_stream << endl;
1462 out << "echo \"* Time: $validatetime \"" << out_stream << endl;
1463 out << "echo \"* Dir: $validateout\"" << out_stream << endl;
1464 out << "echo \"* Workdir: $validateworkdir\"" << out_stream << endl;
1465 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1466 out << "ls -la ./" << out_stream << endl;
1467 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl << endl;
1468 out << "##################################################" << endl;
1471 out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
1472 out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
1473 out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
1476 out << "if [ ! -f stderr ] ; then" << endl;
1477 out << " error=1" << endl;
1478 out << " echo \"* ########## Job not validated - no stderr ###\" " << out_stream << endl;
1479 out << " echo \"Error = $error\" " << out_stream << endl;
1480 out << "fi" << endl;
1482 out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
1483 out << " error=1" << endl;
1484 out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << out_stream << endl;
1485 out << " echo \"$parArch\" " << out_stream << endl;
1486 out << " echo \"Error = $error\" " << out_stream << endl;
1487 out << "fi" << endl;
1489 out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
1490 out << " error=1" << endl;
1491 out << " echo \"* ########## Job not validated - Segment. violation ###\" " << out_stream << endl;
1492 out << " echo \"$segViol\" " << out_stream << endl;
1493 out << " echo \"Error = $error\" " << out_stream << endl;
1494 out << "fi" << endl;
1496 out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
1497 out << " error=1" << endl;
1498 out << " echo \"* ########## Job not validated - Segment. fault ###\" " << out_stream << endl;
1499 out << " echo \"$segFault\" " << out_stream << endl;
1500 out << " echo \"Error = $error\" " << out_stream << endl;
1501 out << "fi" << endl;
1503 // Part dedicated to the specific analyses running into the train
1505 TObjArray *arr = fOutputFiles.Tokenize(" ");
1507 TString output_file;
1508 while ((os=(TObjString*)next1())) {
1509 output_file = os->GetString();
1510 Int_t index = output_file.Index("@");
1511 if (index > 0) output_file.Remove(index);
1512 out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
1513 out << " error=1" << endl;
1514 out << " echo \"Output file(s) not found. Job FAILED !\"" << out_stream << endl;
1515 out << " echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
1516 out << "fi" << endl;
1519 out << "if [ $error = 0 ] ; then" << endl;
1520 out << " echo \"* ---------------- Job Validated ------------------*\"" << out_stream << endl;
1521 out << "fi" << endl;
1523 out << "echo \"* ----------------------------------------------------*\"" << out_stream << endl;
1524 out << "echo \"*******************************************************\"" << out_stream << endl;
1525 out << "cd -" << endl;
1526 out << "exit $error" << endl;
1528 Bool_t copy = kTRUE;
1529 if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1532 TString workdir = gGrid->GetHomeDirectory();
1533 workdir += fGridWorkingDir;
1534 Info("CreateJDL", "\n##### Copying validation script <validate.sh> to your AliEn working space");
1535 if (FileExists("validate.sh")) gGrid->Rm("validate.sh");
1536 TFile::Cp("file:validate.sh", Form("alien://%s/validate.sh", workdir.Data()));