New plugin supporting GRID mode for the analysis framework (M.Gheata)
authoragheata <agheata@f7af4fe6-9843-0410-8265-dc069ae4e863>
Fri, 24 Oct 2008 10:53:11 +0000 (10:53 +0000)
committeragheata <agheata@f7af4fe6-9843-0410-8265-dc069ae4e863>
Fri, 24 Oct 2008 10:53:11 +0000 (10:53 +0000)
Details at: http://aliceinfo.cern.ch/Offline/Activities/Analysis/AnalysisFramework/AlienPlugin.html

ANALYSIS/ANALYSISLinkDef.h
ANALYSIS/ANALYSISaliceLinkDef.h
ANALYSIS/AliAnalysisAlien.cxx [new file with mode: 0644]
ANALYSIS/AliAnalysisAlien.h [new file with mode: 0644]
ANALYSIS/AliAnalysisGrid.cxx [new file with mode: 0644]
ANALYSIS/AliAnalysisGrid.h [new file with mode: 0644]
ANALYSIS/AliAnalysisManager.cxx
ANALYSIS/AliAnalysisManager.h
ANALYSIS/libANALYSIS.pkg
ANALYSIS/libANALYSISalice.pkg

index c162c64..0253451 100644 (file)
@@ -12,5 +12,6 @@
 #pragma link C++ class  AliAnalysisSelector+;
 #pragma link C++ class  AliAnalysisFilter+;
 #pragma link C++ class  AliAnalysisCuts+;
+#pragma link C++ class  AliAnalysisGrid+;
 
 #endif
index 38a8413..ce258d6 100644 (file)
@@ -23,6 +23,7 @@
 #ifdef WITHXML
 #pragma link C++ class AliTagAnalysis+;
 #pragma link C++ class AliXMLCollection+;
+#pragma link C++ class AliAnalysisAlien+;
 #endif
 
 #endif
diff --git a/ANALYSIS/AliAnalysisAlien.cxx b/ANALYSIS/AliAnalysisAlien.cxx
new file mode 100644 (file)
index 0000000..b8a2eb1
--- /dev/null
@@ -0,0 +1,1233 @@
+/**************************************************************************
+ * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
+ *                                                                        *
+ * Author: The ALICE Off-line Project.                                    *
+ * Contributors are mentioned in the code where appropriate.              *
+ *                                                                        *
+ * Permission to use, copy, modify and distribute this software and its   *
+ * documentation strictly for non-commercial purposes is hereby granted   *
+ * without fee, provided that the above copyright notice appears in all   *
+ * copies and that both the copyright notice and this permission notice   *
+ * appear in the supporting documentation. The authors make no claims     *
+ * about the suitability of this software for any purpose. It is          *
+ * provided "as is" without express or implied warranty.                  *
+ **************************************************************************/
+
+// Author: Mihaela Gheata, 01/09/2008
+
+//==============================================================================
+//   AliAnalysisAlien - AliEn utility class. Provides interface for creating
+// a personalized JDL, finding and creating a dataset.
+//==============================================================================
+
+#include "Riostream.h"
+#include "TROOT.h"
+#include "TSystem.h"
+#include "TFile.h"
+#include "TObjString.h"
+#include "TObjArray.h"
+#include "TGrid.h"
+#include "TGridResult.h"
+#include "TGridCollection.h"
+#include "TGridJDL.h"
+#include "TFileMerger.h"
+#include "AliAnalysisManager.h"
+#include "AliAnalysisAlien.h"
+
+ClassImp(AliAnalysisAlien)
+
+//______________________________________________________________________________
+AliAnalysisAlien::AliAnalysisAlien()
+                 :AliAnalysisGrid(),
+                  fGridJDL(NULL),
+                  fPrice(0),
+                  fTTL(0),
+                  fSplitMaxInputFileNumber(0),
+                  fMaxInitFailed(0),
+                  fMasterResubmitThreshold(0),
+                  fRunNumbers(),
+                  fExecutable(),
+                  fArguments(),
+                  fAnalysisMacro(),
+                  fAnalysisSource(),
+                  fAdditionalLibs(),
+                  fSplitMode(),
+                  fAPIVersion(),
+                  fROOTVersion(),
+                  fAliROOTVersion(),
+                  fUser(),
+                  fGridWorkingDir(),
+                  fGridDataDir(),
+                  fDataPattern(),
+                  fGridOutputDir(),
+                  fOutputArchive(),
+                  fOutputFiles(),
+                  fInputFormat(),
+                  fJDLName(),
+                  fInputFiles(0)
+{
+// Dummy ctor.
+   SetDefaults();
+}
+
+//______________________________________________________________________________
+AliAnalysisAlien::AliAnalysisAlien(const char *name)
+                 :AliAnalysisGrid(name),
+                  fGridJDL(NULL),
+                  fPrice(0),
+                  fTTL(0),
+                  fSplitMaxInputFileNumber(0),
+                  fMaxInitFailed(0),
+                  fMasterResubmitThreshold(0),
+                  fRunNumbers(),
+                  fExecutable(),
+                  fArguments(),
+                  fAnalysisMacro(),
+                  fAnalysisSource(),
+                  fAdditionalLibs(),
+                  fSplitMode(),
+                  fAPIVersion(),
+                  fROOTVersion(),
+                  fAliROOTVersion(),
+                  fUser(),
+                  fGridWorkingDir(),
+                  fGridDataDir(),
+                  fDataPattern(),
+                  fGridOutputDir(),
+                  fOutputArchive(),
+                  fOutputFiles(),
+                  fInputFormat(),
+                  fJDLName(),
+                  fInputFiles(0)
+{
+// Default ctor.
+   SetDefaults();
+}
+
+//______________________________________________________________________________
+AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
+                 :AliAnalysisGrid(other),
+                  fGridJDL(NULL),
+                  fPrice(other.fPrice),
+                  fTTL(other.fTTL),
+                  fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
+                  fMaxInitFailed(other.fMaxInitFailed),
+                  fMasterResubmitThreshold(other.fMasterResubmitThreshold),
+                  fRunNumbers(other.fRunNumbers),
+                  fExecutable(other.fExecutable),
+                  fArguments(other.fArguments),
+                  fAnalysisMacro(other.fAnalysisMacro),
+                  fAnalysisSource(other.fAnalysisSource),
+                  fAdditionalLibs(other.fAdditionalLibs),
+                  fSplitMode(other.fSplitMode),
+                  fAPIVersion(other.fAPIVersion),
+                  fROOTVersion(other.fROOTVersion),
+                  fAliROOTVersion(other.fAliROOTVersion),
+                  fUser(other.fUser),
+                  fGridWorkingDir(other.fGridWorkingDir),
+                  fGridDataDir(other.fGridDataDir),
+                  fDataPattern(other.fDataPattern),
+                  fGridOutputDir(other.fGridOutputDir),
+                  fOutputArchive(other.fOutputArchive),
+                  fOutputFiles(other.fOutputFiles),
+                  fInputFormat(other.fInputFormat),
+                  fJDLName(other.fJDLName),
+                  fInputFiles(0)
+{
+// Copy ctor.
+   fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
+   if (other.fInputFiles) {
+      fInputFiles = new TObjArray();
+      TIter next(other.fInputFiles);
+      TObject *obj;
+      while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
+      fInputFiles->SetOwner();
+   }   
+}
+
+//______________________________________________________________________________
+AliAnalysisAlien::~AliAnalysisAlien()
+{
+// Destructor.
+   if (fGridJDL) delete fGridJDL;
+   if (fInputFiles) delete fInputFiles;
+}   
+
+//______________________________________________________________________________
+AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
+{
+// Assignment.
+   if (this != &other) {
+      AliAnalysisGrid::operator=(other);
+      fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
+      fPrice                   = other.fPrice;
+      fTTL                     = other.fTTL;
+      fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
+      fMaxInitFailed           = other.fMaxInitFailed;
+      fMasterResubmitThreshold = other.fMasterResubmitThreshold;
+      fRunNumbers              = other.fRunNumbers;
+      fExecutable              = other.fExecutable;
+      fArguments               = other.fArguments;
+      fAnalysisMacro           = other.fAnalysisMacro;
+      fAnalysisSource          = other.fAnalysisSource;
+      fAdditionalLibs          = other.fAdditionalLibs;
+      fSplitMode               = other.fSplitMode;
+      fAPIVersion              = other.fAPIVersion;
+      fROOTVersion             = other.fROOTVersion;
+      fAliROOTVersion          = other.fAliROOTVersion;
+      fUser                    = other.fUser;
+      fGridWorkingDir          = other.fGridWorkingDir;
+      fGridDataDir             = other.fGridDataDir;
+      fDataPattern             = other.fDataPattern;
+      fGridOutputDir           = other.fGridOutputDir;
+      fOutputArchive           = other.fOutputArchive;
+      fOutputFiles             = other.fOutputFiles;
+      fInputFormat             = other.fInputFormat;
+      fJDLName                 = other.fJDLName;
+      if (other.fInputFiles) {
+         fInputFiles = new TObjArray();
+         TIter next(other.fInputFiles);
+         TObject *obj;
+         while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
+         fInputFiles->SetOwner();
+      }   
+   }
+   return *this;
+}
+
+//______________________________________________________________________________
+void AliAnalysisAlien::AddRunNumber(Int_t run)
+{
+// Add a run number to the list of runs to be processed.
+   if (fRunNumbers.Length()) fRunNumbers += " ";
+   fRunNumbers += Form("%d", run);
+}   
+
+//______________________________________________________________________________
+void AliAnalysisAlien::AddDataFile(const char *lfn)
+{
+// Adds a data file to the input to be analysed. The file should be a valid LFN
+// or point to an existing file in the alien workdir.
+   if (!fInputFiles) fInputFiles = new TObjArray();
+   fInputFiles->Add(new TObjString(lfn));
+}
+   
+//______________________________________________________________________________
+Bool_t AliAnalysisAlien::Connect()
+{
+// Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
+   if (gGrid && gGrid->IsConnected()) return kTRUE;
+   if (!gSystem->Getenv("alien_API_USER")) {
+      Error("Connect", "Make sure you:\n 1. Have called: alien-token-init <username> today\n 2. Have sourced /tmp/gclient_env_%s",
+            gSystem->Getenv("UID"));
+      return kFALSE;
+   }         
+   if (!gGrid) {
+      Info("Connect", "Trying to connect to AliEn ...");
+      TGrid::Connect("alien://");
+   }
+   if (!gGrid || !gGrid->IsConnected()) {
+      Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
+      return kFALSE;
+   }  
+   fUser = gGrid->GetUser();
+   Info("Connect", "\n#####   Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
+   return kTRUE;
+}
+
+//______________________________________________________________________________
+void AliAnalysisAlien::CdWork()
+{
+// Check validity of alien workspace. Create directory if possible.
+   if (!Connect()) {
+      Error("CdWork", "Alien connection required");
+      return;
+   } 
+   TString homedir = gGrid->GetHomeDirectory();
+   TString workdir = homedir + fGridWorkingDir;
+   if (!gGrid->Cd(workdir)) {
+      gGrid->Cd(homedir);
+      if (gGrid->Mkdir(workdir)) {
+         gGrid->Cd(fGridWorkingDir);
+         Info("CreateJDL", "\n#####   Created alien working directory %s", fGridWorkingDir.Data());
+      } else {
+         Warning("CreateJDL", "Working directory %s cannot be created.\n Using %s instead.",
+                 workdir.Data(), homedir.Data());
+         fGridWorkingDir = "";
+      }          
+   }      
+}
+
+//______________________________________________________________________________
+Bool_t AliAnalysisAlien::CheckInputData()
+{
+// Check validity of input data. If necessary, create xml files.
+   if (!fInputFiles && !fRunNumbers.Length()) {
+       Error("CheckInputData", "You have to specify either a set of run numbers or some existing grid files. Use AddRunNumber()/AddDataFile().");
+      return kFALSE;
+   }
+   // Process declared files
+   Bool_t is_collection = kFALSE;
+   Bool_t is_xml = kFALSE;
+   Bool_t use_tags = kFALSE;
+   Bool_t checked = kFALSE;
+   CdWork();
+   TString file;
+   TString workdir = gGrid->GetHomeDirectory();
+   workdir += fGridWorkingDir;
+   if (fInputFiles) {
+      TObjString *objstr;
+      TIter next(fInputFiles);
+      while ((objstr=(TObjString*)next())) {
+         file = workdir;
+         file += "/";
+         file += objstr->GetString();
+         // Store full lfn path
+         if (FileExists(file)) objstr->SetString(file);
+         else {
+            file = objstr->GetName();
+            if (!FileExists(objstr->GetName())) {
+               Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
+                     objstr->GetName(), workdir.Data());
+               return kFALSE;
+            }         
+         }
+         Bool_t iscoll, isxml, usetags;
+         CheckDataType(file, iscoll, isxml, usetags);
+         if (!checked) {
+            checked = kTRUE;
+            is_collection = iscoll;
+            is_xml = isxml;
+            use_tags = usetags;
+            TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
+         } else {
+            if ((iscoll != is_collection) || (isxml != is_xml) || (usetags != use_tags)) {
+               Error("CheckInputData", "Some conflict was found in the types of inputs");
+               return kFALSE;
+            } 
+         }
+      }
+   }
+   // Process requested run numbers
+   if (!fRunNumbers.Length()) return kTRUE;
+   // Check validity of alien data directory
+   if (!fGridDataDir.Length()) {
+      Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
+      return kFALSE;
+   }
+   if (!gGrid->Cd(fGridDataDir)) {
+      Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
+      return kFALSE;
+   }
+   if (is_collection) {
+      Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
+      return kFALSE;   
+   }
+   
+   if (checked && !is_xml) {
+      Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
+      return kFALSE;   
+   }
+   // Check validity of run number(s)
+   TObjArray *arr;
+   TObjString *os;
+   TString path;
+   if (!checked) {
+      checked = kTRUE;
+      use_tags = fDataPattern.Contains("tag");
+      TObject::SetBit(AliAnalysisGrid::kUseTags, use_tags);
+   }   
+   if (use_tags != fDataPattern.Contains("tag")) {
+      Error("CheckInputData", "Cannot mix input files using/not using tags");
+      return kFALSE;
+   }
+   if (fRunNumbers.Length()) {
+      arr = fRunNumbers.Tokenize(" ");
+      TIter next(arr);
+      while ((os=(TObjString*)next())) {
+         path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
+         if (!gGrid->Cd(path)) {
+            Error("CheckInputData", "Run number %s not found in path: %s", os->GetString().Data(), path.Data());
+            return kFALSE;
+         }
+         path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
+         TString msg = "\n#####   file: ";
+         msg += path;
+         msg += " type: xml_collection;";
+         if (use_tags) msg += " using_tags: Yes";
+         else          msg += " using_tags: No";
+         Info("CheckDataType", msg.Data());
+         AddDataFile(path);
+      }
+      delete arr;   
+   }
+   return kTRUE;      
+}   
+
+//______________________________________________________________________________
+Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
+{
+// Create dataset for the grid data directory + run number.
+   if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
+   if (!Connect()) {
+      Error("CreateDataset", "Cannot create dataset with no grid connection");
+      return kFALSE;
+   }   
+
+   // Cd workspace
+   CdWork();
+   TString workdir = gGrid->GetHomeDirectory();
+   workdir += fGridWorkingDir;
+
+   // Compose the 'find' command arguments
+   TString command;
+   TString options = "-x collection ";
+   if (TestBit(AliAnalysisGrid::kTest)) options += "-l 10 ";
+   TString conditions = "";
+   
+   TString file;
+   TString path;
+   if (!fRunNumbers.Length()) return kTRUE;   
+   // Several runs
+   TObjArray *arr = fRunNumbers.Tokenize(" ");
+   TObjString *os;
+   TIter next(arr);
+   while ((os=(TObjString*)next())) {
+      path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
+      file = Form("%s.xml", os->GetString().Data());
+      if (FileExists(file)) {
+         Info("CreateDataset", "\n#####   Removing previous dataset %s", file.Data());
+         gGrid->Rm(file); 
+      }
+      command = "find ";
+      command += options;
+      command += path;
+      command += pattern;
+      conditions = Form(" > %s", file.Data());
+      command += conditions;
+      TGridResult *res = gGrid->Command(command);
+      if (res) delete res;
+      if (!FileExists(file)) {
+         Error("CreateDataset", "Command %s did NOT succeed", command.Data());
+         delete arr;
+         return kFALSE;
+      }
+      if (TestBit(AliAnalysisGrid::kTest)) break;
+   }   
+   delete arr;
+   // Copy the file back to client if local testing is requested.
+   if (TestBit(AliAnalysisGrid::kTest)) {
+      Info("CreateDataset", "\n#####   Copying dataset <%s> to <wn.xml> in your current directory...", file.Data());
+      TFile::Cp(Form("alien://%s/%s", workdir.Data(), file.Data()), "file:wn.xml",file.Data());
+   }   
+   return kTRUE;
+}
+
+//______________________________________________________________________________
+Bool_t AliAnalysisAlien::CreateJDL()
+{
+// Generate a JDL file according to current settings. The name of the file is 
+// specified by fJDLName.
+   Bool_t error = kFALSE;
+   TObjArray *arr = 0;
+   Bool_t copy = kTRUE;
+   if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
+   Bool_t generate = kTRUE;
+   if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
+   if (!Connect()) {
+      Error("CreateJDL", "Alien connection required");
+      return kFALSE;
+   }   
+   // Check validity of alien workspace
+   CdWork();
+   TString workdir = gGrid->GetHomeDirectory();
+   workdir += fGridWorkingDir;
+   if (generate) {
+      TObjString *os;
+      if (!fInputFiles) {
+         Error("CreateJDL()", "Define some input files for your analysis.");
+         error = kTRUE;
+      }
+      // Compose list of input files   
+      // Check if output files were defined
+      if (!fOutputFiles.Length()) {
+         Error("CreateJDL", "You must define at least one output file");
+         error = kTRUE;
+      }   
+      // Check if an output directory was defined and valid
+      if (!fGridOutputDir.Length()) {
+         Error("CreateJDL", "You must define AliEn output directory");
+         error = kTRUE;
+      } else {
+         if (!gGrid->Cd(fGridOutputDir)) {
+            if (gGrid->Mkdir(fGridOutputDir)) {
+               Info("CreateJDL", "\n#####   Created alien output directory %s", fGridOutputDir.Data());
+            } else {
+               Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
+               error = kTRUE;
+            }
+         }
+         gGrid->Cd(workdir);
+      }   
+      // Exit if any error up to now
+      if (error) return kFALSE;   
+      // Set JDL fields
+      fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
+      fGridJDL->SetExecutable(fExecutable);
+//      fGridJDL->SetTTL((UInt_t)fTTL);
+      fGridJDL->SetValue("TTL", Form("\"%d\"", fTTL));
+      if (fMaxInitFailed > 0) 
+         fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
+      if (fSplitMaxInputFileNumber > 0) 
+         fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
+      if (fSplitMode.Length()) 
+         fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
+//      fGridJDL->SetSplitMode(fSplitMode, (UInt_t)fSplitMaxInputFileNumber);
+      if (fAliROOTVersion.Length())  
+         fGridJDL->AddToPackages("AliRoot", fAliROOTVersion);
+      if (fROOTVersion.Length()) 
+         fGridJDL->AddToPackages("ROOT", fROOTVersion);
+      if (fAPIVersion.Length()) 
+         fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
+      fGridJDL->SetInputDataListFormat(fInputFormat);
+      fGridJDL->SetInputDataList("wn.xml");
+      if (fInputFiles) {
+         TIter next(fInputFiles);
+         while ((os=(TObjString*)next()))
+            fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetString().Data()));
+      }      
+      fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()));
+      fGridJDL->AddToInputSandbox(Form("LF:%s/analysis.root", workdir.Data()));
+      if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C"))
+         fGridJDL->AddToInputSandbox(Form("LF:%s/ConfigureCuts.C", workdir.Data()));
+      if (fAdditionalLibs.Length()) {
+         arr = fAdditionalLibs.Tokenize(" ");
+         TIter next(arr);
+         while ((os=(TObjString*)next())) {
+            if (os->GetString().Contains(".so")) continue;
+            fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
+         }   
+         delete arr;   
+      }
+      if (fOutputArchive.Length()) {
+         arr = fOutputArchive.Tokenize(" ");
+         TIter next(arr);
+         while ((os=(TObjString*)next()))
+            fGridJDL->AddToOutputArchive(os->GetString().Data());
+         delete arr;
+      }      
+      fGridJDL->SetOutputDirectory(Form("%s/%s/#alien_counter_03i#", workdir.Data(), fGridOutputDir.Data()));
+      arr = fOutputFiles.Tokenize(" ");
+      TIter next(arr);
+      while ((os=(TObjString*)next())) fGridJDL->AddToOutputSandbox(os->GetString());
+      delete arr;
+//      fGridJDL->SetPrice((UInt_t)fPrice);
+      fGridJDL->SetValue("Price", Form("\"%d\"", fPrice));
+      fGridJDL->SetValidationCommand(Form("%s/validate.sh", workdir.Data()));
+      if (fMasterResubmitThreshold) fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
+      // Generate the JDL as a string
+      TString sjdl = fGridJDL->Generate();
+      Int_t index;
+      index = sjdl.Index("Executable");
+      if (index >= 0) sjdl.Insert(index, "\n# This is the startup script\n");
+      index = sjdl.Index("Split ");
+      if (index >= 0) sjdl.Insert(index, "\n# We split per storage element\n");
+      index = sjdl.Index("SplitMaxInputFileNumber");
+      if (index >= 0) sjdl.Insert(index, "\n# We want each subjob to get maximum this number of input files\n");
+      index = sjdl.Index("InputDataCollection");
+      if (index >= 0) sjdl.Insert(index, "# Input xml collections\n");
+      index = sjdl.Index("InputFile");
+      if (index >= 0) sjdl.Insert(index, "\n# List of input files to be uploaded to wn's\n");
+      index = sjdl.Index("InputDataList ");
+      if (index >= 0) sjdl.Insert(index, "\n# Collection to be processed on wn\n");
+      index = sjdl.Index("InputDataListFormat");
+      if (index >= 0) sjdl.Insert(index, "\n# Format of input data\n");
+      index = sjdl.Index("Price");
+      if (index >= 0) sjdl.Insert(index, "\n# AliEn price for this job\n");
+      index = sjdl.Index("Requirements");
+      if (index >= 0) sjdl.Insert(index, "\n# Additional requirements for the computing element\n");
+      index = sjdl.Index("Packages");
+      if (index >= 0) sjdl.Insert(index, "\n# Packages to be used\n");
+      index = sjdl.Index("User");
+      if (index >= 0) sjdl.Insert(index, "\n# AliEn user\n");
+      index = sjdl.Index("TTL");
+      if (index >= 0) sjdl.Insert(index, "\n# Time to live for the job\n");
+      index = sjdl.Index("OutputFile");
+      if (index >= 0) sjdl.Insert(index, "\n# List of output files to be registered\n");
+      index = sjdl.Index("OutputDir");
+      if (index >= 0) sjdl.Insert(index, "\n# Output directory\n");
+      index = sjdl.Index("OutputArchive");
+      if (index >= 0) sjdl.Insert(index, "\n# Files to be archived\n");
+      index = sjdl.Index("MaxInitFailed");
+      if (index >= 0) sjdl.Insert(index, "\n# Maximum number of first failing jobs to abort the master job\n");
+      index = sjdl.Index("MasterResubmitThreshold");
+      if (index >= 0) sjdl.Insert(index, "\n# Resubmit failed jobs until DONE rate reaches this percentage\n");
+      sjdl.ReplaceAll("ValidationCommand", "Validationcommand");
+      index = sjdl.Index("Validationcommand");
+      if (index >= 0) sjdl.Insert(index, "\n# Validation script to be run for each subjob\n");
+      sjdl.ReplaceAll("\"LF:", "\n   \"LF:");
+      sjdl.ReplaceAll("(member", "\n   (member");
+      sjdl.ReplaceAll("\",\"VO_", "\",\n   \"VO_");
+      sjdl.ReplaceAll("{", "{\n   ");
+      sjdl.ReplaceAll("};", "\n};");
+      sjdl.ReplaceAll("{\n   \n", "{\n");
+      sjdl.ReplaceAll("\n\n", "\n");
+      sjdl.ReplaceAll("OutputDirectory", "OutputDir");
+      sjdl += "JDLVariables = \n{\n   \"Packages\",\n   \"OutputDir\"\n};\n";
+      sjdl.Prepend("JobTag = \"Automatically generated analysis JDL\";\n");
+      index = sjdl.Index("JDLVariables");
+      if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
+      // Write jdl to file
+      ofstream out;
+      out.open(fJDLName.Data(), ios::out);
+      if (out.bad()) {
+         Error("CreateJDL", "Bad file name: %s", fJDLName.Data());
+         return kFALSE;
+      }
+      out << sjdl << endl;
+   }
+   // Copy jdl to grid workspace   
+   if (!copy) {
+      Info("CreateJDL", "\n#####   You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
+   } else {
+      Info("CreateJDL", "\n#####   Copying JDL file <%s> to your AliEn working space", fJDLName.Data());
+      if (FileExists(fJDLName)) gGrid->Rm(fJDLName);
+      TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s/%s", workdir.Data(), fJDLName.Data()));
+      if (fAdditionalLibs.Length()) {
+         TObjArray *arr = fAdditionalLibs.Tokenize(" ");
+         TObjString *os;
+         TIter next(arr);
+         while ((os=(TObjString*)next())) {
+            Info("CreateJDL", "\n#####   Copying dependency: <%s> to your alien workspace", os->GetString().Data());
+            if (os->GetString().Contains(".so")) continue;
+            if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
+            TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
+         }   
+         delete arr;   
+      }
+   } 
+   return kTRUE;
+}
+
+//______________________________________________________________________________
+Bool_t AliAnalysisAlien::FileExists(const char *lfn) const
+{
+// Returns true if file exists.
+   if (!gGrid) {
+      Error("FileExists", "No connection to grid");
+      return kFALSE;
+   }
+   TGridResult *res = gGrid->Ls(lfn);
+   if (!res) return kFALSE;
+   TMap *map = dynamic_cast<TMap*>(res->At(0));
+   if (!map) {
+      delete res;
+      return kFALSE;
+   }   
+   TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
+   if (!objs || !objs->GetString().Length()) {
+      delete res;
+      return kFALSE;
+   }
+   delete res;   
+   return kTRUE;
+}
+
+//______________________________________________________________________________
+void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags)
+{
+// Check input data type.
+   is_collection = kFALSE;
+   is_xml = kFALSE;
+   use_tags = kFALSE;
+   if (!gGrid) {
+      Error("CheckDataType", "No connection to grid");
+      return;
+   }
+   is_collection = IsCollection(lfn);
+   TString msg = "\n#####   file: ";
+   msg += lfn;
+   if (is_collection) {
+      msg += " type: raw_collection;";
+   // special treatment for collections
+      is_xml = kFALSE;
+      // check for tag files in the collection
+      TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
+      if (!res) {
+         msg += " using_tags: No (unknown)";
+         Info("CheckDataType", msg.Data());
+         return;
+      }   
+      const char* typeStr = res->GetKey(0, "origLFN");
+      if (!typeStr || !strlen(typeStr)) {
+         msg += " using_tags: No (unknown)";
+         Info("CheckDataType", msg.Data());
+         return;
+      }   
+      TString file = typeStr;
+      use_tags = file.Contains(".tag");
+      if (use_tags) msg += " using_tags: Yes";
+      else          msg += " using_tags: No";
+      Info("CheckDataType", msg.Data());
+      return;
+   }
+   TString slfn(lfn);
+   slfn.ToLower();
+   is_xml = slfn.Contains(".xml");
+   if (is_xml) {
+   // Open xml collection and check if there are tag files inside
+      msg += " type: xml_collection;";
+      TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
+      if (!coll) {
+         msg += " using_tags: No (unknown)";
+         Info("CheckDataType", msg.Data());
+         return;
+      }   
+      TMap *map = coll->Next();
+      if (!map) {
+         msg += " using_tags: No (unknown)";
+         Info("CheckDataType", msg.Data());
+         return;
+      }   
+      map = (TMap*)map->GetValue("");
+      TString file;
+      if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
+      use_tags = file.Contains(".tag");
+      delete coll;
+      if (use_tags) msg += " using_tags: Yes";
+      else          msg += " using_tags: No";
+      Info("CheckDataType", msg.Data());
+      return;
+   }
+   use_tags = slfn.Contains(".tag");
+   if (slfn.Contains(".root")) msg += " type: root file;";
+   else                        msg += " type: unhnown file;";
+   if (use_tags) msg += " using_tags: Yes";
+   else          msg += " using_tags: No";
+   Info("CheckDataType", msg.Data());
+}
+
+//______________________________________________________________________________
+Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
+{
+// Returns true if file is a collection. Functionality duplicated from
+// TAlien::Type() because we don't want to directly depend on TAlien.
+   if (!gGrid) {
+      Error("IsCollection", "No connection to grid");
+      return kFALSE;
+   }
+   TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
+   if (!res) return kFALSE;
+   const char* typeStr = res->GetKey(0, "type");
+   if (!typeStr || !strlen(typeStr)) return kFALSE;
+   if (!strcmp(typeStr, "collection")) return kTRUE;
+   delete res;
+   return kFALSE;
+}   
+
+//______________________________________________________________________________
+void AliAnalysisAlien::SetDefaults()
+{
+// Set default values for everything. What cannot be filled will be left empty.
+   if (fGridJDL) delete fGridJDL;
+   fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
+   fPrice                      = 1;
+   fTTL                        = 30000;
+   fSplitMaxInputFileNumber    = 100;
+   fMaxInitFailed              = 0;
+   fMasterResubmitThreshold    = 0;
+   fRunNumbers                 = "";
+   fExecutable                 = "analysis.sh";
+   fArguments                  = "";
+   fAnalysisMacro              = "myAnalysis.C";
+   fAnalysisSource             = "";
+   fAdditionalLibs             = "";
+   fSplitMode                  = "se";
+   fAPIVersion                 = "";
+   fROOTVersion                = "";
+   fAliROOTVersion             = "";
+   fUser                       = "";  // Your alien user name
+   fGridWorkingDir             = "";
+   fGridDataDir                = "";  // Can be like: /alice/sim/PDC_08a/LHC08c9/
+   fDataPattern                = "*AliESDs.root";  // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
+   fGridOutputDir              = "output";
+   fOutputArchive              = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
+   fOutputFiles                = "";  // Like "AliAODs.root histos.root"
+   fInputFormat                = "xml-single";
+   fJDLName                    = "analysis.jdl";
+}   
+
+//______________________________________________________________________________
+Bool_t AliAnalysisAlien::MergeOutputs()
+{
+// Merge analysis outputs existing in the AliEn space.
+   if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
+   if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
+   if (!Connect()) {
+      Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
+      return kFALSE;
+   }   
+   // Get the output path
+   TString output = Form("/%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
+   if (!gGrid->Cd(output)) output = Form("/%s/%s", gGrid->GetHomeDirectory(), fGridOutputDir.Data());
+   if (!gGrid->Cd(output)) {
+      Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
+      return kFALSE;
+   }
+   if (!fOutputFiles.Length()) {
+      Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
+      return kFALSE;
+   }   
+   TObjArray *list = fOutputFiles.Tokenize(" ");
+   TIter next(list);
+   TObjString *str;
+   TString command;
+   TString output_file;
+   Bool_t merged = kTRUE;
+   while((str=(TObjString*)next())) {
+      output_file = str->GetString();
+      Int_t index = output_file.Index("@");
+      if (index > 0) output_file.Remove(index);
+      command = Form("find %s/ *%s", output.Data(), output_file.Data());
+      printf("command: %s\n", command.Data());
+      TGridResult *res = gGrid->Command(command);
+      if (!res) continue;
+      TFileMerger *fm = 0;
+      TIter nextmap(res);
+      TMap *map;
+      while ((map=(TMap*)nextmap())) {
+         TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
+         if (!objs || !objs->GetString().Length()) {
+            delete res;
+            continue;
+         }   
+         if (!fm) {
+            fm = new TFileMerger(kFALSE);
+            fm->SetFastMethod(kTRUE);
+            fm->OutputFile(output_file);
+         }
+         fm->AddFile(objs->GetString());   
+      }
+      if (!fm || !fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
+         Warning("MergeOutputs", "No <%s> files found.", output_file.Data());
+         merged = kFALSE;
+         delete res;
+         continue;
+      }
+      if (!fm->Merge()) {
+         Error("MergeOutputs", "Could not merge all <%s> files", output_file.Data());
+         merged = kFALSE;
+      } else {
+         Info("MergeOutputs", "\n#####   Merged %d output files <%s>", fm->GetMergeList()->GetSize(), output_file.Data());
+      }   
+      delete fm;
+      delete res;
+   } 
+   if (!merged) {
+      Error("MergeOutputs", "Terminate() will  NOT be executed");
+   }  
+   return merged;
+}   
+
+//______________________________________________________________________________
+void AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
+{
+// Start remote grid analysis.
+   
+   if (TestBit(AliAnalysisGrid::kOffline)) {
+      Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
+      \n                         there nor any job run. You can revise the JDL and analysis \
+      \n                         macro then run the same in \"submit\" mode.");
+   } else if (TestBit(AliAnalysisGrid::kTest)) {
+      Info("StartAnalysis","\n##### LOCAL MODE #####   Your analysis will be run locally on a subset of the requested \
+      \n                         dataset.");
+   } else if (TestBit(AliAnalysisGrid::kSubmit)) {
+      Info("StartAnalysis","\n##### SUBMIT MODE #####  Files required by your analysis are copied to your grid working \
+      \n                         space and job submitted.");
+   } else if (TestBit(AliAnalysisGrid::kMerge)) {
+      Info("StartAnalysis","\n##### MERGE MODE #####   The registered outputs of the analysis will be merged");
+      return;
+   } else {
+      Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");   
+   }   
+      
+   if (!Connect()) {
+      Error("StartAnalysis", "Cannot start grid analysis without grid connection");
+      return;
+   }   
+   if (!CheckInputData()) {
+      Error("StartAnalysis", "There was an error in preprocessing your requested input data");
+      return;
+   }   
+   CreateDataset(fDataPattern);
+   WriteAnalysisFile();   
+   WriteAnalysisMacro();
+   WriteExecutable();
+   WriteValidationScript();
+   if (!CreateJDL()) return;
+   if (TestBit(AliAnalysisGrid::kOffline)) return;
+   if (TestBit(AliAnalysisGrid::kTest)) {
+      // Locally testing the analysis
+      Info("StartAnalysis", "\n_______________________________________________________________________ \
+      \n   Running analysis script in a daughter shell as on a worker node \
+      \n_______________________________________________________________________");
+      TObjArray *list = fOutputFiles.Tokenize(" ");
+      TIter next(list);
+      TObjString *str;
+      TString output_file;
+      while((str=(TObjString*)next())) {
+         output_file = str->GetString();
+         Int_t index = output_file.Index("@");
+         if (index > 0) output_file.Remove(index);         
+         gSystem->Exec(Form("rm %s", output_file.Data()));
+      }
+      delete list;
+      gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
+      gSystem->Exec("bash validate.sh");
+//      gSystem->Exec("cat stdout");
+      return;
+   }
+   // Submit AliEn job
+   CdWork();
+   TGridResult *res = gGrid->Command(Form("submit %s", fJDLName.Data()));
+   TString jobID = "";
+   if (res) {
+      const char *cjobId = res->GetKey(0,"jobId");
+      if (!cjobId) {
+         Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
+         return;
+      } else {
+         Info("StartAnalysis", "\n_______________________________________________________________________ \
+         \n#####   Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
+      \n_______________________________________________________________________",
+                fJDLName.Data(), cjobId);
+         jobID = cjobId;      
+      }          
+      delete res;
+   }   
+   Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
+   \n You may exit at any time and terminate the job later using the option <terminate> \
+   \n ##################################################################################", jobID.Data());
+   gGrid->Shell();
+}
+
+//______________________________________________________________________________
+void AliAnalysisAlien::WriteAnalysisFile()
+{
+// Write current analysis manager into the file analysis.root
+   if (!TestBit(AliAnalysisGrid::kSubmit)) {  
+      AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
+      if (!mgr || !mgr->IsInitialized()) {
+         Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
+         return;
+      }
+      // Check analysis type
+      TObject *handler;
+      if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
+      handler = (TObject*)mgr->GetInputEventHandler();
+      if (handler) {
+         if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
+         if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
+      }
+      TDirectory *cdir = gDirectory;
+      TFile *file = TFile::Open("analysis.root", "RECREATE");
+      if (file) {
+         mgr->Write();
+         delete file;
+      }
+      if (cdir) cdir->cd();
+      Info("WriteAnalysisFile", "\n#####   Analysis manager: %s wrote to file <analysis.root>\n", mgr->GetName());
+   }   
+   Bool_t copy = kTRUE;
+   if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
+   if (copy) {
+      CdWork();
+      TString workdir = gGrid->GetHomeDirectory();
+      workdir += fGridWorkingDir;
+      Info("CreateJDL", "\n#####   Copying file <analysis.root> containing your initialized analysis manager to your alien workspace");
+      if (FileExists("analysis.root")) gGrid->Rm("analysis.root");
+      TFile::Cp("file:analysis.root", Form("alien://%s/analysis.root", workdir.Data()));
+   }   
+}
+
+//______________________________________________________________________________
+void AliAnalysisAlien::WriteAnalysisMacro()
+{
+// Write the analysis macro that will steer the analysis in grid mode.
+   if (!TestBit(AliAnalysisGrid::kSubmit)) {  
+      ofstream out;
+      out.open(fAnalysisMacro.Data(), ios::out);
+      if (!out.good()) {
+         Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
+         return;
+      }
+      TString func = fAnalysisMacro;
+      TString type = "ESD";
+      TString comment = "// Analysis using ";
+      if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
+      if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
+         type = "AOD";
+         comment += "AOD";
+      }   
+      if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
+      else comment += " data";
+      out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
+      func.ReplaceAll(".C", "");
+      out << "void " << func.Data() << "()" << endl; 
+      out << "{" << endl;
+      out << comment.Data() << endl;
+      out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
+      out << "// load base root libraries" << endl;
+      out << "   gSystem->Load(\"libTree\");" << endl;
+      out << "   gSystem->Load(\"libGeom\");" << endl;
+      out << "   gSystem->Load(\"libVMC\");" << endl;
+      out << "   gSystem->Load(\"libPhysics\");" << endl << endl;
+      out << "// load analysis framework libraries" << endl;
+      out << "   gSystem->Load(\"libSTEERBase\");" << endl;
+      out << "   gSystem->Load(\"libESD\");" << endl;
+      out << "   gSystem->Load(\"libAOD\");" << endl;
+      out << "   gSystem->Load(\"libANALYSIS\");" << endl;
+      out << "   gSystem->Load(\"libANALYSISalice\");" << endl << endl;
+      out << "// add aditional AliRoot libraries below" << endl;
+      if (fAdditionalLibs.Length()) {
+         TObjArray *list = fAdditionalLibs.Tokenize(" ");
+         TIter next(list);
+         TObjString *str;
+         while((str=(TObjString*)next())) {
+            if (str->GetString().Contains(".so"))
+               out << "   gSystem->Load(\"" << str->GetString().Data() << "\");" << endl;
+         }
+         if (list) delete list;
+      }
+      out << endl;
+      out << "// include path (remove if using par files)" << endl;
+      out << "   gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl << endl;
+      out << "// analysis source to be compiled at runtime (if any)" << endl;
+      if (fAnalysisSource.Length()) {
+         TObjArray *list = fAnalysisSource.Tokenize(" ");
+         TIter next(list);
+         TObjString *str;
+         while((str=(TObjString*)next())) {
+            out << "   gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
+         }   
+         if (list) delete list;
+      }
+      out << endl;
+      out << "// connect to AliEn and make the chain" << endl;
+      out << "   if (!TGrid::Connect(\"alien://\")) return;" << endl;
+      if (IsUsingTags()) {
+         out << "   TChain *chain = CreateChainFromTags(\"wn.xml\", anatype);" << endl << endl;
+      } else {
+         out << "   TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;      
+      }   
+      out << "// read the analysis manager from file" << endl;
+      out << "   TFile *file = TFile::Open(\"analysis.root\");" << endl;
+      out << "   if (!file) return;" << endl; 
+      out << "   TIter nextkey(file->GetListOfKeys());" << endl;
+      out << "   AliAnalysisManager *mgr = 0;" << endl;
+      out << "   TKey *key;" << endl;
+      out << "   while ((key=(TKey*)nextkey())) {" << endl;
+      out << "      if (!strcmp(key->GetClassName(), \"AliAnalysisManager\"))" << endl;
+      out << "         mgr = (AliAnalysisManager*)file->Get(key->GetName());" << endl;
+      out << "   };" << endl;
+      out << "   if (!mgr) {" << endl;
+      out << "      ::Error(\"" << func.Data() << "\", \"No analysis manager found in file analysis.root\");" << endl;
+      out << "      return;" << endl;
+      out << "   }" << endl << endl;
+      out << "   mgr->PrintStatus();" << endl;
+      out << "   mgr->StartAnalysis(\"localfile\", chain);" << endl;
+      out << "}" << endl << endl;
+      if (IsUsingTags()) {
+         out << "TChain* CreateChainFromTags(const char *xmlfile, const char *type=\"ESD\")" << endl;
+         out << "{" << endl;
+         out << "// Create a chain using tags from the xml file." << endl;
+         out << "   TAlienCollection* coll = TAlienCollection::Open(xmlfile);" << endl;
+         out << "   if (!coll) {" << endl;
+         out << "      ::Error(\"CreateChainFromTags\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
+         out << "      return NULL;" << endl;
+         out << "   }" << endl;
+         out << "   TGridResult* tagResult = coll->GetGridResult(\"\",kFALSE,kFALSE);" << endl;
+         out << "   AliTagAnalysis *tagAna = new AliTagAnalysis(type);" << endl;
+         out << "   tagAna->ChainGridTags(tagResult);" << endl << endl;
+         out << "   AliRunTagCuts      *runCuts = new AliRunTagCuts();" << endl;
+         out << "   AliLHCTagCuts      *lhcCuts = new AliLHCTagCuts();" << endl;
+         out << "   AliDetectorTagCuts *detCuts = new AliDetectorTagCuts();" << endl;
+         out << "   AliEventTagCuts    *evCuts  = new AliEventTagCuts();" << endl;
+         out << "   // Check if the cuts configuration file was provided" << endl;
+         out << "   if (!gSystem->AccessPathName(\"ConfigureCuts.C\")) {" << endl;
+         out << "      gROOT->LoadMacro(\"ConfigureCuts.C\");" << endl;
+         out << "      ConfigureCuts(runCuts, lhcCuts, detCuts, evCuts);" << endl;
+         out << "   }" << endl;
+         out << "   TChain *chain = tagAna->QueryTags(runCuts, lhcCuts, detCuts, evCuts);" << endl;
+         out << "   if (!chain || !chain->GetNtrees()) return NULL;" << endl;
+         out << "   chain->ls();" << endl;
+         out << "   return chain;" << endl;
+         out << "}" << endl;
+         if (gSystem->AccessPathName("ConfigureCuts.C")) {
+            TString msg = "\n#####   You may want to provide a macro ConfigureCuts.C with a method:\n";
+            msg += "   void ConfigureCuts(AliRunTagCuts *runCuts,\n";
+            msg += "                      AliLHCTagCuts *lhcCuts,\n";
+            msg += "                      AliDetectorTagCuts *detCuts,\n";
+            msg += "                      AliEventTagCuts *evCuts)";
+            Info("WriteAnalysisMacro", msg.Data());
+         }
+      } else {
+         out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\")" << endl;
+         out << "{" << endl;
+         out << "// Create a chain using url's from xml file" << endl;
+         out << "   TString treename = type;" << endl;
+         out << "   treename.ToLower();" << endl;
+         out << "   treename += \"Tree\";" << endl;
+         out << "   printf(\"***************************************\");" << endl;
+         out << "   printf(\"    Getting chain of trees %s\\n\", treename);" << endl;
+         out << "   printf(\"***************************************\");" << endl;
+         out << "   TAlienCollection *coll = TAlienCollection::Open(xmlfile);" << endl;
+         out << "   if (!coll) {" << endl;
+         out << "      ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
+         out << "      return NULL;" << endl;
+         out << "   }" << endl;
+         out << "   TChain *chain = new TChain(treename);" << endl;
+         out << "   coll->Reset();" << endl;
+         out << "   while (coll->Next()) chain->Add(coll->GetTURL(\"\"));" << endl;
+         out << "   if (!chain->GetNtrees()) {" << endl;
+         out << "      ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
+         out << "      return NULL;" << endl;
+         out << "   }" << endl;
+         out << "   return chain;" << endl;
+         out << "}" << endl;
+      }   
+      Info("WriteAnalysisMacro", "\n#####   Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
+   }   
+   Bool_t copy = kTRUE;
+   if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
+   if (copy) {
+      CdWork();
+      TString workdir = gGrid->GetHomeDirectory();
+      workdir += fGridWorkingDir;
+      if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
+      if (IsUsingTags() && !gSystem->AccessPathName("ConfigureCuts.C")) {
+         if (FileExists("ConfigureCuts.C")) gGrid->Rm("ConfigureCuts.C");
+         Info("WriteAnalysisMacro", "\n#####   Copying cuts configuration macro: <ConfigureCuts.C> to your alien workspace");
+         TFile::Cp("file:ConfigureCuts.C", Form("alien://%s/ConfigureCuts.C", workdir.Data()));
+      }   
+      Info("WriteAnalysisMacro", "\n#####   Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
+      TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
+   }
+}
+
+//______________________________________________________________________________
+void AliAnalysisAlien::WriteExecutable()
+{
+// Generate the alien executable script.
+   if (!TestBit(AliAnalysisGrid::kSubmit)) {  
+      ofstream out;
+      out.open(fExecutable.Data(), ios::out);
+      if (out.bad()) {
+         Error("CreateJDL", "Bad file name for executable: %s", fExecutable.Data());
+         return;
+      }
+      out << "#!/bin/bash" << endl;
+      out << "export GCLIENT_SERVER_LIST=\"pcapiserv04.cern.ch:10000|pcapiserv05.cern.ch:10000|pcapiserv06.cern.ch:10000|pcapiserv07.cern.ch:10000\"" << endl;
+      out << "echo \"=========================================\"" << endl; 
+      out << "echo \"############## PATH : ##############\"" << endl;
+      out << "echo $PATH" << endl;
+      out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
+      out << "echo $LD_LIBRARY_PATH" << endl;
+      out << "echo \"############## ROOTSYS : ##############\"" << endl;
+      out << "echo $ROOTSYS" << endl;
+      out << "echo \"############## which root : ##############\"" << endl;
+      out << "which root" << endl;
+      out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
+      out << "echo $ALICE_ROOT" << endl;
+      out << "echo \"############## which aliroot : ##############\"" << endl;
+      out << "which aliroot" << endl;
+      out << "echo \"=========================================\"" << endl << endl;
+//      if (TestBit(AliAnalysisGrid::kTest)) out << "root ";
+      out << "root -b -q "; 
+      out << fAnalysisMacro.Data() << endl << endl;
+      out << "echo \"======== " << fAnalysisMacro.Data() << " finished ========\"" << endl;
+   }   
+   Bool_t copy = kTRUE;
+   if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
+   if (copy) {
+      CdWork();
+      TString workdir = gGrid->GetHomeDirectory();
+      workdir += fGridWorkingDir;
+      TString executable = Form("%s/bin/%s", gGrid->GetHomeDirectory(), fExecutable.Data());
+      if (FileExists(executable)) gGrid->Rm(executable);
+      Info("CreateJDL", "\n#####   Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
+      TFile::Cp(Form("file:%s",fExecutable.Data()), Form("alien://%s", executable.Data()));
+   } 
+}
+
+//______________________________________________________________________________
+void AliAnalysisAlien::WriteValidationScript()
+{
+// Generate the alien validation script.
+   // Generate the validation script
+   TObjString *os;
+   if (!Connect()) {
+      Error("WriteValidationScript", "Alien connection required");
+      return;
+   }
+   TString out_stream = "";
+   if (!TestBit(AliAnalysisGrid::kTest)) out_stream = " >> stdout";
+   if (!TestBit(AliAnalysisGrid::kSubmit)) {  
+      ofstream out;
+      out.open("validate.sh", ios::out);
+      out << "#!/bin/bash" << endl;
+      out << "##################################################" << endl;
+      out << "validateout=`dirname $0`" << endl;
+      out << "validatetime=`date`" << endl;
+      out << "validated=\"0\";" << endl;
+      out << "error=0" << endl;
+      out << "if [ -z $validateout ]" << endl;
+      out << "then" << endl;
+      out << "    validateout=\".\"" << endl;
+      out << "fi" << endl << endl;
+      out << "cd $validateout;" << endl;
+      out << "validateworkdir=`pwd`;" << endl << endl;
+      out << "echo \"*******************************************************\"" << out_stream << endl;
+      out << "echo \"* Automatically generated validation script           *\""  << out_stream << endl;
+      out << "" << endl;
+      out << "echo \"* Time:    $validatetime \""  << out_stream << endl;
+      out << "echo \"* Dir:     $validateout\""  << out_stream << endl;
+      out << "echo \"* Workdir: $validateworkdir\""  << out_stream << endl;
+      out << "echo \"* ----------------------------------------------------*\""  << out_stream << endl;
+      out << "ls -la ./"  << out_stream << endl;
+      out << "echo \"* ----------------------------------------------------*\""  << out_stream << endl << endl;
+      out << "##################################################" << endl;
+      TObjArray *arr = fOutputFiles.Tokenize(" ");
+      TIter next1(arr);
+      TString output_file;
+      while ((os=(TObjString*)next1())) { 
+         output_file = os->GetString();
+         Int_t index = output_file.Index("@");
+         if (index > 0) output_file.Remove(index);
+         out << "if ! [ -f " << output_file.Data() << " ] ; then" << endl;
+         out << "   error=1" << endl;
+         out << "   echo \"Output file(s) not found. Job FAILED !\""  << out_stream << endl;
+         out << "   echo \"Output file(s) not found. Job FAILED !\" >> stderr" << endl;
+         out << "fi" << endl;
+      }   
+      delete arr;
+      out << "if [ $error = 0 ] ; then" << endl;
+      out << "   echo \"* ----------------   Job Validated  ------------------*\""  << out_stream << endl;
+      out << "fi" << endl;
+
+      out << "echo \"* ----------------------------------------------------*\""  << out_stream << endl;
+      out << "echo \"*******************************************************\""  << out_stream << endl;
+      out << "cd -" << endl;
+      out << "exit $error" << endl;
+   }    
+   Bool_t copy = kTRUE;
+   if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
+   if (copy) {
+      CdWork();
+      TString workdir = gGrid->GetHomeDirectory();
+      workdir += fGridWorkingDir;
+      Info("CreateJDL", "\n#####   Copying validation script <validate.sh> to your AliEn working space");
+      if (FileExists("validate.sh")) gGrid->Rm("validate.sh");
+      TFile::Cp("file:validate.sh", Form("alien://%s/validate.sh", workdir.Data()));
+   } 
+}
diff --git a/ANALYSIS/AliAnalysisAlien.h b/ANALYSIS/AliAnalysisAlien.h
new file mode 100644 (file)
index 0000000..f06771d
--- /dev/null
@@ -0,0 +1,112 @@
+#ifndef ALIANALYSISALIEN_H
+#define ALIANALYSISALIEN_H
+/* Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
+ * See cxx source for full Copyright notice                               */
+
+// Author: Mihaela Gheata, 01/09/2008
+
+//==============================================================================
+//   AliAnalysisAlien - AliEn utility class. Provides interface for creating
+// a personalized JDL, finding and creating a dataset.
+//==============================================================================
+
+#ifndef ALIANALYSISGRID_H
+#include "AliAnalysisGrid.h"
+#endif
+
+#ifndef ROOT_TString
+#include <TString.h>
+#endif
+
+class TGridJDL;
+
+class AliAnalysisAlien : public AliAnalysisGrid {
+
+public:
+
+   AliAnalysisAlien();
+   AliAnalysisAlien(const char *name);
+   virtual ~AliAnalysisAlien();
+   AliAnalysisAlien(const AliAnalysisAlien& other); 
+   AliAnalysisAlien& operator=(const AliAnalysisAlien& other);
+// Setters   
+   virtual void        AddRunNumber(Int_t run);
+   virtual void        AddDataFile(const char *lfn);
+   virtual void        SetExecutable(const char *name="analysis.sh")     {fExecutable = name;}
+   virtual void        SetArguments(const char *name="")                 {fArguments = name;}
+   virtual void        SetAnalysisMacro(const char *name="myAnalysis.C") {fAnalysisMacro = name;}
+   virtual void        SetAnalysisSource(const char *name="myAnalysisClass.cxx") {fAnalysisSource = name;}
+   virtual void        SetAdditionalLibs(const char *list)               {fAdditionalLibs = list;}
+   virtual void        SetPrice(Int_t price=1)                           {fPrice = price;}
+   virtual void        SetSplitMode(const char *type="se")               {fSplitMode = type;}
+   virtual void        SetSplitMaxInputFileNumber(Int_t nfiles=100)      {fSplitMaxInputFileNumber = nfiles;}
+   virtual void        SetAPIVersion(const char *version="V2.4") {fAPIVersion = version;}
+   virtual void        SetROOTVersion(const char *version="v5-21-01-alice") {fROOTVersion = version;}
+   virtual void        SetAliROOTVersion(const char *version="v4-14-Rev-02") {fAliROOTVersion=version;}
+   virtual void        SetUser(const char *user)                         {fUser = user;}
+   virtual void        SetTTL(Int_t ttl=30000)                           {fTTL = ttl;}
+   virtual void        SetGridWorkingDir(const char *name="workdir")     {fGridWorkingDir = name;}
+   virtual void        SetGridDataDir(const char *name)                  {fGridDataDir = name;}
+   virtual void        SetDataPattern(const char *pattern="*AliESDs.root") {fDataPattern = pattern;}
+   virtual void        SetGridOutputDir(const char *name="output")       {fGridOutputDir = name;}
+   virtual void        SetOutputArchive(const char *list="log_archive.zip:stdout,stderr root_archive.zip:*.root") {fOutputArchive = list;}
+   virtual void        SetOutputFiles(const char *list)                  {fOutputFiles = list;}
+   virtual void        SetInputFormat(const char *format="xml-single")   {fInputFormat = format;}
+   virtual void        SetMaxInitFailed(Int_t nfail=5)                   {fMaxInitFailed = nfail;}
+   virtual void        SetMasterResubmitThreshold(Int_t percentage)      {fMasterResubmitThreshold = percentage;}
+   virtual void        SetJDLName(const char *name="analysis.jdl")       {fJDLName = name;}
+
+   TGridJDL           *GetGridJDL() {return fGridJDL;}
+//Utilities
+   virtual Bool_t      CreateDataset(const char *pattern);
+   virtual Bool_t      CreateJDL();
+   virtual Bool_t      MergeOutputs();
+   virtual void        StartAnalysis(Long64_t nentries=123456789, Long64_t firstentry=0);
+   virtual void        WriteAnalysisFile();
+   virtual void        WriteAnalysisMacro();
+   virtual void        WriteExecutable();
+   virtual void        WriteValidationScript();
+
+protected:
+   void                CdWork();
+   Bool_t              CheckInputData();
+   void                CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags);
+   virtual Bool_t      Connect();
+   virtual void        SetDefaults();  
+
+   Bool_t              FileExists(const char *lfn) const;
+   Bool_t              IsCollection(const char *lfn) const;
+   Bool_t              IsUsingTags() const {return TObject::TestBit(AliAnalysisGrid::kUseTags);}
+
+private:
+   TGridJDL        *fGridJDL;         //! JDL maker
+   Int_t            fPrice;           // Grid price for the job;
+   Int_t            fTTL;             // Time to live.
+   Int_t            fSplitMaxInputFileNumber; // Maximum number of files to be processed per subjob
+   Int_t            fMaxInitFailed;   // Maximum initial consecutive subjobs accepted to fail
+   Int_t            fMasterResubmitThreshold; // Failed jobs will be resubmitted until this DONE ratio
+   TString          fRunNumbers;      // List of runs to be processed
+   TString          fExecutable;      // Executable script for AliEn job
+   TString          fArguments;       // Arguments for the executable script
+   TString          fAnalysisMacro;   // Root macro steering the analysis
+   TString          fAnalysisSource;  // User analysis implementation (.cxx) file(s)
+   TString          fAdditionalLibs;  // List (separated by blacs) of additional libraries needed for the analysis
+   TString          fSplitMode;       // Job split mode
+   TString          fAPIVersion;      // API version
+   TString          fROOTVersion;     // ROOT version
+   TString          fAliROOTVersion;  // AliROOT version
+   TString          fUser;            // AliEn user name
+   TString          fGridWorkingDir;  // AliEn directory containing the input packages
+   TString          fGridDataDir;     // AliEn data production directory
+   TString          fDataPattern;     // Data pattern for 'find' command
+   TString          fGridOutputDir;   // AliEn directory (wrt work dir) where the output should be written
+   TString          fOutputArchive;   // List of output archives separated by blancs
+   TString          fOutputFiles;     // List of output files separated by blancs
+   TString          fInputFormat;     // Input format (xml-single)
+   TString          fDatasetName;     // Dataset xml file to be created
+   TString          fJDLName;         // JDL file to be generated
+   TObjArray       *fInputFiles;      // List of input files to be processed by the job
+   
+   ClassDef(AliAnalysisAlien, 1)   // Class providing some AliEn utilities
+};
+#endif
diff --git a/ANALYSIS/AliAnalysisGrid.cxx b/ANALYSIS/AliAnalysisGrid.cxx
new file mode 100644 (file)
index 0000000..5b7165e
--- /dev/null
@@ -0,0 +1,160 @@
+/**************************************************************************
+ * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
+ *                                                                        *
+ * Author: The ALICE Off-line Project.                                    *
+ * Contributors are mentioned in the code where appropriate.              *
+ *                                                                        *
+ * Permission to use, copy, modify and distribute this software and its   *
+ * documentation strictly for non-commercial purposes is hereby granted   *
+ * without fee, provided that the above copyright notice appears in all   *
+ * copies and that both the copyright notice and this permission notice   *
+ * appear in the supporting documentation. The authors make no claims     *
+ * about the suitability of this software for any purpose. It is          *
+ * provided "as is" without express or implied warranty.                  *
+ **************************************************************************/
+
+// Author: Mihaela Gheata, 01/09/2008
+
+//==============================================================================
+//   AliAnalysisGrid - Base grid utility class. Provides interface for creating
+// a personalized JDL, finding and creating a dataset.
+//==============================================================================
+
+#include "TSystem.h"
+#include "AliAnalysisGrid.h"
+
+ClassImp(AliAnalysisGrid)
+
+//______________________________________________________________________________
+AliAnalysisGrid::AliAnalysisGrid(const AliAnalysisGrid& other)
+                :TNamed(other)
+{
+// Copy ctor.
+}
+
+//______________________________________________________________________________
+AliAnalysisGrid &AliAnalysisGrid::operator=(const AliAnalysisGrid& other)
+{
+// Assignment.
+   if (this != &other) {
+      TNamed::operator=(other);
+   }
+   return *this;
+}
+
+//______________________________________________________________________________
+Bool_t AliAnalysisGrid::CreateToken(const char *username)
+{
+// Check if a valid token exists - if not create one
+   TString user = gSystem->Getenv("USER");
+   if (!user.Length()) {
+      printf("Error in AliAnalysisGrid::CreateToken: $USER environment empty");
+      return kFALSE;
+   }
+   Int_t err_msg = gSystem->Exec("no_command > /dev/null 2>/dev/null");
+   Int_t token_value = gSystem->Exec("bash alien-token-info > /dev/null 2>/dev/null");
+   if (token_value == err_msg) {
+      printf("Error in AliAnalysisGrid::CreateToken: You do not seem to have <alien-token-info> in your path.");
+      return kFALSE;
+   }
+   
+   Bool_t to_create_token = kFALSE;
+   if (!token_value) {
+      // Token still valid, check alien_API_USER environment
+      TString token_user = gSystem->Getenv("alien_API_USER");
+      if (token_user.Length()) {
+         // Environment file sourced
+         if (!username) return kTRUE;                // for default $USER
+         if (token_user == username) return kTRUE;   // for <username>
+         // A valid token existing, and environment sourced, but for a different user
+         to_create_token = kTRUE;
+      }
+   } else {
+      // Token not valid anymore for <username>. Call alien-token-init   
+      to_create_token = kTRUE;
+   }
+   if (to_create_token) {   
+      printf("______________________________________________________________________________________\n");
+      printf("AliAnalysisGrid::CreateToken: Seems you need a token. Calling alien-token-init for you\n");
+      printf("______________________________________________________________________________________\n");
+      Int_t token_init = 0;
+      if (username) token_init = gSystem->Exec(Form("alien-token-init %s", username));
+      else          token_init = gSystem->Exec("alien-token-init");
+      if (token_init == err_msg) {
+         printf("   Woops - semms alien-token-init is not in your path...\n");
+         return kFALSE;
+      } else if (token_init != 0) {
+         printf("   Woops - did not succeed...\n");
+         return kFALSE;
+      }
+   }
+   // We have a valid token - just source it
+   printf("______________________________________________________________________________________\n");
+   printf("AliAnalysisGrid::CreateToken: Your token needs to be sourced in the current shell\n");
+   printf("   USE:    > source /tmp/gclient_env_%d\n", gSystem->GetUid(user));
+   printf("______________________________________________________________________________________\n");
+   return kFALSE;
+}
+
+//______________________________________________________________________________
+AliAnalysisGrid::EPluginRunMode AliAnalysisGrid::GetRunMode() const
+{
+// Get the current run mode.
+   if (TObject::TestBit(kTest)) return AliAnalysisGrid::kTest;
+   if (TObject::TestBit(kOffline)) return AliAnalysisGrid::kOffline;
+   if (TObject::TestBit(kSubmit)) return AliAnalysisGrid::kSubmit;
+   if (TObject::TestBit(kMerge)) return AliAnalysisGrid::kMerge;
+   return AliAnalysisGrid::kFull;
+}
+   
+//______________________________________________________________________________
+void AliAnalysisGrid::SetRunMode(const char *mode)
+{
+// Set the alien plugin run mode. All modes require presence of a valid token
+// and sourcing the AliEn environment. Supported modes are:
+// - full (default): Generates requested datasets, locally generates the JDL,
+//                   saves existing analysis manager to the file analysis.root,
+//                   generates analysis macro, execution and validation scripts,
+//                   copies all these files to AliEn working space and submits 
+//                   the job leaving user in an AliEn shell.
+// - test          : Generates only 10 entries of the first requested dataset and
+//                   copies this locally as wn.xml, generates all files from the
+//                   full run mode except the JDL and executes the analysis locally.
+//                   This mode can be used to test if the analysis may run in grid.
+// - offline       : No dataset is produced, but all other files are locally generated.
+//                   No file is copied in AliEn workspace. This mode can be used to
+//                   customize the automatic JDL/analysis macro.
+// - submit        : Datasets are generated in AliEn but the JDL and all the other
+//                   files are supposed to exist in the local directory. The files
+//                   are copied to AliEn and the job is submitted. This mode should
+//                   be used in correlation with "offline mode" to submit customized
+//                   analysis macro/jdl.
+// - merge         : Only MergeOutputs() method called to merge the registered
+//                   outputs of a job that finished.
+   TString smode(mode);
+   smode.ToLower();
+   TObject::SetBit(kTest, kFALSE);
+   TObject::SetBit(kOffline, kFALSE);
+   TObject::SetBit(kSubmit, kFALSE);
+   TObject::SetBit(kMerge, kFALSE);
+   if (smode.Contains("test")) {
+      TObject::SetBit(kTest, kTRUE);
+      return;
+   }
+   if (smode.Contains("offline")) {
+      TObject::SetBit(kOffline, kTRUE);
+      return;
+   }
+   if (smode.Contains("submit")) {
+      TObject::SetBit(kSubmit, kTRUE);
+      return;
+   }
+   if (smode.Contains("merge") || smode.Contains("terminate")) {
+      TObject::SetBit(kMerge, kTRUE);
+      return;
+   }
+   if (!smode.Contains("full")) {
+      Warning("SetRunMode","Run mode \"%s\" not known. Supported modes: \"full\", \"test\", \"offline\", \"submit\" and \"merge\"", mode);
+      Warning("SetRunMode","Run mode set to FULL");
+   }   
+}
diff --git a/ANALYSIS/AliAnalysisGrid.h b/ANALYSIS/AliAnalysisGrid.h
new file mode 100644 (file)
index 0000000..a63fce6
--- /dev/null
@@ -0,0 +1,85 @@
+#ifndef ALIANALYSISGRID_H
+#define ALIANALYSISGRID_H
+/* Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
+ * See cxx source for full Copyright notice                               */
+
+// Author: Mihaela Gheata, 01/09/2008
+
+//==============================================================================
+//   AliAnalysisGrid - Base grid utility class. Provides interface for creating
+// a personalized JDL, finding and creating a dataset.
+//==============================================================================
+
+#ifndef ROOT_TNamed
+#include <TNamed.h>
+#endif
+
+class AliAnalysisGrid : public TNamed {
+
+public:
+
+enum EPluginRunMode {
+   kFull     = 0,
+   kTest     = BIT(14),
+   kOffline  = BIT(15),
+   kSubmit   = BIT(16),
+   kMerge    = BIT(17),
+   kUseTags  = BIT(18),
+   kUseESD   = BIT(19),
+   kUseAOD   = BIT(20),
+   kUseMC    = BIT(21)
+};   
+
+   AliAnalysisGrid() {}
+   AliAnalysisGrid(const char *name) : TNamed(name,"") {}
+   virtual ~AliAnalysisGrid() {}
+   AliAnalysisGrid(const AliAnalysisGrid& other); 
+   AliAnalysisGrid& operator=(const AliAnalysisGrid& other);
+// Getters
+   virtual EPluginRunMode GetRunMode() const;
+// Setters   
+   virtual void        AddRunNumber(Int_t run)                           = 0;
+   virtual void        AddDataFile(const char *lfn)                      = 0;
+   virtual void        SetExecutable(const char *name="analysis.sh")     = 0;
+   virtual void        SetArguments(const char *name="")                 = 0;
+   virtual void        SetAnalysisMacro(const char *name="myAnalysis.C") = 0;
+   virtual void        SetAnalysisSource(const char *name="myAnalysisClass.cxx") = 0;
+   virtual void        SetAdditionalLibs(const char *list)               = 0;
+   virtual void        SetPrice(Int_t price=1)                           = 0;
+   virtual void        SetSplitMode(const char *type="se")               = 0;
+   virtual void        SetSplitMaxInputFileNumber(Int_t nfiles=100)      = 0;
+   virtual void        SetAPIVersion(const char *version)                = 0;
+   virtual void        SetROOTVersion(const char *version)               = 0;
+   virtual void        SetAliROOTVersion(const char *version)            = 0;
+   virtual void        SetUser(const char *user)                         = 0;
+   virtual void        SetTTL(Int_t ttl=30000)                           = 0;
+   virtual void        SetGridWorkingDir(const char *name="workdir")     = 0;
+   virtual void        SetGridDataDir(const char *name)                  = 0;
+   virtual void        SetDataPattern(const char *pattern)               = 0;
+   virtual void        SetGridOutputDir(const char *name="output")       = 0;
+   virtual void        SetOutputArchive(const char *list="log_archive.zip:stdout,stderr root_archive.zip:*.root") = 0;
+   virtual void        SetOutputFiles(const char *list)                  = 0;
+   virtual void        SetInputFormat(const char *format="xml-single")   = 0;
+   virtual void        SetMaxInitFailed(Int_t nfail=5)                   = 0;
+   virtual void        SetMasterResubmitThreshold(Int_t percentage)      = 0;
+   virtual void        SetJDLName(const char *name="analysis.jdl")       = 0;
+   
+ // Set run mode.  Can be "full", "test", "offline", "submit" or "merge"
+   virtual void        SetRunMode(const char *mode="full");
+//Utilities
+   static  Bool_t      CreateToken(const char *username=0);
+   virtual Bool_t      CreateDataset(const char *pattern)                = 0;
+   virtual Bool_t      CreateJDL()                                       = 0;
+   virtual Bool_t      MergeOutputs()                                    = 0;
+   virtual void        StartAnalysis(Long64_t nentries=123456789, Long64_t firstentry=0) = 0;
+   virtual void        WriteAnalysisFile()                               = 0;
+   virtual void        WriteAnalysisMacro()                              = 0;
+   virtual void        WriteExecutable()                                 = 0;
+   virtual void        WriteValidationScript()                           = 0;
+
+protected:
+   virtual Bool_t      Connect()                                         = 0;
+   virtual void        SetDefaults()                                     = 0;   
+   ClassDef(AliAnalysisGrid, 1)   // Base class for GRID utilities
+};
+#endif
index 6f19e35..d07b415 100644 (file)
@@ -37,6 +37,7 @@
 #include <TCanvas.h>
 
 #include "AliAnalysisSelector.h"
+#include "AliAnalysisGrid.h"
 #include "AliAnalysisTask.h"
 #include "AliAnalysisDataContainer.h"
 #include "AliAnalysisDataSlot.h"
@@ -56,7 +57,7 @@ AliAnalysisManager::AliAnalysisManager(const char *name, const char *title)
                     fInputEventHandler(NULL),
                     fOutputEventHandler(NULL),
                     fMCtruthEventHandler(NULL),
-                   fEventPool(NULL),
+                    fEventPool(NULL),
                     fCurrentEntry(-1),
                     fNSysInfo(0),
                     fMode(kLocalAnalysis),
@@ -69,7 +70,8 @@ AliAnalysisManager::AliAnalysisManager(const char *name, const char *title)
                     fContainers(NULL),
                     fInputs(NULL),
                     fOutputs(NULL),
-                    fSelector(NULL)
+                    fSelector(NULL),
+                    fGridHandler(NULL)
 {
 // Default constructor.
    fgAnalysisManager = this;
@@ -102,7 +104,8 @@ AliAnalysisManager::AliAnalysisManager(const AliAnalysisManager& other)
                     fContainers(NULL),
                     fInputs(NULL),
                     fOutputs(NULL),
-                    fSelector(NULL)
+                    fSelector(NULL),
+                    fGridHandler(NULL)
 {
 // Copy constructor.
    fTasks      = new TObjArray(*other.fTasks);
@@ -137,6 +140,7 @@ AliAnalysisManager& AliAnalysisManager::operator=(const AliAnalysisManager& othe
       fInputs     = new TObjArray(*other.fInputs);
       fOutputs    = new TObjArray(*other.fOutputs);
       fSelector   = NULL;
+      fGridHandler = NULL;
       fgAnalysisManager = this;
    }
    return *this;
@@ -152,6 +156,7 @@ AliAnalysisManager::~AliAnalysisManager()
    if (fContainers) {fContainers->Delete(); delete fContainers;}
    if (fInputs) delete fInputs;
    if (fOutputs) delete fOutputs;
+   if (fGridHandler) delete fGridHandler;
    if (fgAnalysisManager==this) fgAnalysisManager = NULL;
 }
 
@@ -532,42 +537,49 @@ void AliAnalysisManager::ImportWrappers(TList *source)
    AliAnalysisDataContainer *cont;
    AliAnalysisDataWrapper   *wrap;
    Int_t icont = 0;
+   Bool_t inGrid = (fMode == kGridAnalysis)?kTRUE:kFALSE;
    while ((cont=(AliAnalysisDataContainer*)next())) {
       wrap = 0;
-      if (cont->GetProducer()->IsPostEventLoop()) continue;
+      if (cont->GetProducer()->IsPostEventLoop() && !inGrid) continue;
       const char *filename = cont->GetFileName();
       Bool_t isManagedByHandler = kFALSE;
       if (!(strcmp(filename, "default")) && fOutputEventHandler) {
          isManagedByHandler = kTRUE;
          filename = fOutputEventHandler->GetOutputFileName();
       }
-      if (cont->IsSpecialOutput()) {
+      if (cont->IsSpecialOutput() || inGrid) {
          if (strlen(fSpecialOutputLocation.Data()) && !isManagedByHandler) continue;
-         // Copy merged file from PROOF scratch space
-         char full_path[512];
-         char ch_url[512];
-         TObject *pof =  source->FindObject(filename);
-         if (!pof || !pof->InheritsFrom("TProofOutputFile")) {
-            Error("ImportWrappers", "TProofOutputFile object not found in output list for container %s", cont->GetName());
-            continue;
-         }
-         gROOT->ProcessLine(Form("sprintf((char*)0x%lx, \"%%s\", ((TProofOutputFile*)0x%lx)->GetOutputFileName();)", full_path, pof));
-         gROOT->ProcessLine(Form("sprintf((char*)0x%lx, \"%%s\", gProof->GetUrl();)", ch_url));
-         TString clientUrl(ch_url);
-         TString full_path_str(full_path);
-         if (clientUrl.Contains("localhost")){
-            TObjArray* array = full_path_str.Tokenize ( "//" );
-            TObjString *strobj = ( TObjString *)array->At(1);
-            full_path_str.ReplaceAll(strobj->GetString().Data(),"localhost:11094");
-            if (fDebug > 1) Info("ImportWrappers","Using tunnel from %s to %s",full_path_str.Data(),filename);
-         }
-         if (fDebug > 1) 
-            printf("   Copying file %s from PROOF scratch space\n", full_path_str.Data());
-         Bool_t gotit = TFile::Cp(full_path_str.Data(), filename); 
-         if (!gotit) {
-            Error("ImportWrappers", "Could not get file %s from proof scratch space", cont->GetFileName());
-            continue;
-         }
+         // Copy merged file from PROOF scratch space. 
+         // In case of grid the files are already in the current directory.
+         if (!inGrid) {
+            char full_path[512];
+            char ch_url[512];
+            TObject *pof =  source->FindObject(filename);
+            if (!pof || !pof->InheritsFrom("TProofOutputFile")) {
+               Error("ImportWrappers", "TProofOutputFile object not found in output list for container %s", cont->GetName());
+               continue;
+            }
+            gROOT->ProcessLine(Form("sprintf((char*)0x%lx, \"%%s\", ((TProofOutputFile*)0x%lx)->GetOutputFileName();)", full_path, pof));
+            gROOT->ProcessLine(Form("sprintf((char*)0x%lx, \"%%s\", gProof->GetUrl();)", ch_url));
+            TString clientUrl(ch_url);
+            TString full_path_str(full_path);
+            if (clientUrl.Contains("localhost")){
+               TObjArray* array = full_path_str.Tokenize ( "//" );
+               TObjString *strobj = ( TObjString *)array->At(1);
+               TObjArray* arrayPort = strobj->GetString().Tokenize ( ":" );
+               TObjString *strobjPort = ( TObjString *) arrayPort->At(1);
+               full_path_str.ReplaceAll(strobj->GetString().Data(),"localhost:PORT");
+               full_path_str.ReplaceAll(":PORT",Form(":%s",strobjPort->GetString().Data()));
+               if (fDebug > 1) Info("ImportWrappers","Using tunnel from %s to %s",full_path_str.Data(),filename);
+            }
+            if (fDebug > 1) 
+               printf("   Copying file %s from PROOF scratch space\n", full_path_str.Data());
+            Bool_t gotit = TFile::Cp(full_path_str.Data(), filename); 
+            if (!gotit) {
+               Error("ImportWrappers", "Could not get file %s from proof scratch space", cont->GetFileName());
+               continue;
+            }
+         }   
          // Normally we should connect data from the copied file to the
          // corresponding output container, but it is not obvious how to do this
          // automatically if several objects in file...
@@ -663,8 +675,9 @@ void AliAnalysisManager::Terminate()
    TIter next1(fOutputs);
    AliAnalysisDataContainer *output;
    while ((output=(AliAnalysisDataContainer*)next1())) {
-      // Special outputs have the files already closed and written.
-     if (output->IsSpecialOutput()&&(fMode == kProofAnalysis)) continue;
+      // Special outputs or grid files have the files already closed and written.
+      if (fMode == kGridAnalysis) continue;
+      if (output->IsSpecialOutput()&&(fMode == kProofAnalysis)) continue;
       const char *filename = output->GetFileName();
       if (!(strcmp(filename, "default"))) {
          if (fOutputEventHandler) filename = fOutputEventHandler->GetOutputFileName();
@@ -971,13 +984,39 @@ void AliAnalysisManager::StartAnalysis(const char *type, TTree *tree, Long64_t n
    TString anaType = type;
    anaType.ToLower();
    fMode = kLocalAnalysis;
+   Bool_t runlocalinit = kTRUE;
+   if (anaType.Contains("file")) runlocalinit = kFALSE;
    if (anaType.Contains("proof"))     fMode = kProofAnalysis;
    else if (anaType.Contains("grid")) fMode = kGridAnalysis;
    else if (anaType.Contains("mix"))  fMode = kMixingAnalysis;
 
    if (fMode == kGridAnalysis) {
-      Warning("StartAnalysis", "GRID analysis mode not implemented. Running local.");
-      fMode = kLocalAnalysis;
+      if (!fGridHandler) {
+         Error("StartAnalysis", "Cannot start grid analysis without a grid handler.");
+         Info("===", "Add an AliAnalysisAlien object as plugin for this manager and configure it.");
+         return;
+      }
+      // Write analysis manager in the analysis file
+      cout << "===== RUNNING GRID ANALYSIS: " << GetName() << endl;
+      // run local task configuration
+      TIter nextTask(fTasks);
+      AliAnalysisTask *task;
+      while ((task=(AliAnalysisTask*)nextTask())) {
+         task->LocalInit();
+      }
+      fGridHandler->StartAnalysis(nentries, firstentry);
+
+      // Terminate grid analysis
+      if (fGridHandler->GetRunMode() == AliAnalysisGrid::kOffline) return;
+      cout << "===== MERGING OUTPUTS REGISTERED BY YOUR ANALYSIS JOB: " << GetName() << endl;
+      if (!fGridHandler->MergeOutputs()) {
+         // Return if outputs could not be merged or if it alien handler
+         // was configured for offline mode or local testing.
+         return;
+      }
+      ImportWrappers(NULL);
+      Terminate();
+      return;
    }
    char line[256];
    SetEventLoop(kFALSE);
@@ -998,9 +1037,11 @@ void AliAnalysisManager::StartAnalysis(const char *type, TTree *tree, Long64_t n
    // Initialize locally all tasks (happens for all modes)
    TIter next(fTasks);
    AliAnalysisTask *task;
-   while ((task=(AliAnalysisTask*)next())) {
-      task->LocalInit();
-   }
+   if (runlocalinit) {
+      while ((task=(AliAnalysisTask*)next())) {
+         task->LocalInit();
+      }
+   }   
    
    switch (fMode) {
       case kLocalAnalysis:
index ce0659e..9cd143c 100644 (file)
@@ -25,6 +25,7 @@ class AliAnalysisDataContainer;
 class AliAnalysisTask;
 class AliVEventHandler;
 class AliVEventPool;
+class AliAnalysisGrid;
 
 
 class AliAnalysisManager : public TNamed {
@@ -95,12 +96,14 @@ enum EAliAnalysisFlags {
    void                SetInputEventHandler(AliVEventHandler*  handler)  {fInputEventHandler   = handler;}
    void                SetOutputEventHandler(AliVEventHandler*  handler) {fOutputEventHandler  = handler;}
    void                SetMCtruthEventHandler(AliVEventHandler* handler) {fMCtruthEventHandler = handler;}
+   void                SetGridHandler(AliAnalysisGrid *handler) {fGridHandler = handler;}
    void                SetEventPool(AliVEventPool* epool) {fEventPool = epool;}
    void                SetNSysInfo(Long64_t nevents) {fNSysInfo = nevents;}
    void                SetSelector(AliAnalysisSelector *sel) {fSelector = sel;}
    AliVEventHandler*   GetInputEventHandler()   {return fInputEventHandler;}
    AliVEventHandler*   GetOutputEventHandler()  {return fOutputEventHandler;}
    AliVEventHandler*   GetMCtruthEventHandler() {return fMCtruthEventHandler;}
+   AliAnalysisGrid*    GetGridHandler()         {return fGridHandler;}
    AliVEventPool*      GetEventPool()           {return fEventPool;}
 
    // Container handling
@@ -152,6 +155,7 @@ private:
    TObjArray              *fInputs;              // List of containers with input data
    TObjArray              *fOutputs;             // List of containers with results
    AliAnalysisSelector    *fSelector;            //! Current selector
+   AliAnalysisGrid        *fGridHandler;         //! Grid handler plugin
 
    static AliAnalysisManager *fgAnalysisManager; //! static pointer to object instance
    ClassDef(AliAnalysisManager,3)  // Analysis manager class
index 01fa903..d611c8b 100644 (file)
@@ -2,7 +2,7 @@
 
 SRCS = AliAnalysisDataContainer.cxx AliAnalysisDataSlot.cxx \
        AliAnalysisManager.cxx AliAnalysisTask.cxx \
-       AliAnalysisSelector.cxx \
+       AliAnalysisSelector.cxx AliAnalysisGrid.cxx \
        AliAnalysisFilter.cxx AliAnalysisCuts.cxx 
 
 
index c33e467..fced272 100644 (file)
@@ -15,7 +15,7 @@ CHECKXML = $(shell root-config --has-xml)
 ifeq (yes,$(CHECKXML))
 PACKCXXFLAGS += $(CXXFLAGS) -DWITHXML
 CINTFLAGS += -DWITHXML
-SRCS += AliTagAnalysis.cxx AliXMLCollection.cxx
+SRCS += AliTagAnalysis.cxx AliXMLCollection.cxx AliAnalysisAlien.cxx
 endif