1. SetDefaultOutputs(kTRUE) used by default. This will automatically archive the...

2. Specific SE's are not allowed anymore via the plugin.
3. To specify the number of replicas for the output archives, one should use SetNumberOfReplicas(n) where the default is 2 and the maximum allowed is 4.
4. stdout and stderr files are not registered in the catalog in case a job enters in DONE state.

diff --git a/ANALYSIS/AliAnalysisAlien.cxx b/ANALYSIS/AliAnalysisAlien.cxx
index f999df0..ea46416 100644 (file)
--- a/ANALYSIS/AliAnalysisAlien.cxx
+++ b/ANALYSIS/AliAnalysisAlien.cxx
@@ -61,6 +61,7 @@ AliAnalysisAlien::AliAnalysisAlien()
                   fMergeViaJDL(0),
                   fFastReadOption(0),
                   fOverwriteMode(1),
+                  fNreplicas(2),
                   fRunNumbers(),
                   fExecutable(),
                   fExecutableCommand(),
@@ -118,6 +119,7 @@ AliAnalysisAlien::AliAnalysisAlien(const char *name)
                   fMergeViaJDL(0),
                   fFastReadOption(0),
                   fOverwriteMode(1),
+                  fNreplicas(2),
                   fRunNumbers(),
                   fExecutable(),
                   fExecutableCommand(),
@@ -175,6 +177,7 @@ AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
                   fMergeViaJDL(other.fMergeViaJDL),
                   fFastReadOption(other.fFastReadOption),
                   fOverwriteMode(other.fOverwriteMode),
+                  fNreplicas(other.fNreplicas),
                   fRunNumbers(other.fRunNumbers),
                   fExecutable(other.fExecutable),
                   fExecutableCommand(other.fExecutableCommand),
@@ -262,6 +265,7 @@ AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
       fMergeViaJDL             = other.fMergeViaJDL;
       fFastReadOption          = other.fFastReadOption;
       fOverwriteMode           = other.fOverwriteMode;
+      fNreplicas               = other.fNreplicas;
       fRunNumbers              = other.fRunNumbers;
       fExecutable              = other.fExecutable;
       fExecutableCommand       = other.fExecutableCommand;
@@ -1045,7 +1049,7 @@ Bool_t AliAnalysisAlien::CreateJDL()
          }      
          delete arr;         
       }      
-      arr = fOutputFiles.Tokenize(" ");
+      arr = fOutputFiles.Tokenize(",");
       TIter next(arr);
       Bool_t first = kTRUE;
       const char *comment = "Files to be archived";
@@ -1529,6 +1533,7 @@ void AliAnalysisAlien::Print(Option_t *) const
    printf("=   Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
    if (fMasterResubmitThreshold>0) 
    printf("=   Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
+   printf("=   Number of replicas for the output files_______ %d\n", fNreplicas);
    if (fNrunsPerMaster>0)
    printf("=   Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
    printf("=   Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
@@ -1573,6 +1578,7 @@ void AliAnalysisAlien::SetDefaults()
    fMaxInitFailed              = 0;
    fMasterResubmitThreshold    = 0;
    fNtestFiles                 = 10;
+   fNreplicas                  = 2;
    fRunRange[0]                = 0;
    fRunRange[1]                = 0;
    fNrunsPerMaster             = 1;
@@ -1595,7 +1601,7 @@ void AliAnalysisAlien::SetDefaults()
    fDataPattern                = "*AliESDs.root";  // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
    fFriendChainName            = "";
    fGridOutputDir              = "output";
-   fOutputArchive              = "log_archive.zip:stdout,stderr root_archive.zip:*.root";
+   fOutputArchive              = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
    fOutputFiles                = "";  // Like "AliAODs.root histos.root"
    fInputFormat                = "xml-single";
    fJDLName                    = "analysis.jdl";
@@ -1604,6 +1610,7 @@ void AliAnalysisAlien::SetDefaults()
    fMergeViaJDL                = 0;
    SetUseCopy(kTRUE);
    SetCheckCopy(kTRUE);
+   SetDefaultOutputs(kTRUE);
    fOverwriteMode              = 1;
 }   
 
@@ -1759,7 +1766,7 @@ Bool_t AliAnalysisAlien::MergeOutputs()
       gEnv->SetValue("XNet.ReconnectTimeout",10);
       gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
    }   
-   TObjArray *list = fOutputFiles.Tokenize(" ");
+   TObjArray *list = fOutputFiles.Tokenize(",");
    TIter next(list);
    TObjString *str;
    TString output_file;
@@ -1814,6 +1821,53 @@ void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
    TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
 }
       
+//______________________________________________________________________________
+void AliAnalysisAlien::SetOutputFiles(const char *list)
+{
+// Manually set the output files list.
+// Removes duplicates. Not allowed if default outputs are not disabled.
+   if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
+      Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
+      return;
+   }
+   Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
+   fOutputFiles = "";
+   TString slist = list;
+   if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
+   TObjArray *arr = slist.Tokenize(" "); 
+   TObjString *os;
+   TIter next(arr);
+   TString sout;
+   while ((os=(TObjString*)next())) {
+      sout = os->GetString();
+      if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
+      if (fOutputFiles.Contains(sout)) continue;
+      if (!fOutputFiles.IsNull()) fOutputFiles += ",";
+      fOutputFiles += sout;
+   }
+   delete arr;   
+}      
+
+//______________________________________________________________________________
+void AliAnalysisAlien::SetOutputArchive(const char *list)
+{
+// Manually set the output archive list. Free text - you are on your own...
+// Not allowed if default outputs are not disabled.
+   if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
+      Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
+      return;
+   }
+   Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
+   fOutputArchive = list;
+}
+
+//______________________________________________________________________________
+void AliAnalysisAlien::SetPreferedSE(const char */*se*/)
+{
+// Setting a prefered output SE is not allowed anymore.
+   Warning("SetPreferedSE", "Setting a preferential SE is not allowed anymore via the plugin. Use SetNumberOfReplicas() and SetDefaultOutputs()");
+}
+
 //______________________________________________________________________________
 Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEntry*/)
 {
@@ -1836,14 +1890,21 @@ Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEn
             filename = mgr->GetOutputEventHandler()->GetOutputFileName();
          }
          if (fOutputFiles.Contains(filename)) continue;
-         if (fOutputFiles.Length()) fOutputFiles += " ";
+         if (fOutputFiles.Length()) fOutputFiles += ",";
          fOutputFiles += filename;
       }
       // Add extra files registered to the analysis manager
       if (mgr->GetExtraFiles().Length()) {
-         if (fOutputFiles.Length()) fOutputFiles += " ";
-         fOutputFiles += mgr->GetExtraFiles();
+         if (fOutputFiles.Length()) fOutputFiles += ",";
+         TString extra = mgr->GetExtraFiles();
+         extra.ReplaceAll(" ", ",");
+         // Protection in case extra files do not exist (will it work?)
+         extra.ReplaceAll(".root", "*.root");
+         fOutputFiles += extra;
       }
+      // Compose the output archive.
+      fOutputArchive = "log_archive.zip:std*@disk=1 ";
+      fOutputArchive += Form("root_archive.zip:%s@disk=%d",fOutputFiles.Data(),fNreplicas);
    }
 //   if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
    if (TestBit(AliAnalysisGrid::kOffline)) {
@@ -1899,7 +1960,7 @@ Bool_t AliAnalysisAlien::StartAnalysis(Long64_t /*nentries*/, Long64_t /*firstEn
       Info("StartAnalysis", "\n_______________________________________________________________________ \
       \n   Running analysis script in a daughter shell as on a worker node \
       \n_______________________________________________________________________");
-      TObjArray *list = fOutputFiles.Tokenize(" ");
+      TObjArray *list = fOutputFiles.Tokenize(",");
       TIter next(list);
       TObjString *str;
       TString output_file;
@@ -2620,7 +2681,8 @@ void AliAnalysisAlien::WriteMergingMacro()
          }   
          if (list) delete list;
       }
-      out << endl;
+      out << endl;      
+
       if (fFastReadOption) {
          Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
          out << "// fast xrootd reading enabled" << endl;
@@ -2818,8 +2880,6 @@ void AliAnalysisAlien::WriteExecutable()
       out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $? ========\"" << endl;
       out << "echo \"############## memory after: ##############\"" << endl;
       out << "free -m" << endl;
-      out << "echo \"############## Last 10 lines from dmesg : ##############\"" << endl;
-      out << "dmesg | tail -n 10" << endl;
    }   
    Bool_t copy = kTRUE;
    if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
@@ -2878,8 +2938,6 @@ void AliAnalysisAlien::WriteMergeExecutable()
       out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
       out << "echo \"############## memory after: ##############\"" << endl;
       out << "free -m" << endl;
-      out << "echo \"############## Last 10 lines from dmesg : ##############\"" << endl;
-      out << "dmesg | tail -n 10" << endl;
    }   
    Bool_t copy = kTRUE;
    if (TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
@@ -3004,7 +3062,7 @@ void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
 
       // Part dedicated to the specific analyses running into the train
 
-      TObjArray *arr = fOutputFiles.Tokenize(" ");
+      TObjArray *arr = fOutputFiles.Tokenize(",");
       TIter next1(arr);
       TString output_file;
       AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
@@ -3032,6 +3090,11 @@ void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
       
       out << "if [ $error = 0 ] ; then" << endl;
       out << "   echo \"* ----------------   Job Validated  ------------------*\""  << out_stream << endl;
+      if (!IsKeepLogs()) {
+         out << "   echo \"* === Logs std* will be deleted === \"" << endl;
+         out_stream = "";
+         out << "   rm -f std*" << endl;
+      }            
       out << "fi" << endl;
 
       out << "echo \"* ----------------------------------------------------*\""  << out_stream << endl;

diff --git a/ANALYSIS/AliAnalysisAlien.h b/ANALYSIS/AliAnalysisAlien.h
index d988c55..0e7ab11 100644 (file)
--- a/ANALYSIS/AliAnalysisAlien.h
+++ b/ANALYSIS/AliAnalysisAlien.h
@@ -18,6 +18,10 @@
 #include <TString.h>
 #endif
 
+#ifndef ROOT_TMath
+#include <TMath.h>
+#endif
+
 class TGridJDL;
 
 class AliAnalysisAlien : public AliAnalysisGrid {
@@ -50,9 +54,9 @@ public:
    virtual void        SetMaxMergeFiles(Int_t nfiles)                    {fMaxMergeFiles = nfiles;}
    virtual void        SetSplitMode(const char *type="se")               {fSplitMode = type;}
    virtual void        SetSplitMaxInputFileNumber(Int_t nfiles=100)      {fSplitMaxInputFileNumber = nfiles;}
-   virtual void        SetAPIVersion(const char *version="V2.4") {fAPIVersion = version;}
-   virtual void        SetROOTVersion(const char *version="v5-21-01-alice") {fROOTVersion = version;}
-   virtual void        SetAliROOTVersion(const char *version="v4-14-Rev-02") {fAliROOTVersion=version;}
+   virtual void        SetAPIVersion(const char *version)                {fAPIVersion = version;}
+   virtual void        SetROOTVersion(const char *version)               {fROOTVersion = version;}
+   virtual void        SetAliROOTVersion(const char *version)            {fAliROOTVersion=version;}
    virtual void        SetUser(const char *user)                         {fUser = user;}
    virtual void        SetTTL(Int_t ttl=30000)                           {fTTL = ttl;}
    virtual void        SetGridWorkingDir(const char *name="workdir")     {fGridWorkingDir = name;}
@@ -61,8 +65,8 @@ public:
    virtual void        SetFriendChainName(const char *name="")           {fFriendChainName = name;}
    virtual void        SetDefaultOutputs(Bool_t flag);
    virtual void        SetGridOutputDir(const char *name="output")       {fGridOutputDir = name;}
-   virtual void        SetOutputArchive(const char *list="log_archive.zip:stdout,stderr root_archive.zip:*.root") {fOutputArchive = list;}
-   virtual void        SetOutputFiles(const char *list)                  {fOutputFiles = list;}
+   virtual void        SetOutputArchive(const char *list="log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2");
+   virtual void        SetOutputFiles(const char *list);
    virtual void        SetOutputToRunNo(Int_t mode=1)                    {fOutputToRunNo = mode;}
    virtual void        SetInputFormat(const char *format="xml-single")   {fInputFormat = format;}
    virtual void        SetMaxInitFailed(Int_t nfail=5)                   {fMaxInitFailed = nfail;}
@@ -70,8 +74,9 @@ public:
    virtual void        SetMergeViaJDL(Bool_t on=kTRUE)                   {fMergeViaJDL = on ? 1 : 0;}
    virtual void        SetMasterResubmitThreshold(Int_t percentage)      {fMasterResubmitThreshold = percentage;}
    virtual void        SetNtestFiles(Int_t nfiles)                       {fNtestFiles = nfiles;}
+   virtual void        SetNumberOfReplicas(Int_t ncopies)                {fNreplicas = TMath::Min(ncopies,4);}
    virtual void        SetJDLName(const char *name="analysis.jdl")       {fJDLName = name;}
-   virtual void        SetPreferedSE(const char *se)                     {fCloseSE = se;}
+   virtual void        SetPreferedSE(const char *se);
    virtual void        SetProductionMode(Int_t mode=1)                   {fProductionMode = mode;}
    virtual void        SetRunPrefix(const char *prefix)                  {fRunPrefix = prefix;}
    virtual void        SetOutputSingleFolder(const char *folder)         {fOutputSingle = folder; fSplitMode="file"; fSplitMaxInputFileNumber=1;}
@@ -135,6 +140,7 @@ private:
    Int_t            fMergeViaJDL;     // Enable merging via automatic JDL
    Int_t            fFastReadOption;  // Use xrootd tweaks to reduce timeouts in file access
    Int_t            fOverwriteMode;   // Overwrite existing files if any
+   Int_t            fNreplicas;       // Number of replicas for the output files
    TString          fRunNumbers;      // List of runs to be processed
    TString          fExecutable;      // Executable script for AliEn job
    TString          fExecutableCommand;  // Command(s) to be executed in the executable script
@@ -169,6 +175,6 @@ private:
    TObjArray       *fInputFiles;      // List of input files to be processed by the job
    TObjArray       *fPackages;        // List of packages to be used
    
-   ClassDef(AliAnalysisAlien, 12)   // Class providing some AliEn utilities
+   ClassDef(AliAnalysisAlien, 13)   // Class providing some AliEn utilities
 };
 #endif

diff --git a/ANALYSIS/AliAnalysisGrid.h b/ANALYSIS/AliAnalysisGrid.h
index 6f15888..60bd6f1 100644 (file)
--- a/ANALYSIS/AliAnalysisGrid.h
+++ b/ANALYSIS/AliAnalysisGrid.h
@@ -34,7 +34,8 @@ enum EPluginRunMode {
 enum EPluginBits {
    kBitMask32  = 0xffffffff,
    kUseCopy    = BIT(0),
-   kCheckCopy  = BIT(1)
+   kCheckCopy  = BIT(1),
+   kKeepLogs   = BIT(2)
 };
 
    AliAnalysisGrid() : TNamed(), fSpecialBits(0) {}
@@ -69,7 +70,7 @@ enum EPluginBits {
    virtual void        SetDataPattern(const char *pattern)               = 0;
    virtual void        SetDefaultOutputs(Bool_t flag=kTRUE)              = 0;
    virtual void        SetGridOutputDir(const char *name="output")       = 0;
-   virtual void        SetOutputArchive(const char *list="log_archive.zip:stdout,stderr root_archive.zip:*.root") = 0;
+   virtual void        SetOutputArchive(const char *list="log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2") = 0;
    virtual void        SetOutputFiles(const char *list)                  = 0;
    virtual void        SetInputFormat(const char *format="xml-single")   = 0;
    virtual void        SetMaxInitFailed(Int_t nfail=5)                   = 0;
@@ -104,6 +105,8 @@ enum EPluginBits {
    void                SetUseCopy(Bool_t flag=kTRUE) {SetSpecialBit(kUseCopy,flag);}
    Bool_t              IsCheckCopy() const {return TestSpecialBit(kCheckCopy);}
    void                SetCheckCopy(Bool_t flag=kTRUE) {SetSpecialBit(kCheckCopy,flag);}
+   Bool_t              IsKeepLogs() const {return TestSpecialBit(kKeepLogs);}
+   void                SetKeepLogs(Bool_t flag=kTRUE) {SetSpecialBit(kKeepLogs,flag);}   
 
 protected:
 // Methods