Added possibility to collect input file information from the analysis manager, via...
authoragheata <agheata@f7af4fe6-9843-0410-8265-dc069ae4e863>
Thu, 12 Jul 2012 11:58:37 +0000 (11:58 +0000)
committeragheata <agheata@f7af4fe6-9843-0410-8265-dc069ae4e863>
Thu, 12 Jul 2012 11:58:37 +0000 (11:58 +0000)
ANALYSIS/ANALYSISLinkDef.h
ANALYSIS/AliAnalysisDataContainer.cxx
ANALYSIS/AliAnalysisDataContainer.h
ANALYSIS/AliAnalysisManager.cxx
ANALYSIS/AliAnalysisManager.h

index 8a29b7c..609a03d 100644 (file)
@@ -13,5 +13,6 @@
 #pragma link C++ class  AliAnalysisGrid+;
 #pragma link C++ class  AliAnalysisStatistics+;
 #pragma link C++ class  AliAnalysisTaskCfg+;
+#pragma link C++ class  AliAnalysisFileDescriptor+;
 
 #endif
index f875e86..fee2617 100644 (file)
 #include <TMethodCall.h>
 
 #include <TClass.h>
+#include <TSystem.h>
 #include <TFile.h>
 #include <TTree.h>
 #include <TH1.h>
 #include <TROOT.h>
+#include <TStopwatch.h>
 
 #include "AliAnalysisManager.h"
 #include "AliAnalysisDataContainer.h"
@@ -495,3 +497,136 @@ Long64_t AliAnalysisDataWrapper::Merge(TCollection *list)
 
    return count+1;
 }
+
+ClassImp(AliAnalysisFileDescriptor)
+
+//______________________________________________________________________________
+AliAnalysisFileDescriptor::AliAnalysisFileDescriptor()
+                          :TObject(), fLfn(), fGUID(), fUrl(), fPfn(), fSE(),
+                           fIsArchive(kFALSE), fImage(0), fNreplicas(0), 
+                           fStartBytes(0), fReadBytes(0), fSize(0), fOpenedAt(0), 
+                           fOpenTime(0.), fProcessingTime(0.), fThroughput(0.)
+{
+// I/O constructor
+}
+
+//______________________________________________________________________________
+AliAnalysisFileDescriptor::AliAnalysisFileDescriptor(const TFile *file)
+                          :TObject(), fLfn(), fGUID(), fUrl(), fPfn(), fSE(),
+                           fIsArchive(kFALSE), fImage(0), fNreplicas(0), 
+                           fStartBytes(0), fReadBytes(0), fSize(0), fOpenedAt(0), 
+                           fOpenTime(0.), fProcessingTime(0.), fThroughput(0.)
+{
+// Normal constructor
+   if (file->InheritsFrom("TAlienFile")) {
+      fLfn =(const char*)gROOT->ProcessLine(Form("((TAlienFile*)%p)->GetLfn();", file));
+      fGUID =(const char*)gROOT->ProcessLine(Form("((TAlienFile*)%p)->GetGUID();", file));
+      fUrl =(const char*)gROOT->ProcessLine(Form("((TAlienFile*)%p)->GetUrl();", file));
+      fPfn =(const char*)gROOT->ProcessLine(Form("((TAlienFile*)%p)->GetPfn();", file));
+      fSE = (const char*)gROOT->ProcessLine(Form("((TAlienFile*)%p)->GetSE();", file));
+      fImage = (Int_t)gROOT->ProcessLine(Form("((TAlienFile*)%p)->GetImage();", file));
+      fNreplicas = (Int_t)gROOT->ProcessLine(Form("((TAlienFile*)%p)->GetNreplicas();", file));
+      fOpenedAt = gROOT->ProcessLine(Form("((TAlienFile*)%p)->GetOpenTime();", file));
+      gROOT->ProcessLine(Form("((AliAnalysisFileDescriptor*)%p)->SetOpenTime(((TAlienFile*)%p)->GetElapsed());", this, file));
+   } else {
+      fLfn = file->GetName();
+      fPfn = file->GetName();
+      fUrl = file->GetName();
+      fSE = "local";
+      if (!fPfn.BeginsWith("/")) fPfn.Prepend(Form("%s/",gSystem->WorkingDirectory()));
+      fOpenedAt = time(0);
+   }
+   fStartBytes = TFile::GetFileBytesRead();
+   fIsArchive = file->IsArchive();
+   fSize = file->GetSize();
+}
+
+//______________________________________________________________________________
+AliAnalysisFileDescriptor::AliAnalysisFileDescriptor(const AliAnalysisFileDescriptor &other)
+                          :TObject(other), fLfn(other.fLfn), fGUID(other.fGUID),
+                           fUrl(other.fUrl), fPfn(other.fPfn), fSE(other.fSE),
+                           fIsArchive(other.fIsArchive), fImage(other.fImage),
+                           fNreplicas(other.fNreplicas), fStartBytes(other.fStartBytes), fReadBytes(other.fReadBytes),
+                           fSize(other.fSize), fOpenedAt(other.fOpenedAt), fOpenTime(other.fOpenTime),
+                           fProcessingTime(other.fProcessingTime), fThroughput(other.fThroughput)
+{
+// CC
+}
+
+//______________________________________________________________________________
+AliAnalysisFileDescriptor &AliAnalysisFileDescriptor::operator=(const AliAnalysisFileDescriptor &other)
+{
+// Assignment.
+   if (&other == this) return *this;
+   TObject::operator=(other);
+   fLfn       = other.fLfn;
+   fGUID      = other.fGUID;
+   fUrl       = other.fUrl; 
+   fPfn       = other.fPfn;
+   fSE        = other.fSE;
+   fIsArchive = other.fIsArchive;
+   fImage     = other.fImage;
+   fNreplicas = other.fNreplicas;
+   fStartBytes = other.fStartBytes;;
+   fReadBytes = other.fReadBytes;
+   fSize      = other.fSize;
+   fOpenedAt  = other.fOpenedAt;
+   fOpenTime  = other.fOpenTime;
+   fProcessingTime = other.fProcessingTime;
+   fThroughput = other.fThroughput;
+   return *this;
+}
+
+//______________________________________________________________________________
+AliAnalysisFileDescriptor::~AliAnalysisFileDescriptor()
+{
+// Destructor
+}
+
+//______________________________________________________________________________
+void AliAnalysisFileDescriptor::Done()
+{
+// Must be called at the end of processing, providing file->GetBytesRead() as argument.
+   const Double_t megabyte = 1048576.;
+   Long64_t stampnow = time(0);
+   fReadBytes = TFile::GetFileBytesRead()-fStartBytes;
+   fProcessingTime = stampnow-fOpenedAt;
+   Double_t readsize = fReadBytes/megabyte;
+   fThroughput = readsize/fProcessingTime;
+}   
+
+//______________________________________________________________________________
+void AliAnalysisFileDescriptor::Print(Option_t*) const
+{
+// Print info about the file descriptor
+   const Double_t megabyte = 1048576.;
+   printf("===== Logical file name: %s =====\n", fLfn.Data());
+   printf("      Pfn: %s\n", fPfn.Data());
+   printf("      url: %s\n", fUrl.Data());
+   printf("      access time: %lld from SE: %s  image %d/%d\n", fOpenedAt, fSE.Data(), fImage, fNreplicas);
+   printf("      open time: %g [sec]\n", fOpenTime);
+   printf("      file size: %g [MB],  read size: %g [MB]\n", fSize/megabyte, fReadBytes/megabyte);
+   printf("      processing time [sec]: %g\n", fProcessingTime);
+   printf("      average throughput: %g [MB/sec]\n", fThroughput);
+}
+
+//______________________________________________________________________________
+void AliAnalysisFileDescriptor::SavePrimitive(ostream &out, Option_t *)
+{
+// Stream info to file
+   const Double_t megabyte = 1048576.;
+   out << "#################################################################" << endl;
+   out << "pfn          " << fPfn.Data() << endl;
+   out << "url          " << fUrl.Data() << endl;
+   out << "se           " << fSE.Data() << endl;
+   out << "image        " << fImage << endl;
+   out << "nreplicas    " << fNreplicas << endl;
+   out << "openstamp    " << fOpenedAt << endl;
+   out << setiosflags(ios::fixed) << setprecision(3);
+   out << "opentime     " << fOpenTime << endl;
+   out << "runtime      " << fProcessingTime << endl;
+   out << "filesize     " << fSize/megabyte << endl;
+   out << "readsize     " << fReadBytes/megabyte << endl;
+   out << "throughput   " << fThroughput << endl;
+}
+   
index 7297a3a..0a6df3c 100644 (file)
@@ -140,4 +140,62 @@ protected:
    ClassDef(AliAnalysisDataWrapper, 1) // Data wrapper class for exchange via the net
 };
 
+//==============================================================================
+//   AliAnalysisFileDescriptor - A simple wrapper for data related to accessing
+//      an analysis input file.
+//==============================================================================
+class TStopwatch;
+
+class AliAnalysisFileDescriptor : public TObject {
+
+public:
+   AliAnalysisFileDescriptor();
+   AliAnalysisFileDescriptor(const TFile *file);
+   AliAnalysisFileDescriptor(const AliAnalysisFileDescriptor &other);
+   virtual ~AliAnalysisFileDescriptor();
+
+   // Assignment
+   AliAnalysisFileDescriptor &operator=(const AliAnalysisFileDescriptor &other);
+   
+   void         Done();
+   
+   const char  *GetGUID() const  {return fGUID;}
+   Int_t        GetImage() const {return fImage;}
+   const char  *GetLfn() const   {return fLfn;}
+   Int_t        GetNreplicas() const {return fNreplicas;}
+   Long64_t     GetOpenAt() const {return fOpenedAt;}
+   Double_t     GetOpenTime() const {return fOpenTime;}
+   const char  *GetPfn() const   {return fPfn;}
+   Long64_t     GetReadBytes() const {return fReadBytes;}
+   Long64_t     GetSize() const   {return fSize;}
+   const char  *GetSE() const     {return fSE;}
+   Double_t     GetThroughput() const {return fThroughput;}
+   Double_t     GetProcessingTime() const {return fProcessingTime;}
+   const char  *GetUrl() const    {return fUrl;}
+   Bool_t       IsArchive() const {return fIsArchive;}
+   virtual void Print(Option_t *option="") const;
+   virtual void SavePrimitive(ostream &out, Option_t *option = "");
+   void         SetOpenTime(Double_t time) {fOpenTime = time;}
+   
+
+protected:   
+   TString      fLfn;       // logical file name
+   TString      fGUID;      // GUID
+   TString      fUrl;       // url for the opened copy
+   TString      fPfn;       // physical file name
+   TString      fSE;        // Storage element
+   Bool_t       fIsArchive; // Archive flag
+   Int_t        fImage;     // Image number
+   Int_t        fNreplicas; // Number of replicas
+   Long64_t     fStartBytes;// Total number of read bytes at start
+   Long64_t     fReadBytes; // Number of bytes read
+   Long64_t     fSize;      // Size of the file in bytes
+   Long64_t     fOpenedAt;  // Absolute value for time when opened
+   Double_t     fOpenTime;  // Time elapsed to open file
+   Double_t     fProcessingTime; // Processing
+   Double_t     fThroughput; // Throughput
+   
+   ClassDef(AliAnalysisFileDescriptor,1)  // Class describing a a file processed in the analysis
+};
+
 #endif
index ed0a154..b738df2 100644 (file)
@@ -66,11 +66,11 @@ Int_t AliAnalysisManager::fPBUpdateFreq = 1;
 //______________________________________________________________________________
 AliAnalysisManager::AliAnalysisManager(const char *name, const char *title)
                    :TNamed(name,title),
-                    fTree(NULL),
-                    fInputEventHandler(NULL),
-                    fOutputEventHandler(NULL),
-                    fMCtruthEventHandler(NULL),
-                    fEventPool(NULL),
+                    fTree(0),
+                    fInputEventHandler(0),
+                    fOutputEventHandler(0),
+                    fMCtruthEventHandler(0),
+                    fEventPool(0),
                     fCurrentEntry(-1),
                     fNSysInfo(0),
                     fMode(kLocalAnalysis),
@@ -79,19 +79,22 @@ AliAnalysisManager::AliAnalysisManager(const char *name, const char *title)
                     fIsRemote(kFALSE),
                     fDebug(0),
                     fSpecialOutputLocation(""), 
-                    fTasks(NULL),
-                    fTopTasks(NULL),
-                    fZombies(NULL),
-                    fContainers(NULL),
-                    fInputs(NULL),
-                    fOutputs(NULL),
-                    fParamCont(NULL),
-                    fDebugOptions(NULL),
-                    fCommonInput(NULL),
-                    fCommonOutput(NULL),
-                    fSelector(NULL),
-                    fGridHandler(NULL),
-                    fExtraFiles(""),
+                    fTasks(0),
+                    fTopTasks(0),
+                    fZombies(0),
+                    fContainers(0),
+                    fInputs(0),
+                    fOutputs(0),
+                    fParamCont(0),
+                    fDebugOptions(0),
+                    fFileDescriptors(new TObjArray()),
+                    fCurrentDescriptor(0),
+                    fCommonInput(0),
+                    fCommonOutput(0),
+                    fSelector(0),
+                    fGridHandler(0),
+                    fExtraFiles(),
+                    fFileInfoLog(),
                     fAutoBranchHandling(kTRUE), 
                     fTable(),
                     fRunFromPath(0),
@@ -142,11 +145,14 @@ AliAnalysisManager::AliAnalysisManager(const AliAnalysisManager& other)
                     fOutputs(NULL),
                     fParamCont(NULL),
                     fDebugOptions(NULL),
+                    fFileDescriptors(new TObjArray()),
+                    fCurrentDescriptor(0),
                     fCommonInput(NULL),
                     fCommonOutput(NULL),
                     fSelector(NULL),
                     fGridHandler(NULL),
-                    fExtraFiles(),
+                    fExtraFiles(other.fExtraFiles),
+                    fFileInfoLog(other.fFileInfoLog),
                     fAutoBranchHandling(other.fAutoBranchHandling), 
                     fTable(),
                     fRunFromPath(0),
@@ -194,11 +200,14 @@ AliAnalysisManager& AliAnalysisManager::operator=(const AliAnalysisManager& othe
       fOutputs    = new TObjArray(*other.fOutputs);
       fParamCont  = new TObjArray(*other.fParamCont);
       fDebugOptions = NULL;
+      fFileDescriptors = new TObjArray();
+      fCurrentDescriptor = 0;
       fCommonInput = NULL;
       fCommonOutput = NULL;
       fSelector   = NULL;
       fGridHandler = NULL;
       fExtraFiles = other.fExtraFiles;
+      fFileInfoLog = other.fFileInfoLog;
       fgCommonFileName = "AnalysisResults.root";
       fgAnalysisManager = this;
       fAutoBranchHandling = other.fAutoBranchHandling;
@@ -233,6 +242,7 @@ AliAnalysisManager::~AliAnalysisManager()
    if (fEventPool) delete fEventPool;
    if (fgAnalysisManager==this) fgAnalysisManager = NULL;
    if (fGlobals) {fGlobals->DeleteAll(); delete fGlobals;}
+   if (fFileDescriptors) {fFileDescriptors->Delete(); delete fFileDescriptors;}
 }
 
 //______________________________________________________________________________
@@ -242,7 +252,9 @@ Int_t AliAnalysisManager::GetEntry(Long64_t entry, Int_t getall)
    fCurrentEntry = entry;
    if (!fAutoBranchHandling)
      return 123456789;
-   return fTree ? fTree->GetTree()->GetEntry(entry, getall) : -1;
+   if (!fTree) return -1;  
+   Long64_t readbytes = fTree->GetTree()->GetEntry(entry, getall);
+   return (Int_t)readbytes;
 }
 
 //______________________________________________________________________________
@@ -464,6 +476,11 @@ Bool_t AliAnalysisManager::Notify()
       Error("Notify","No current file");
       return kFALSE;
    }   
+   if (IsCollectThroughput()) {
+      if (fCurrentDescriptor) fCurrentDescriptor->Done();
+      fCurrentDescriptor = new AliAnalysisFileDescriptor(curfile);
+      fFileDescriptors->Add(fCurrentDescriptor);
+   }   
    
    if (fDebug > 1) printf("->AliAnalysisManager::Notify() file: %s\n", curfile->GetName());
    Int_t run = AliAnalysisManager::GetRunFromAlienPath(curfile->GetName());
@@ -526,6 +543,19 @@ void AliAnalysisManager::PackOutput(TList *target)
   // Pack all output data containers in the output list. Called at SlaveTerminate
   // stage in PROOF case for each slave.
    if (fDebug > 1) printf("->AliAnalysisManager::PackOutput()\n");
+   if (IsCollectThroughput()) {
+      if (fCurrentDescriptor) fCurrentDescriptor->Done();
+      fFileDescriptors->Print();
+      if (fFileInfoLog.IsNull()) fFileInfoLog = "fileinfo.log";
+      std::ofstream out;
+      out.open(fFileInfoLog, std::ios::out);
+      if (out.bad()) Error("SavePrimitive", "Bad file name: %s", fFileInfoLog.Data());
+      else {
+         TIter nextflog(fFileDescriptors);
+         TObject *log;
+         while ((log=nextflog())) log->SavePrimitive(out,"");
+      }
+   }   
    if (!target) {
       Error("PackOutput", "No target. Exiting.");
       return;
@@ -1457,7 +1487,9 @@ void AliAnalysisManager::CheckBranches(Bool_t load)
          }
       }   
       fTable.Add(br);
-      if (load && br->GetReadEntry()!=GetCurrentEntry()) br->GetEntry(GetCurrentEntry());
+      if (load && br->GetReadEntry()!=GetCurrentEntry()) {
+         br->GetEntry(GetCurrentEntry());
+      }      
    }
   delete arr;
 }
@@ -2380,8 +2412,8 @@ void AliAnalysisManager::DoLoadBranch(const char *name)
     fTable.Add(br);
   }
   if (br->GetReadEntry()==fCurrentEntry) return;
-  Int_t ret = br->GetEntry(GetCurrentEntry());
-  if (ret<0) {
+  Long64_t readbytes = br->GetEntry(GetCurrentEntry());
+  if (readbytes<0) {
     Error("DoLoadBranch", "Could not load entry %lld from branch %s",GetCurrentEntry(), name);
     if (crtEntry != fCurrentEntry) {
       CountEvent(1,0,1,0);
index 0ff257c..2ec01aa 100644 (file)
@@ -30,6 +30,7 @@ class TStopwatch;
 class TMap;
 class AliAnalysisSelector;
 class AliAnalysisDataContainer;
+class AliAnalysisFileDescriptor;
 class AliAnalysisTask;
 class AliVEventHandler;
 class AliVEventPool;
@@ -64,8 +65,9 @@ enum EAliAnalysisFlags {
    kSkipTerminate    = BIT(19),
    kUseProgressBar   = BIT(20),
    kTrueNotify       = BIT(21),
-   kTasksInitialized = BIT(22)
-};   
+   kTasksInitialized = BIT(22),
+   kCollectThroughput= BIT(23)
+};
 
    AliAnalysisManager(const char *name = "mgr", const char *title="");
    virtual            ~AliAnalysisManager();
@@ -107,6 +109,7 @@ enum EAliAnalysisFlags {
    TString             GetExtraFiles() const      {return fExtraFiles;}
    AliVEventPool*      GetEventPool()  const      {return fEventPool;}
    Bool_t              GetFileFromWrapper(const char *filename, const TList *source);
+   const char         *GetFileInfoLog() const     {return fFileInfoLog.Data();}
    static Int_t        GetRunFromAlienPath(const char *path);
    AliAnalysisGrid*    GetGridHandler()           {return fGridHandler;}
    TObjArray          *GetInputs() const          {return fInputs;}
@@ -129,6 +132,7 @@ enum EAliAnalysisFlags {
    static Bool_t       IsPipe(std::ostream &out);
    Bool_t              IsProofMode() const        {return (fMode==kProofAnalysis)?kTRUE:kFALSE;}
    Bool_t              IsRemote() const           {return fIsRemote;}
+   Bool_t              IsCollectThroughput()      {return TObject::TestBit(kCollectThroughput);}
    Bool_t              IsUsingDataSet() const     {return TObject::TestBit(kUseDataSet);}
    void                LoadBranch(const char *n)  { if(fAutoBranchHandling) return; DoLoadBranch(n); }
    void                RunLocalInit();
@@ -136,11 +140,13 @@ enum EAliAnalysisFlags {
    void                SetAutoBranchLoading(Bool_t b) { fAutoBranchHandling = b; }
    void                SetCurrentEntry(Long64_t entry)            {fCurrentEntry = entry;}
    void                SetCollectSysInfoEach(Int_t nevents=0)     {fNSysInfo = nevents;}
+   void                SetCollectThroughput(Bool_t flag)          {TObject::SetBit(kCollectThroughput,flag);}
    static void         SetCommonFileName(const char *name)        {fgCommonFileName = name;}
    void                SetDebugLevel(UInt_t level);
    void                SetDisableBranches(Bool_t disable=kTRUE)   {TObject::SetBit(kDisableBranches,disable);}
    void                SetExternalLoop(Bool_t flag)               {TObject::SetBit(kExternalLoop,flag);}
    void                SetEventPool(AliVEventPool* const epool)   {fEventPool = epool;}
+   void                SetFileInfoLog(const char *name) {TObject::SetBit(kCollectThroughput,kTRUE); fFileInfoLog = name;}
    void                SetGridHandler(AliAnalysisGrid * const handler) {fGridHandler = handler;}
    void                SetInputEventHandler(AliVEventHandler* const handler);
    void                SetMCtruthEventHandler(AliVEventHandler* const handler) {fMCtruthEventHandler = handler;}
@@ -234,11 +240,14 @@ private:
    TObjArray              *fOutputs;             // List of containers with results
    TObjArray              *fParamCont;           // List of containers with results
    TObjArray              *fDebugOptions;        // List of debug options
+   TObjArray              *fFileDescriptors;     //! List of file descriptors
+   AliAnalysisFileDescriptor *fCurrentDescriptor; //! Current file descriptor
    AliAnalysisDataContainer *fCommonInput;       // Common input container
    AliAnalysisDataContainer *fCommonOutput;      // Common output container
    AliAnalysisSelector    *fSelector;            //! Current selector
    AliAnalysisGrid        *fGridHandler;         //! Grid handler plugin
    TString                 fExtraFiles;          // List of extra files to be merged
+   TString                 fFileInfoLog;         // File name for fileinfo logs
    Bool_t                  fAutoBranchHandling;  // def=kTRUE, turn off if you use LoadBranch
    THashTable              fTable;               // keep branch ptrs in case of manual branch loading
    Int_t                   fRunFromPath;         // Run number retrieved from path to input data
@@ -251,6 +260,6 @@ private:
    TMap                   *fGlobals;             // Map with global variables
    static TString          fgCommonFileName;     //! Common output file name (not streamed)
    static AliAnalysisManager *fgAnalysisManager; //! static pointer to object instance
-   ClassDef(AliAnalysisManager,16)  // Analysis manager class
+   ClassDef(AliAnalysisManager,17)  // Analysis manager class
 };   
 #endif