From 8a4881e38c98f761a2ffe42eb73bad5876d46fef Mon Sep 17 00:00:00 2001 From: pcrochet Date: Tue, 6 Aug 2013 15:04:27 +0000 Subject: [PATCH] Better use of TFileMerger to remove intermediate files. Add pass name in final output (Diego) --- PWGPP/MUON/lite/mergeGridFiles.C | 62 ++++++++++---------------------- PWGPP/MUON/lite/runQA.sh | 44 +++++++++++------------ 2 files changed, 38 insertions(+), 68 deletions(-) diff --git a/PWGPP/MUON/lite/mergeGridFiles.C b/PWGPP/MUON/lite/mergeGridFiles.C index 640c9e6d416..758ccb0b1b2 100644 --- a/PWGPP/MUON/lite/mergeGridFiles.C +++ b/PWGPP/MUON/lite/mergeGridFiles.C @@ -36,78 +36,52 @@ void mergeGridFiles(TString outFilename, TString inFileList, TString addPrefix = TString currList = fileList; TString currOutput = "", mergedFiles = ""; - TFileMerger* fileMerger = 0x0; - TObjArray* dirsToMergeArray = 0x0; - TList* sourceList = new TList(); - if ( ! dirsToMerge.IsNull() ) { - dirsToMergeArray = dirsToMerge.Tokenize(" "); - dirsToMergeArray->SetOwner(); - } Bool_t showProgressBar = ! gROOT->IsBatch(); for ( Int_t istep = 0; istep < 100; istep++ ) { + TFileMerger fileMerger(copyLocal); + Int_t mergeType = ( TFileMerger::kRegular | TFileMerger::kAll ); + if ( ! dirsToMerge.IsNull() ) { + fileMerger.AddObjectNames(dirsToMerge.Data()); + mergeType |= TFileMerger::kOnlyListed; + } TObjArray* array = currList.Tokenize(" "); - array->SetOwner(); currList = ""; Int_t nFiles = array->GetEntries(); Int_t subStep = -1; for (Int_t ifile = 0; ifile < nFiles; ifile++ ) { - if ( ! fileMerger ) fileMerger = new TFileMerger(copyLocal); TString currFilename = array->At(ifile)->GetName(); - Bool_t isFileAdded = fileMerger->AddFile(currFilename.Data(), showProgressBar); + Bool_t isFileAdded = fileMerger.AddFile(currFilename.Data(), showProgressBar); if ( ! isFileAdded ) continue; - //printf("New file %s\n", gROOT->GetListOfFiles()->Last()->GetName()); // REMEMBER TO CUT - if ( dirsToMergeArray ) { - if ( ! sourceList ) sourceList = new TList(); - sourceList->Add(gROOT->GetListOfFiles()->Last()); - } - Int_t nFilesToMerge = fileMerger->GetMergeList()->GetEntries(); + Int_t nFilesToMerge = fileMerger.GetMergeList()->GetEntries(); if ( nFilesToMerge % nFilesPerStep != 0 && ifile < nFiles - 1 ) - continue; + continue; // The following part is executed only at the end of each step currOutput = outFilename; if ( nFiles > nFilesPerStep ) { - subStep++; - currOutput.ReplaceAll(".root",Form("_%i_%i.root", istep, subStep)); - AddFileList(currOutput, currList, ""); - } - if ( dirsToMergeArray ) { - TFile* outFile = new TFile(currOutput.Data(), "recreate"); - for ( Int_t idir=0; idirGetEntries(); idir++ ) { - outFile->cd(); - TDirectory* currDir = outFile->mkdir(dirsToMergeArray->At(idir)->GetName()); - fileMerger->MergeRecursive(currDir, sourceList); - } - outFile->Close(); - } - else { - fileMerger->OutputFile(currOutput.Data()); - fileMerger->Merge(); + subStep++; + currOutput.ReplaceAll(".root",Form("_%i_%i.root", istep, subStep)); + AddFileList(currOutput, currList, ""); } + fileMerger.OutputFile(currOutput.Data(),kTRUE,1); // needed when merging single files for specific directories + fileMerger.PartialMerge(mergeType); printf("\nMerged in %s:\n", currOutput.Data()); mergedFiles = ""; for ( Int_t ientry=0; ientryGetMergeList()->At(ientry)->GetName()); + mergedFiles += Form("%s ", fileMerger.GetMergeList()->At(ientry)->GetName()); printf("%s\n\n", mergedFiles.Data()); // remove merged files if ( istep > 0 ) - gSystem->Exec(Form("rm %s", mergedFiles.Data())); - - delete fileMerger; - fileMerger = 0x0; - if ( dirsToMergeArray ) { - delete sourceList; - sourceList = 0x0; - } + gSystem->Exec(Form("rm %s", mergedFiles.Data())); // Write log file to keep trace of files to be merged ofstream logFile(logFilename.Data()); TString logString = ""; for ( Int_t jfile = ifile + 1; jfile < nFiles; jfile++ ) { - logString += Form("%s ", array->At(jfile)->GetName()); + logString += Form("%s ", array->At(jfile)->GetName()); } logString.Append(currList.Data()); logString.ReplaceAll(" ", "\n"); @@ -115,7 +89,6 @@ void mergeGridFiles(TString outFilename, TString inFileList, TString addPrefix = logFile.close(); } // loop on files - delete array; printf("Step %i completed!\n", istep); @@ -251,6 +224,7 @@ void completeProd(TString runListName="runList.txt", TString prodDir = "", TStri mergeFilename = objs->GetString(); if ( mergeStage > 0 && ! mergeFilename.Contains(stageName.Data()) ) continue; + if ( mergeFilename.Contains(".resubmit") ) continue; // Avoid double counting for runs where a resubmission was performed tmpFile << mergeFilename.Data() << endl; } // loop on grid lfns diff --git a/PWGPP/MUON/lite/runQA.sh b/PWGPP/MUON/lite/runQA.sh index 774037ed33b..4637aab64ae 100755 --- a/PWGPP/MUON/lite/runQA.sh +++ b/PWGPP/MUON/lite/runQA.sh @@ -60,21 +60,32 @@ if [[ $# -ne 3 || "$EXIT" -eq 1 ]]; then exit 4 fi +runListName="$1" +qaProdName="$2" +if [ $isPrivateProd -eq 1 ]; then + tmpName=${qaProdName//"private"/""} + if [ "$tmpName" == "$qaProdName" ]; then + qaProdName="${qaProdName}_private" + fi +fi +alienBaseDir="$3" +lhcPeriod=`echo ${alienBaseDir%"/"} | awk -F "/" ' { print $NF } '` +outFileSuffix="${lhcPeriod}_${qaProdName}" +outFileSuffix=${outFileSuffix//"__"/"_"} + loadAnalysisLibs="gSystem->Load(\"libANALYSIS.so\");gSystem->Load(\"libOADB.so\");gSystem->Load(\"libANALYSISalice.so\");gSystem->Load(\"libCORRFW.so\");gSystem->Load(\"libPWGmuon.so\");gSystem->Load(\"libPWGmuondep.so\");" -includeAliroot="gSystem->AddIncludePath(\"-I${ALICE_ROOT}/include\");" +includeAliroot="gSystem->AddIncludePath(\"-I${ALICE_ROOT}/include -I${ALICE_INSTALL}/include\");" includeMuon="gSystem->AddIncludePath(\"-I${ALICE_ROOT}/MUON\");" + function mergePerRun() { echo "Merging each run..." cd $baseOutDir - runListName="$1" - prodDir="$2" - alienBaseDir="$3" aliroot -b < $mergeLog .L $qaMacroDir/mergeGridFiles.C+ -completeProd("${runListName}","${prodDir}","${alienBaseDir}","${outTaskName}",50,"MUON_QA MTR_ChamberEffMap MUON.TrigEfficiencyMap MUON.TriggerEfficiencyMap",${mergeFast}); +completeProd("${runListName}","${qaProdName}","${alienBaseDir}","${outTaskName}",50,"MUON_QA MTR_ChamberEffMap MUON.TrigEfficiencyMap MUON.TriggerEfficiencyMap",${mergeFast}); .q EOF } @@ -156,11 +167,9 @@ function runTrackQA() { echo "Running tracker QA" cd $terminateDir physSel="$1" - alienBaseDir="$2" - lhcPeriod=`echo ${alienBaseDir%"/"} | awk -F "/" ' { print $NF } '` aliroot -b < logTrackQA.txt ${includeAliroot} ${loadAnalysisLibs} -.x $qaMacroDir/PlotMuonQA.C+("${terminateDir}",0x0,${inputTriggerList},${physSel},"${lhcPeriod}","${outTaskName}"); +.x $qaMacroDir/PlotMuonQA.C+("${terminateDir}",0x0,${inputTriggerList},${physSel},"${outFileSuffix}","${outTaskName}"); .q EOF cd $baseOutDir @@ -175,16 +184,8 @@ if [ "${inputTriggerList}" != "0x0" ]; then inputTriggerList="\"${inputTriggerList}\"" fi -qaProdName="$2" -if [ $isPrivateProd -eq 1 ]; then - tmpName=${qaProdName//"private"/""} - if [ "$tmpName" == "$qaProdName" ]; then - qaProdName="${qaProdName}_private" - fi -fi - if [ $execMerge -eq 1 ]; then - mergePerRun $1 $qaProdName $3 + mergePerRun fi mergeOut=`grep -A 1 "Output written" ${mergeLog} | grep -v written` mergeOutAll=${mergeOut//".txt"/"_merged.txt"} @@ -204,16 +205,11 @@ if [ $execTerminate -eq 1 ]; then fi if [ $execTrigQA -eq 1 ]; then -# minRun=`echo ${mergeOut} | cut -d "_" -f 2` -# maxRun=`echo ${mergeOut} | cut -d "_" -f 3 | cut -d "." -f 1` -# trigOutSuffix=`echo ${qaProdName} | awk -F "/" '{ print $NF }'` - trigOutSuffix=`echo $3 | awk -F "/" '{ print $NF }'` -# outName="trigEffQA_${minRun}_${maxRun}_${trigOutSuffix}.root" - outName="trigEffQA_${trigOutSuffix}.root" + outName="trigEffQA_${outFileSuffix}.root" runTrigQA "${mergeOut}" "${outName}" fi if [ $execTrackQA -eq 1 ]; then - runTrackQA 1 $3 + runTrackQA 1 fi -- 2.43.0