1 #if !defined(__CINT__) || defined(__MAKECINT__)
8 #include "TObjString.h"
9 #include "TFileMerger.h"
13 #include "TGridResult.h"
19 Bool_t AddFileList(TString, TString&, TString);
20 void ReadListFromFile(TString, TString&, TString);
21 Int_t GetLastStage(TGridResult*);
23 void mergeGridFiles(TString outFilename, TString inFileList, TString addPrefix = "alien://", Int_t nFilesPerStep = 100, Bool_t copyLocal = kTRUE, TString dirsToMerge = "")
25 TString fileList = "";
26 ReadListFromFile(inFileList, fileList, addPrefix);
28 if ( fileList.IsNull() ) {
29 printf("List of merging files is null: Nothing done!\n");
33 TString logFilename = "toBeMerged.txt";
35 if ( ! gGrid ) copyLocal = kFALSE;
37 TString currList = fileList;
38 TString currOutput = "", mergedFiles = "";
39 TFileMerger* fileMerger = 0x0;
40 TObjArray* dirsToMergeArray = 0x0;
41 TList* sourceList = new TList();
42 if ( ! dirsToMerge.IsNull() ) {
43 dirsToMergeArray = dirsToMerge.Tokenize(" ");
44 dirsToMergeArray->SetOwner();
47 Bool_t showProgressBar = ! gROOT->IsBatch();
49 for ( Int_t istep = 0; istep < 100; istep++ ) {
50 TObjArray* array = currList.Tokenize(" ");
53 Int_t nFiles = array->GetEntries();
55 for (Int_t ifile = 0; ifile < nFiles; ifile++ ) {
56 if ( ! fileMerger ) fileMerger = new TFileMerger(copyLocal);
57 TString currFilename = array->At(ifile)->GetName();
58 Bool_t isFileAdded = fileMerger->AddFile(currFilename.Data(), showProgressBar);
59 if ( ! isFileAdded ) continue;
60 //printf("New file %s\n", gROOT->GetListOfFiles()->Last()->GetName()); // REMEMBER TO CUT
61 if ( dirsToMergeArray ) {
62 if ( ! sourceList ) sourceList = new TList();
63 sourceList->Add(gROOT->GetListOfFiles()->Last());
66 Int_t nFilesToMerge = fileMerger->GetMergeList()->GetEntries();
67 if ( nFilesToMerge % nFilesPerStep != 0 && ifile < nFiles - 1 )
69 // The following part is executed only at the end of each step
70 currOutput = outFilename;
71 if ( nFiles > nFilesPerStep ) {
73 currOutput.ReplaceAll(".root",Form("_%i_%i.root", istep, subStep));
74 AddFileList(currOutput, currList, "");
76 if ( dirsToMergeArray ) {
77 TFile* outFile = new TFile(currOutput.Data(), "recreate");
78 for ( Int_t idir=0; idir<dirsToMergeArray->GetEntries(); idir++ ) {
80 TDirectory* currDir = outFile->mkdir(dirsToMergeArray->At(idir)->GetName());
81 fileMerger->MergeRecursive(currDir, sourceList);
86 fileMerger->OutputFile(currOutput.Data());
89 printf("\nMerged in %s:\n", currOutput.Data());
91 for ( Int_t ientry=0; ientry<nFilesToMerge; ientry++ )
92 mergedFiles += Form("%s ", fileMerger->GetMergeList()->At(ientry)->GetName());
93 printf("%s\n\n", mergedFiles.Data());
95 // remove merged files
97 gSystem->Exec(Form("rm %s", mergedFiles.Data()));
101 if ( dirsToMergeArray ) {
106 // Write log file to keep trace of files to be merged
107 ofstream logFile(logFilename.Data());
108 TString logString = "";
109 for ( Int_t jfile = ifile + 1; jfile < nFiles; jfile++ ) {
110 logString += Form("%s ", array->At(jfile)->GetName());
112 logString.Append(currList.Data());
113 logString.ReplaceAll(" ", "\n");
114 logFile << logString.Data() << endl;;
120 printf("Step %i completed!\n", istep);
122 if ( nFiles <= nFilesPerStep ) break;
125 gSystem->Exec(Form("rm %s", logFilename.Data()));
129 //___________________________________________________
130 Bool_t AddFileList(TString filename, TString& fileList, TString addPrefix)
132 if ( filename.IsNull() || ! filename.Contains(".root") ) return kFALSE;
134 if ( ! addPrefix.IsNull() && ! filename.Contains(addPrefix.Data()) )
135 filename.Prepend(addPrefix.Data());
137 if ( filename.Contains("alien://") && ! gGrid )
138 TGrid::Connect("alien://");
140 if ( ! fileList.IsNull() )
141 fileList.Append(" ");
142 fileList.Append(filename.Data());
147 //___________________________________________________
148 void ReadListFromFile(TString filename, TString& fileList, TString addPrefix)
150 ifstream inFile(filename.Data());
151 TString currLine = "";
152 if ( inFile.is_open() ) {
153 while ( ! inFile.eof() ) {
154 currLine.ReadLine(inFile,kTRUE); // Read line
155 AddFileList(currLine, fileList, addPrefix);
162 //___________________________________________________
163 void completeProd(TString runListName="runList.txt", TString prodDir = "", TString baseDir="/alice/data/2010/LHC10h", TString outTaskFilename="QAresults.root", Int_t nFilesPerStep = 50, TString dirsToMerge = "MUON_QA MTR_ChamberEffMap", Bool_t mergeFast = kFALSE, Bool_t overwriteExisting = kFALSE)
165 TString outFilename = "completeFileList.txt";
167 // Get run list from file
168 ifstream inFile(runListName.Data());
172 Int_t minRun = 99999999;
174 if (inFile.is_open()) {
175 while (! inFile.eof() ) {
176 currRun.ReadLine(inFile,kTRUE); // Read line
177 if ( currRun.IsNull() || ! currRun.IsDigit() ) continue;
178 Int_t currRunInt = currRun.Atoi();
179 minRun = TMath::Min(currRunInt, minRun);
180 maxRun = TMath::Max(currRunInt, maxRun);
181 runList.Add(new TObjString(Form("%d", currRunInt)));
186 outFilename.ReplaceAll(".txt", Form("_%i_%i.txt", minRun, maxRun));
188 //TString filePattern[3] = {"", "Stage*/","*/"};
189 TString filePattern[2] = {"","*/"};
191 ofstream outFile(outFilename.Data());
193 const Int_t kNlibs = 5; // 1
194 TString loadLibs[kNlibs] = {"libANALYSIS.so", "libOADB.so", "libANALYSISalice.so", "libCORRFW.so", "libPWGmuon.so"};
195 for ( Int_t ilib=0; ilib<kNlibs; ilib++ ) {
196 Int_t exitVal = gSystem->Load(loadLibs[ilib].Data());
198 printf("Please run with aliroot if you're merging QA objects!\n");
204 TGrid::Connect("alien://");
206 baseDir.ReplaceAll("alien://","");
209 TString stageName = "";
210 TString runsWithoutOut = "";
211 for ( Int_t irun=0; irun<runList.GetEntries(); irun++ ) {
212 TString currRunString = ((TObjString*)runList.At(irun))->GetString();
214 TString localOut = outTaskFilename;
215 localOut.ReplaceAll(".root", Form("_%s.root", currRunString.Data()));
216 if ( ! gSystem->AccessPathName(localOut.Data()) ) {
217 if ( overwriteExisting )
218 printf("Overwriting existing file %s\n", localOut.Data());
220 printf("Warning: merged file %s already exist: do not overwrite\n", localOut.Data());
221 outFile << gSystem->pwd() << "/" << localOut.Data() << endl;
226 TString tmpFilename = Form("/tmp/mergeListRun%s.txt", currRunString.Data());
227 ofstream tmpFile(tmpFilename.Data());
228 TString mergeFilename = "";
230 Int_t nPatterns = ( mergeFast ) ? 1 : 2;
232 for ( Int_t ipattern=0; ipattern<nPatterns; ipattern++ ) {
233 TString command = ( prodDir.Contains("private") ) ? Form("find %s/ *%s/%s%s", baseDir.Data(), currRunString.Data(), filePattern[ipattern].Data(), outTaskFilename.Data()) : Form("find %s/*%s %s/%s%s", baseDir.Data(), currRunString.Data(), prodDir.Data(), filePattern[ipattern].Data(), outTaskFilename.Data());
235 printf("%s\n", command.Data());
237 TGridResult* res = gGrid->Command(command);
239 if ( ! res || res->GetEntries() == 0 ) continue;
241 Int_t mergeStage = ( ipattern == 1 ) ? GetLastStage(res) : -1;
242 stageName = Form("Stage_%i", mergeStage);
245 while ( ( map = (TMap*)nextmap() ) ) {
246 // Loop 'find' results and get next LFN
247 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
248 if (!objs || !objs->GetString().Length())
251 mergeFilename = objs->GetString();
253 if ( mergeStage > 0 && ! mergeFilename.Contains(stageName.Data()) ) continue;
255 tmpFile << mergeFilename.Data() << endl;
256 } // loop on grid lfns
262 // If the merging of specific directories is required
263 // run merging also if there is 1 file
264 // so that we get rid of other sub-directories
265 if ( ipattern == 1 || ! dirsToMerge.IsNull() ) {
266 mergeFilename = outTaskFilename;
267 mergeFilename.ReplaceAll(".root", Form("_%s.root", currRunString.Data()));
268 mergeGridFiles(mergeFilename, tmpFilename, "alien://", nFilesPerStep, kTRUE, dirsToMerge);
271 if ( ! mergeFilename.Contains("alien://") )
272 outFile << gSystem->pwd() << "/";
273 outFile << mergeFilename.Data() << endl;
274 gSystem->Exec(Form("rm %s", tmpFilename.Data()));
277 if ( mergeFilename.IsNull() ) runsWithoutOut += currRunString + " ";
279 if ( ! runsWithoutOut.IsNull() )
280 printf("\nNo output found in runs\n%s\n",runsWithoutOut.Data());
281 printf("\nOutput written in:\n%s\n", outFilename.Data());
285 //___________________________________________________
286 Int_t GetLastStage(TGridResult* res)
292 TString filename = "", currToken = "";
293 while ( ( map = (TMap*)nextmap() ) ) {
294 // Loop 'find' results and get next LFN
295 TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
296 if (!objs || !objs->GetString().Length())
299 filename = objs->GetString();
301 if ( ! filename.Contains("Stage_") ) continue;
303 TObjArray* array = filename.Tokenize("/");
305 for ( Int_t ientry=0; ientry<array->GetEntries(); ientry++ ) {
306 currToken = array->At(ientry)->GetName();
307 if ( currToken.Contains("Stage_") ) {
308 currToken.Remove(0,6);
309 Int_t currStage = currToken.Atoi();
310 lastStage = TMath::Max(currStage, lastStage);
315 } // loop on grid lfns