]>
Commit | Line | Data |
---|---|---|
674af238 | 1 | /* |
2 | merge output calib objects on Alien | |
3 | using AliFileMerger functionality | |
4 | ||
5 | Directory with runCalibTrain output: outputDir | |
3e8637f0 | 6 | pattern: AliESDfriends_v1.root |
674af238 | 7 | Output file name: CalibObjects.root |
8 | ||
3e8637f0 | 9 | Example: |
674af238 | 10 | .L $ALICE_ROOT/ANALYSIS/CalibMacros/MergeCalibration/merge.C |
11 | merge("alien:///alice/cern.ch/user/j/jotwinow/CalibTrain/output","AliESDfriends_v1.root"); | |
12 | */ | |
a21c8945 | 13 | void mergeInChunksTXT(const char* mlist, const char* dest, int maxFiles=700); |
674af238 | 14 | |
a21c8945 | 15 | |
16 | void merge(const char* outputDir, const char* pattern, Bool_t copyLocal=kFALSE) | |
3e8637f0 | 17 | { |
674af238 | 18 | // |
19 | // load libraries | |
20 | // | |
a21c8945 | 21 | printf("Merging with chunks copying turned %s\n",copyLocal ? "ON":"OFF"); |
674af238 | 22 | gROOT->Macro("LoadLibraries.C"); |
23 | // | |
a21c8945 | 24 | cpTimeOut(outputDir, pattern,10, copyLocal); |
674af238 | 25 | // |
3e8637f0 | 26 | // local |
a21c8945 | 27 | mergeInChunksTXT("calib.list","CalibObjects.root"); |
28 | // AliFileMerger merger; | |
29 | // merger.AddReject("esdFriend"); // do not merge | |
30 | // merger.SetMaxFilesOpen(700); | |
31 | // merger.IterTXT("calib.list","CalibObjects.root",kFALSE); | |
3e8637f0 | 32 | |
33 | // alien | |
34 | //merger.IterAlien(outputDir, "CalibObjects.root", pattern); | |
674af238 | 35 | |
a21c8945 | 36 | return; |
674af238 | 37 | } |
3e8637f0 | 38 | |
a21c8945 | 39 | void cpTimeOut(const char * searchdir, const char* pattern, Int_t timeOut=10, Bool_t copyLocal) |
3e8637f0 | 40 | { |
41 | ||
42 | gSystem->Setenv("XRDCLIENTMAXWAIT",Form("%d",timeOut)); | |
43 | gEnv->SetValue("XNet.RequestTimeout", timeOut); | |
44 | gEnv->SetValue("XNet.ConnectTimeout", timeOut); | |
45 | gEnv->SetValue("XNet.TransactionTimeout", timeOut); | |
46 | TFile::SetOpenTimeout(timeOut); | |
47 | ||
48 | TGrid::Connect("alien"); | |
49 | ||
50 | TString filelist; | |
51 | TString command; | |
52 | command = Form("find %s/ %s", searchdir, pattern); | |
53 | cerr<<"command: "<<command<<endl; | |
54 | TGridResult *res = gGrid->Command(command); | |
55 | if (!res) return; | |
a21c8945 | 56 | res->Print(); |
3e8637f0 | 57 | TIter nextmap(res); |
58 | TMap *map = 0; | |
59 | ||
60 | ofstream outputFile; | |
61 | outputFile.open(Form("calib.list")); | |
62 | Int_t counter=0; | |
63 | ||
a21c8945 | 64 | while((map=(TMap*)nextmap())) |
3e8637f0 | 65 | { |
66 | TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl")); | |
67 | if (!objs || !objs->GetString().Length()) | |
68 | { | |
69 | delete res; | |
70 | break; | |
71 | } | |
72 | ||
73 | TString src=Form("%s",objs->GetString().Data()); | |
74 | TString dst=src; | |
a21c8945 | 75 | Bool_t result = kTRUE; |
76 | if (copyLocal) { | |
77 | dst.ReplaceAll("alien:///",""); | |
78 | dst.ReplaceAll("/","_"); | |
79 | TTimeStamp s1; | |
80 | result = TFile::Cp(src.Data(),dst.Data(),kTRUE); | |
81 | TTimeStamp s2; | |
82 | AliSysInfo::AddStamp(dst.Data(),counter, result); | |
83 | } | |
84 | if (result) { | |
3e8637f0 | 85 | counter++; |
86 | outputFile << dst.Data()<< endl; | |
87 | } | |
88 | } | |
a21c8945 | 89 | if (copyLocal) cout<<counter<<" files copied!"<<endl; |
90 | else cout<<counter<<" files registerd!"<<endl; | |
3e8637f0 | 91 | outputFile.close(); |
278563e2 | 92 | gSystem->Exec("mv syswatch.log syswatch_copy.log"); |
3e8637f0 | 93 | return; |
94 | } | |
95 | ||
a21c8945 | 96 | void mergeInChunksTXT(const char* mlist, const char* dest, int maxFiles) |
97 | { | |
98 | TH1::AddDirectory(0); | |
99 | AliFileMerger merger; | |
100 | // merger.SetMaxFilesOpen(999); | |
101 | merger.AddReject("esdFriend"); // do not merge | |
102 | // | |
103 | if (maxFiles<2) maxFiles = 2; | |
104 | TString filesToMerge = mlist, fileDest = dest; | |
105 | if (filesToMerge.IsNull()) {printf("List to merge is not provided\n"); return;} | |
106 | if (fileDest.IsNull()) {printf("Merging destination is not provided\n"); return;} | |
107 | const char* tmpMerge[3]={"__merge_part0.root","__merge_part1.root","__part_of_calib.list"}; | |
108 | // | |
109 | gSystem->ExpandPathName(filesToMerge); | |
110 | ofstream outfile; | |
111 | ifstream infile(filesToMerge.Data()); | |
112 | if (!infile) {printf("No %s file\n",filesToMerge.Data()); return;} | |
113 | // | |
114 | int currTmp = 0, nfiles = 0, nparts = 0; // counter for number of merging operations | |
115 | string line; | |
116 | TString lineS; | |
117 | while ( !infile.eof() ) { | |
118 | getline(infile, line); | |
119 | lineS = line; | |
120 | if (lineS.IsNull() || lineS.BeginsWith("#")) continue; | |
121 | int st = nfiles%maxFiles; | |
122 | if (st==0) { // new chunk should be started | |
123 | if (nfiles) { // merge prev. chunk | |
124 | outfile.close(); | |
125 | merger.IterTXT(tmpMerge[2], tmpMerge[currTmp] ,kFALSE); | |
126 | printf("Merging to %s | %d files at step %d\n",tmpMerge[currTmp], nfiles,nparts); | |
127 | } | |
128 | outfile.open(tmpMerge[2], ios::out); // start list for new chunk | |
129 | if (nparts++) { | |
130 | printf("Adding previous result %s | %d files %d at part\n",tmpMerge[currTmp], nfiles,nparts); | |
131 | outfile << tmpMerge[currTmp] << endl; // result of previous merge goes 1st | |
132 | } | |
133 | currTmp = (currTmp==0) ? 1:0; // swap tmp files | |
134 | } | |
135 | outfile << line << endl; | |
136 | nfiles++; | |
137 | } | |
138 | // merge the rest | |
139 | merger.IterTXT(tmpMerge[2], dest ,kFALSE); | |
140 | outfile.close(); | |
141 | infile.close(); | |
142 | for (int i=0;i<3;i++) gSystem->Exec(Form("if [ -e %s ]; then \nrm %s\nfi",tmpMerge[i],tmpMerge[i])); | |
143 | printf("Merged %d files in %d steps\n",nfiles, nparts); | |
144 | // | |
145 | } | |
146 |