Updated version of the Test TPC toolkits (Marian)
[u/mrichter/AliRoot.git] / TPC / macros / testTPC / AlienToolkit.cxx
CommitLineData
7332008b 1#include "TGrid.h"
2#include "TGridCollection.h"
3#include "TSystem.h"
4#include "TString.h"
5#include "TFile.h"
6#include "TMap.h"
7#include "TGridResult.h"
8#include "TAlien.h"
9#include "TAlienResult.h"
10#include <fstream>
11
12
13
14
15
16/*
17 gSystem->Load("libXrdClient.so");
18 gSystem->Load("libNetx.so");
19 .L $ALICE_ROOT/TPC/macros/testTPC/AlienToolkit.cxx+
20 //Reconstructed data example
21 char *mask = "v14*";
22 char *path = "/alice/cern.ch/user/m/miranov/test2007/"
23 // raw data
027ef365 24 char *mask = "0000191*RecPo*.root";
25 char *path = "/alice/data/2008/LHC08a_TPC/"
7332008b 26
27 AlienToolkit toolkit;
28 toolkit.MakeCollection(path,mask);
29
30 //
027ef365 31 toolkit.MakeJobList("job.list","", "", "rec");
7332008b 32
33*/
34
35
36
37class AlienToolkit: public TObject{
38public:
39 AlienToolkit();
40 TGridCollection* MakeCollection(const char *path, char *mask);
41 void Stage();
42 void LocalCopy(const char* destination);
43 void RemoteCopy(const char* destination="root://gsiaf.gsi.de:1094/", Int_t maxfiles=20);
027ef365 44 void PrintPFN();
7332008b 45 void MakeJobList(const char * outname, const char *outputPrefix, const char *action, const char *suffix);
46 static Bool_t IsDir(const char * name);
47 static Bool_t IsFile(const char * name);
48 static Bool_t ResubmitJobs();
027ef365 49
7332008b 50public:
51 TGridCollection *fCollection;
52 TObjArray fInfoArray;
53 ClassDef(AlienToolkit,1)
54};
55
56ClassImp(AlienToolkit)
57
58AlienToolkit::AlienToolkit():
59 fCollection(0)
60{
61 if (!gGrid)
62 TGrid::Connect("alien://",0,0,"t");
63}
64
65
66
67
68Bool_t AlienToolkit::IsDir(const char * name){
69 //
70 // Check if it is directory
71 //
72 void *dir = gSystem->OpenDirectory(name);
73 return (dir!=0);
74}
75
76
77Bool_t AlienToolkit::IsFile(const char *name ){
78 //
79 // Check if is it file
80 //
81 Long_t id, size, flags, modtime ;
82 Int_t ret = gSystem->GetPathInfo(name,&id, &size, &flags, &modtime);
83 if (ret==0) return kFALSE;
84 if ((flags&2)>0) return kFALSE;
85 return kTRUE;
86}
87
88
89TGridCollection* AlienToolkit::MakeCollection(const char *path, char *mask){
90 //
91 /*char *mask = "root_archive.zip";
92 char *path = "/alice/cern.ch/user/m/miranov/test2007/"
93 */
94 //
95 TGridResult* query = gGrid->Query(path,mask);
96 TGridCollection* collection=gGrid->OpenCollectionQuery(query);
97 // collection->SelectFile(mask,1,20); // select files 0-100
98 collection->LookupSUrls();
99 //collection->CheckIfOnline();
100 collection->Reset();
101 fCollection = collection;
102 //
103 collection->Reset();
104 Int_t counter=0;
105 Int_t counterm=0;
106 Int_t counterf=0;
107 TMap *filemap;
108 while ( (filemap = collection->Next())) {
109 TIterator *nextfile = filemap->MakeIterator();
110 TMap *attributes;
111 while ((attributes = (TMap *) nextfile->Next())) {
112 printf("%d\t%d\t%d\n",counter, counterm,counterf);
113 TMap * map = new TMap;
114 fInfoArray.AddLast(map);
115
116 TObjString *surl = new TObjString(collection->GetSURL(attributes->GetName()));
117 TObjString *turl = new TObjString(collection->GetTURL(attributes->GetName()));
118 TObjString *lfn = new TObjString(collection->GetLFN(attributes->GetName()));
119 map->Add(new TObjString("alienLFN"),lfn);
120 map->Add(new TObjString("alienSURL"),surl);
121 map->Add(new TObjString("alienTURL"),turl);
122
123
124 Int_t isOnline = collection->IsOnline(attributes->GetName());
125 printf("Base Name:\t%s\n",attributes->GetName());
126 printf("Size:\t%d\n",(Int_t)collection->GetSize());
127 printf("IsOnline:\t%d\n",collection->IsOnline());
128 printf("IsSelected:\t%d\n",collection->IsSelected());
129 printf("IsOnline:\t%d\n",isOnline);
130 printf("LFN Name:\t%s\n",lfn->String().Data());
131 printf("TURL Name:\t%s\n",turl->String().Data());
132 printf("SURL Name:\t%s\n",surl->String().Data());
133 counter++;
134 counterf++;
135
136 }
137 counterf=0;
138 counterm++;
139 }
140}
141
142
143void AlienToolkit::Stage(){
144 //
145 // Stage selected alien files
146 //
147 Int_t entries = fInfoArray.GetEntries();
148 ofstream aout("stage.txt");
149 for (Int_t i=0; i<entries;i++){
150 TMap &map = *((TMap*)fInfoArray.At(i));
151 TObjString *lfn = (TObjString*)map("alienLFN");
152 if (!lfn) continue;
153 printf("Staging submitfor\t%s\n",lfn->String().Data());
154 aout<<"stage "<<lfn->String().Data()<<endl;
155 }
156 aout.close();
157 gSystem->Exec("aliensh file:stage.txt");
158}
159
160void AlienToolkit::LocalCopy(const char *destination){
161 //
162 // Copy selected files to the destination directory
163 // the LFN path name translated to the directory name replacing
164 // separtor - the flat structure is created
165
166 Int_t entries = fInfoArray.GetEntries();
167 ofstream aout("stage.txt");
168 for (Int_t i=0; i<entries;i++){
169 TMap &map = *((TMap*)fInfoArray.At(i));
170 TObjString *lfn = (TObjString*)map("alienLFN");
171 if (!lfn) continue;
172 printf("Staging submitfor\t%s\n",lfn->String().Data());
173 TString dnames=lfn->String().Data();
174 dnames.ReplaceAll(".root_dir","");
175 dnames.ReplaceAll("/","_");
176 TString dname=destination;
177 dname+=dnames;
178 aout<<"cp "<<lfn->String().Data()<<" "<<dname.Data()<<endl;
179 }
180 aout.close();
181 gSystem->Exec("aliensh file:stage.txt");
182}
183
184
027ef365 185void AlienToolkit::PrintPFN(){
186 //
187 //
188 //
189 Int_t entries = fInfoArray.GetEntries();
190 for (Int_t i=0; i<entries;i++){
191 TMap &map = *((TMap*)fInfoArray.At(i));
192 TObjString *lfn = (TObjString*)map("alienLFN");
193 TObjString *pfn = (TObjString*)map("alienSURL");
194 if (!lfn) continue;
195 if (!pfn) continue;
196 printf("%s\n",pfn->String().Data());
197 }
198}
199
7332008b 200void AlienToolkit::MakeJobList(const char * outname, const char *outputPrefix, const char *action, const char *suffix){
201 //
202 //
203 //
204 Int_t entries = fInfoArray.GetEntries();
205 ofstream aout(outname);
206 for (Int_t i=0; i<entries;i++){
207 TMap &map = *((TMap*)fInfoArray.At(i));
208 TObjString *lfn = (TObjString*)map("alienLFN");
209 TObjString *pfn = (TObjString*)map("alienSURL");
210 if (!lfn) continue;
211 if (!pfn) continue;
9af56b10 212 if (lfn->String().Contains(".tag.")) continue;
7332008b 213 printf("Job info\t%s\n",lfn->String().Data());
214 TString jobID=lfn->String().Data();
215 jobID.ReplaceAll("/","_");
216 //
217 //
027ef365 218 //
219 //
7332008b 220 TString outputDir=outputPrefix;
221 outputDir+=lfn->String().Data();
222
497c5329 223 aout<<jobID<<" "<<pfn->String().Data()<<" "<<outputDir.Data()<<" "<<action;
224 if (suffix) aout<<" "<<suffix<<"\n";
7332008b 225 aout<<endl;
226 }
227 aout.close();
228 //
229}
230
231
232void AlienToolkit::RemoteCopy(const char *destination,Int_t maxfiles){
233 //
234 // Copy selected files to the destination directory
235 // the LFN path name translated to the directory name replacing
236 // separtor - the flat structure is created
237
238 Int_t entries = fInfoArray.GetEntries();
239 ofstream *aout=0;
240 for (Int_t i=0; i<entries;i++){
241 if (i%maxfiles==0){
242 if (aout) aout->close();
243 aout = new ofstream(Form("stage_%d.sh",i/maxfiles));
244 (*aout)<<"!/bin/bash\n";
245 (*aout)<<"source ~/.balice\n";
246 }
247 TMap &map = *((TMap*)fInfoArray.At(i));
248 TObjString *pfn = (TObjString*)map("alienSURL");
249 TObjString *lfn = (TObjString*)map("alienLFN");
250 if (!pfn) continue;
251 if (!lfn) continue;
252 TString dnames=lfn->String().Data();
253 TString dname=destination;
254 dname+=dnames;
255 (*aout)<<"xrdcp -d 1 "<<pfn->String().Data()<<" "<<dname.Data()<<endl;
256 }
257 aout->close();
258 //gSystem->Exec("aliensh file:stage.txt");
259}
260
261Bool_t AlienToolkit::ResubmitJobs(){
262 //
263 // Resubmit the processes finished in error state
264 //
265 //gSystem->Exec("alien_ps | grep EE > ps.txt")
266 gSystem->Exec("alien_ps | grep EE |gawk '{print $2}' > psEE.txt");
267 gSystem->Exec("alien_ps | grep EXPIRED |gawk '{print $2}' > psEXPIRED.txt");
268 //
269 ifstream in;
270 in.open("psEXPIRED.txt");
271 TString line;
272 TString result;
273 while(in.good()) {
274 in >> line;
275 // line.ReplaceAll("-","");
276 printf("Resubmiting %s\n",line.Data());
277 Int_t pos = line.First("-");
278 printf("%s\n",Form("alien_resubmit %s",&line.Data()[pos+1]));
279 gSystem->Exec(Form("alien_resubmit %s",&line.Data()[pos+1]));
280 }
281 in.close();
282}
283
284
285
286
287
288/*
289 Problems:
2901. ???? fCollection.IsOnline("")
291
292 #12 0x04e9a03f in XrdClientAdmin::IsFileOnline (this=0x94a91b0, vs=@0xbfe07f60, vb=@0xbfe07f80) at ../../src/XrdClient/XrdClientVector.hh:55
293#13 0x0423a975 in TXNetSystem::IsOnline (this=0x9652800,
294 path=0x9661488 "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91")
295 at netx/src/TXNetSystem.cxx:476
296#14 0x04237c6e in TXNetFileStager::IsStaged (this=0x96526b8,
297 path=0x9661400 "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91")
298 at netx/src/TXNetFileStager.cxx:63
299#15 0x00551117 in TFileStager::GetStaged (this=0x96526b8, pathlist=0x9649820) at net/src/TFileStager.cxx:58
300#16 0x010e5d96 in TAlienCollection::CheckIfOnline (this=0x95da8d8, bulk=true) at alien/src/TAlienCollection.cxx:1143
301
302
3032. id xrootd on voalice04.cern.ch ?
304 "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91"
305
3063. How to check if the file exist in redirector ?
307
3084. Ho
309
310*/
311