Filter out tag files (Marian)
[u/mrichter/AliRoot.git] / TPC / macros / testTPC / AlienToolkit.cxx
CommitLineData
7332008b 1#include "TGrid.h"
2#include "TGridCollection.h"
3#include "TSystem.h"
4#include "TString.h"
5#include "TFile.h"
6#include "TMap.h"
7#include "TGridResult.h"
8#include "TAlien.h"
9#include "TAlienResult.h"
10#include <fstream>
11
12
13
14
15
16/*
17 gSystem->Load("libXrdClient.so");
18 gSystem->Load("libNetx.so");
19 .L $ALICE_ROOT/TPC/macros/testTPC/AlienToolkit.cxx+
20 //Reconstructed data example
21 char *mask = "v14*";
22 char *path = "/alice/cern.ch/user/m/miranov/test2007/"
23 // raw data
24 char *mask = "07000015*.root";
25 char *path = "/alice/data/2007/LHC07w_TPC/"
26
27 AlienToolkit toolkit;
28 toolkit.MakeCollection(path,mask);
29
30 //
497c5329 31 toolkit.MakeJobList("job.list","root://gsiaf.gsi.de:1094/", "", "COPY");
7332008b 32
33*/
34
35
36
37class AlienToolkit: public TObject{
38public:
39 AlienToolkit();
40 TGridCollection* MakeCollection(const char *path, char *mask);
41 void Stage();
42 void LocalCopy(const char* destination);
43 void RemoteCopy(const char* destination="root://gsiaf.gsi.de:1094/", Int_t maxfiles=20);
44 void MakeJobList(const char * outname, const char *outputPrefix, const char *action, const char *suffix);
45 static Bool_t IsDir(const char * name);
46 static Bool_t IsFile(const char * name);
47 static Bool_t ResubmitJobs();
48public:
49 TGridCollection *fCollection;
50 TObjArray fInfoArray;
51 ClassDef(AlienToolkit,1)
52};
53
54ClassImp(AlienToolkit)
55
56AlienToolkit::AlienToolkit():
57 fCollection(0)
58{
59 if (!gGrid)
60 TGrid::Connect("alien://",0,0,"t");
61}
62
63
64
65
66Bool_t AlienToolkit::IsDir(const char * name){
67 //
68 // Check if it is directory
69 //
70 void *dir = gSystem->OpenDirectory(name);
71 return (dir!=0);
72}
73
74
75Bool_t AlienToolkit::IsFile(const char *name ){
76 //
77 // Check if is it file
78 //
79 Long_t id, size, flags, modtime ;
80 Int_t ret = gSystem->GetPathInfo(name,&id, &size, &flags, &modtime);
81 if (ret==0) return kFALSE;
82 if ((flags&2)>0) return kFALSE;
83 return kTRUE;
84}
85
86
87TGridCollection* AlienToolkit::MakeCollection(const char *path, char *mask){
88 //
89 /*char *mask = "root_archive.zip";
90 char *path = "/alice/cern.ch/user/m/miranov/test2007/"
91 */
92 //
93 TGridResult* query = gGrid->Query(path,mask);
94 TGridCollection* collection=gGrid->OpenCollectionQuery(query);
95 // collection->SelectFile(mask,1,20); // select files 0-100
96 collection->LookupSUrls();
97 //collection->CheckIfOnline();
98 collection->Reset();
99 fCollection = collection;
100 //
101 collection->Reset();
102 Int_t counter=0;
103 Int_t counterm=0;
104 Int_t counterf=0;
105 TMap *filemap;
106 while ( (filemap = collection->Next())) {
107 TIterator *nextfile = filemap->MakeIterator();
108 TMap *attributes;
109 while ((attributes = (TMap *) nextfile->Next())) {
110 printf("%d\t%d\t%d\n",counter, counterm,counterf);
111 TMap * map = new TMap;
112 fInfoArray.AddLast(map);
113
114 TObjString *surl = new TObjString(collection->GetSURL(attributes->GetName()));
115 TObjString *turl = new TObjString(collection->GetTURL(attributes->GetName()));
116 TObjString *lfn = new TObjString(collection->GetLFN(attributes->GetName()));
117 map->Add(new TObjString("alienLFN"),lfn);
118 map->Add(new TObjString("alienSURL"),surl);
119 map->Add(new TObjString("alienTURL"),turl);
120
121
122 Int_t isOnline = collection->IsOnline(attributes->GetName());
123 printf("Base Name:\t%s\n",attributes->GetName());
124 printf("Size:\t%d\n",(Int_t)collection->GetSize());
125 printf("IsOnline:\t%d\n",collection->IsOnline());
126 printf("IsSelected:\t%d\n",collection->IsSelected());
127 printf("IsOnline:\t%d\n",isOnline);
128 printf("LFN Name:\t%s\n",lfn->String().Data());
129 printf("TURL Name:\t%s\n",turl->String().Data());
130 printf("SURL Name:\t%s\n",surl->String().Data());
131 counter++;
132 counterf++;
133
134 }
135 counterf=0;
136 counterm++;
137 }
138}
139
140
141void AlienToolkit::Stage(){
142 //
143 // Stage selected alien files
144 //
145 Int_t entries = fInfoArray.GetEntries();
146 ofstream aout("stage.txt");
147 for (Int_t i=0; i<entries;i++){
148 TMap &map = *((TMap*)fInfoArray.At(i));
149 TObjString *lfn = (TObjString*)map("alienLFN");
150 if (!lfn) continue;
151 printf("Staging submitfor\t%s\n",lfn->String().Data());
152 aout<<"stage "<<lfn->String().Data()<<endl;
153 }
154 aout.close();
155 gSystem->Exec("aliensh file:stage.txt");
156}
157
158void AlienToolkit::LocalCopy(const char *destination){
159 //
160 // Copy selected files to the destination directory
161 // the LFN path name translated to the directory name replacing
162 // separtor - the flat structure is created
163
164 Int_t entries = fInfoArray.GetEntries();
165 ofstream aout("stage.txt");
166 for (Int_t i=0; i<entries;i++){
167 TMap &map = *((TMap*)fInfoArray.At(i));
168 TObjString *lfn = (TObjString*)map("alienLFN");
169 if (!lfn) continue;
170 printf("Staging submitfor\t%s\n",lfn->String().Data());
171 TString dnames=lfn->String().Data();
172 dnames.ReplaceAll(".root_dir","");
173 dnames.ReplaceAll("/","_");
174 TString dname=destination;
175 dname+=dnames;
176 aout<<"cp "<<lfn->String().Data()<<" "<<dname.Data()<<endl;
177 }
178 aout.close();
179 gSystem->Exec("aliensh file:stage.txt");
180}
181
182
183void AlienToolkit::MakeJobList(const char * outname, const char *outputPrefix, const char *action, const char *suffix){
184 //
185 //
186 //
187 Int_t entries = fInfoArray.GetEntries();
188 ofstream aout(outname);
189 for (Int_t i=0; i<entries;i++){
190 TMap &map = *((TMap*)fInfoArray.At(i));
191 TObjString *lfn = (TObjString*)map("alienLFN");
192 TObjString *pfn = (TObjString*)map("alienSURL");
193 if (!lfn) continue;
194 if (!pfn) continue;
9af56b10 195 if (lfn->String().Contains(".tag.")) continue;
7332008b 196 printf("Job info\t%s\n",lfn->String().Data());
197 TString jobID=lfn->String().Data();
198 jobID.ReplaceAll("/","_");
199 //
200 //
201 TString outputDir=outputPrefix;
202 outputDir+=lfn->String().Data();
203
497c5329 204 aout<<jobID<<" "<<pfn->String().Data()<<" "<<outputDir.Data()<<" "<<action;
205 if (suffix) aout<<" "<<suffix<<"\n";
7332008b 206 aout<<endl;
207 }
208 aout.close();
209 //
210}
211
212
213void AlienToolkit::RemoteCopy(const char *destination,Int_t maxfiles){
214 //
215 // Copy selected files to the destination directory
216 // the LFN path name translated to the directory name replacing
217 // separtor - the flat structure is created
218
219 Int_t entries = fInfoArray.GetEntries();
220 ofstream *aout=0;
221 for (Int_t i=0; i<entries;i++){
222 if (i%maxfiles==0){
223 if (aout) aout->close();
224 aout = new ofstream(Form("stage_%d.sh",i/maxfiles));
225 (*aout)<<"!/bin/bash\n";
226 (*aout)<<"source ~/.balice\n";
227 }
228 TMap &map = *((TMap*)fInfoArray.At(i));
229 TObjString *pfn = (TObjString*)map("alienSURL");
230 TObjString *lfn = (TObjString*)map("alienLFN");
231 if (!pfn) continue;
232 if (!lfn) continue;
233 TString dnames=lfn->String().Data();
234 TString dname=destination;
235 dname+=dnames;
236 (*aout)<<"xrdcp -d 1 "<<pfn->String().Data()<<" "<<dname.Data()<<endl;
237 }
238 aout->close();
239 //gSystem->Exec("aliensh file:stage.txt");
240}
241
242Bool_t AlienToolkit::ResubmitJobs(){
243 //
244 // Resubmit the processes finished in error state
245 //
246 //gSystem->Exec("alien_ps | grep EE > ps.txt")
247 gSystem->Exec("alien_ps | grep EE |gawk '{print $2}' > psEE.txt");
248 gSystem->Exec("alien_ps | grep EXPIRED |gawk '{print $2}' > psEXPIRED.txt");
249 //
250 ifstream in;
251 in.open("psEXPIRED.txt");
252 TString line;
253 TString result;
254 while(in.good()) {
255 in >> line;
256 // line.ReplaceAll("-","");
257 printf("Resubmiting %s\n",line.Data());
258 Int_t pos = line.First("-");
259 printf("%s\n",Form("alien_resubmit %s",&line.Data()[pos+1]));
260 gSystem->Exec(Form("alien_resubmit %s",&line.Data()[pos+1]));
261 }
262 in.close();
263}
264
265
266
267
268
269/*
270 Problems:
2711. ???? fCollection.IsOnline("")
272
273 #12 0x04e9a03f in XrdClientAdmin::IsFileOnline (this=0x94a91b0, vs=@0xbfe07f60, vb=@0xbfe07f80) at ../../src/XrdClient/XrdClientVector.hh:55
274#13 0x0423a975 in TXNetSystem::IsOnline (this=0x9652800,
275 path=0x9661488 "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91")
276 at netx/src/TXNetSystem.cxx:476
277#14 0x04237c6e in TXNetFileStager::IsStaged (this=0x96526b8,
278 path=0x9661400 "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91")
279 at netx/src/TXNetFileStager.cxx:63
280#15 0x00551117 in TFileStager::GetStaged (this=0x96526b8, pathlist=0x9649820) at net/src/TFileStager.cxx:58
281#16 0x010e5d96 in TAlienCollection::CheckIfOnline (this=0x95da8d8, bulk=true) at alien/src/TAlienCollection.cxx:1143
282
283
2842. id xrootd on voalice04.cern.ch ?
285 "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91"
286
2873. How to check if the file exist in redirector ?
288
2894. Ho
290
291*/
292