]>
Commit | Line | Data |
---|---|---|
1 | #include "TGrid.h" | |
2 | #include "TGridCollection.h" | |
3 | #include "TSystem.h" | |
4 | #include "TString.h" | |
5 | #include "TFile.h" | |
6 | #include "TMap.h" | |
7 | #include "TGridResult.h" | |
8 | #include "TAlien.h" | |
9 | #include "TAlienResult.h" | |
10 | #include <fstream> | |
11 | ||
12 | ||
13 | ||
14 | ||
15 | ||
16 | /* | |
17 | gSystem->Load("libXrdClient.so"); | |
18 | gSystem->Load("libNetx.so"); | |
19 | .L $ALICE_ROOT/TPC/macros/testTPC/AlienToolkit.cxx+ | |
20 | //Reconstructed data example | |
21 | char *mask = "v14*"; | |
22 | char *path = "/alice/cern.ch/user/m/miranov/test2007/" | |
23 | // raw data | |
24 | char *mask = "07000015*.root"; | |
25 | char *path = "/alice/data/2007/LHC07w_TPC/" | |
26 | ||
27 | AlienToolkit toolkit; | |
28 | toolkit.MakeCollection(path,mask); | |
29 | ||
30 | // | |
31 | toolkit.MakeJobList("job.list","root://gsiaf.gsi.de:1094/", "COPY"); | |
32 | ||
33 | */ | |
34 | ||
35 | ||
36 | ||
37 | class AlienToolkit: public TObject{ | |
38 | public: | |
39 | AlienToolkit(); | |
40 | TGridCollection* MakeCollection(const char *path, char *mask); | |
41 | void Stage(); | |
42 | void LocalCopy(const char* destination); | |
43 | void RemoteCopy(const char* destination="root://gsiaf.gsi.de:1094/", Int_t maxfiles=20); | |
44 | void MakeJobList(const char * outname, const char *outputPrefix, const char *action, const char *suffix); | |
45 | static Bool_t IsDir(const char * name); | |
46 | static Bool_t IsFile(const char * name); | |
47 | static Bool_t ResubmitJobs(); | |
48 | public: | |
49 | TGridCollection *fCollection; | |
50 | TObjArray fInfoArray; | |
51 | ClassDef(AlienToolkit,1) | |
52 | }; | |
53 | ||
54 | ClassImp(AlienToolkit) | |
55 | ||
56 | AlienToolkit::AlienToolkit(): | |
57 | fCollection(0) | |
58 | { | |
59 | if (!gGrid) | |
60 | TGrid::Connect("alien://",0,0,"t"); | |
61 | } | |
62 | ||
63 | ||
64 | ||
65 | ||
66 | Bool_t AlienToolkit::IsDir(const char * name){ | |
67 | // | |
68 | // Check if it is directory | |
69 | // | |
70 | void *dir = gSystem->OpenDirectory(name); | |
71 | return (dir!=0); | |
72 | } | |
73 | ||
74 | ||
75 | Bool_t AlienToolkit::IsFile(const char *name ){ | |
76 | // | |
77 | // Check if is it file | |
78 | // | |
79 | Long_t id, size, flags, modtime ; | |
80 | Int_t ret = gSystem->GetPathInfo(name,&id, &size, &flags, &modtime); | |
81 | if (ret==0) return kFALSE; | |
82 | if ((flags&2)>0) return kFALSE; | |
83 | return kTRUE; | |
84 | } | |
85 | ||
86 | ||
87 | TGridCollection* AlienToolkit::MakeCollection(const char *path, char *mask){ | |
88 | // | |
89 | /*char *mask = "root_archive.zip"; | |
90 | char *path = "/alice/cern.ch/user/m/miranov/test2007/" | |
91 | */ | |
92 | // | |
93 | TGridResult* query = gGrid->Query(path,mask); | |
94 | TGridCollection* collection=gGrid->OpenCollectionQuery(query); | |
95 | // collection->SelectFile(mask,1,20); // select files 0-100 | |
96 | collection->LookupSUrls(); | |
97 | //collection->CheckIfOnline(); | |
98 | collection->Reset(); | |
99 | fCollection = collection; | |
100 | // | |
101 | collection->Reset(); | |
102 | Int_t counter=0; | |
103 | Int_t counterm=0; | |
104 | Int_t counterf=0; | |
105 | TMap *filemap; | |
106 | while ( (filemap = collection->Next())) { | |
107 | TIterator *nextfile = filemap->MakeIterator(); | |
108 | TMap *attributes; | |
109 | while ((attributes = (TMap *) nextfile->Next())) { | |
110 | printf("%d\t%d\t%d\n",counter, counterm,counterf); | |
111 | TMap * map = new TMap; | |
112 | fInfoArray.AddLast(map); | |
113 | ||
114 | TObjString *surl = new TObjString(collection->GetSURL(attributes->GetName())); | |
115 | TObjString *turl = new TObjString(collection->GetTURL(attributes->GetName())); | |
116 | TObjString *lfn = new TObjString(collection->GetLFN(attributes->GetName())); | |
117 | map->Add(new TObjString("alienLFN"),lfn); | |
118 | map->Add(new TObjString("alienSURL"),surl); | |
119 | map->Add(new TObjString("alienTURL"),turl); | |
120 | ||
121 | ||
122 | Int_t isOnline = collection->IsOnline(attributes->GetName()); | |
123 | printf("Base Name:\t%s\n",attributes->GetName()); | |
124 | printf("Size:\t%d\n",(Int_t)collection->GetSize()); | |
125 | printf("IsOnline:\t%d\n",collection->IsOnline()); | |
126 | printf("IsSelected:\t%d\n",collection->IsSelected()); | |
127 | printf("IsOnline:\t%d\n",isOnline); | |
128 | printf("LFN Name:\t%s\n",lfn->String().Data()); | |
129 | printf("TURL Name:\t%s\n",turl->String().Data()); | |
130 | printf("SURL Name:\t%s\n",surl->String().Data()); | |
131 | counter++; | |
132 | counterf++; | |
133 | ||
134 | } | |
135 | counterf=0; | |
136 | counterm++; | |
137 | } | |
138 | } | |
139 | ||
140 | ||
141 | void AlienToolkit::Stage(){ | |
142 | // | |
143 | // Stage selected alien files | |
144 | // | |
145 | Int_t entries = fInfoArray.GetEntries(); | |
146 | ofstream aout("stage.txt"); | |
147 | for (Int_t i=0; i<entries;i++){ | |
148 | TMap &map = *((TMap*)fInfoArray.At(i)); | |
149 | TObjString *lfn = (TObjString*)map("alienLFN"); | |
150 | if (!lfn) continue; | |
151 | printf("Staging submitfor\t%s\n",lfn->String().Data()); | |
152 | aout<<"stage "<<lfn->String().Data()<<endl; | |
153 | } | |
154 | aout.close(); | |
155 | gSystem->Exec("aliensh file:stage.txt"); | |
156 | } | |
157 | ||
158 | void AlienToolkit::LocalCopy(const char *destination){ | |
159 | // | |
160 | // Copy selected files to the destination directory | |
161 | // the LFN path name translated to the directory name replacing | |
162 | // separtor - the flat structure is created | |
163 | ||
164 | Int_t entries = fInfoArray.GetEntries(); | |
165 | ofstream aout("stage.txt"); | |
166 | for (Int_t i=0; i<entries;i++){ | |
167 | TMap &map = *((TMap*)fInfoArray.At(i)); | |
168 | TObjString *lfn = (TObjString*)map("alienLFN"); | |
169 | if (!lfn) continue; | |
170 | printf("Staging submitfor\t%s\n",lfn->String().Data()); | |
171 | TString dnames=lfn->String().Data(); | |
172 | dnames.ReplaceAll(".root_dir",""); | |
173 | dnames.ReplaceAll("/","_"); | |
174 | TString dname=destination; | |
175 | dname+=dnames; | |
176 | aout<<"cp "<<lfn->String().Data()<<" "<<dname.Data()<<endl; | |
177 | } | |
178 | aout.close(); | |
179 | gSystem->Exec("aliensh file:stage.txt"); | |
180 | } | |
181 | ||
182 | ||
183 | void AlienToolkit::MakeJobList(const char * outname, const char *outputPrefix, const char *action, const char *suffix){ | |
184 | // | |
185 | // | |
186 | // | |
187 | Int_t entries = fInfoArray.GetEntries(); | |
188 | ofstream aout(outname); | |
189 | for (Int_t i=0; i<entries;i++){ | |
190 | TMap &map = *((TMap*)fInfoArray.At(i)); | |
191 | TObjString *lfn = (TObjString*)map("alienLFN"); | |
192 | TObjString *pfn = (TObjString*)map("alienSURL"); | |
193 | if (!lfn) continue; | |
194 | if (!pfn) continue; | |
195 | printf("Job info\t%s\n",lfn->String().Data()); | |
196 | TString jobID=lfn->String().Data(); | |
197 | jobID.ReplaceAll("/","_"); | |
198 | // | |
199 | // | |
200 | TString outputDir=outputPrefix; | |
201 | outputDir+=lfn->String().Data(); | |
202 | ||
203 | aout<<jobID<<"\t"<<pfn->String().Data()<<"\t"<<outputDir.Data()<<"\t"<<action; | |
204 | if (suffix) aout<<"\t"<<suffix<<"\n"; | |
205 | aout<<endl; | |
206 | } | |
207 | aout.close(); | |
208 | // | |
209 | } | |
210 | ||
211 | ||
212 | void AlienToolkit::RemoteCopy(const char *destination,Int_t maxfiles){ | |
213 | // | |
214 | // Copy selected files to the destination directory | |
215 | // the LFN path name translated to the directory name replacing | |
216 | // separtor - the flat structure is created | |
217 | ||
218 | Int_t entries = fInfoArray.GetEntries(); | |
219 | ofstream *aout=0; | |
220 | for (Int_t i=0; i<entries;i++){ | |
221 | if (i%maxfiles==0){ | |
222 | if (aout) aout->close(); | |
223 | aout = new ofstream(Form("stage_%d.sh",i/maxfiles)); | |
224 | (*aout)<<"!/bin/bash\n"; | |
225 | (*aout)<<"source ~/.balice\n"; | |
226 | } | |
227 | TMap &map = *((TMap*)fInfoArray.At(i)); | |
228 | TObjString *pfn = (TObjString*)map("alienSURL"); | |
229 | TObjString *lfn = (TObjString*)map("alienLFN"); | |
230 | if (!pfn) continue; | |
231 | if (!lfn) continue; | |
232 | TString dnames=lfn->String().Data(); | |
233 | TString dname=destination; | |
234 | dname+=dnames; | |
235 | (*aout)<<"xrdcp -d 1 "<<pfn->String().Data()<<" "<<dname.Data()<<endl; | |
236 | } | |
237 | aout->close(); | |
238 | //gSystem->Exec("aliensh file:stage.txt"); | |
239 | } | |
240 | ||
241 | Bool_t AlienToolkit::ResubmitJobs(){ | |
242 | // | |
243 | // Resubmit the processes finished in error state | |
244 | // | |
245 | //gSystem->Exec("alien_ps | grep EE > ps.txt") | |
246 | gSystem->Exec("alien_ps | grep EE |gawk '{print $2}' > psEE.txt"); | |
247 | gSystem->Exec("alien_ps | grep EXPIRED |gawk '{print $2}' > psEXPIRED.txt"); | |
248 | // | |
249 | ifstream in; | |
250 | in.open("psEXPIRED.txt"); | |
251 | TString line; | |
252 | TString result; | |
253 | while(in.good()) { | |
254 | in >> line; | |
255 | // line.ReplaceAll("-",""); | |
256 | printf("Resubmiting %s\n",line.Data()); | |
257 | Int_t pos = line.First("-"); | |
258 | printf("%s\n",Form("alien_resubmit %s",&line.Data()[pos+1])); | |
259 | gSystem->Exec(Form("alien_resubmit %s",&line.Data()[pos+1])); | |
260 | } | |
261 | in.close(); | |
262 | } | |
263 | ||
264 | ||
265 | ||
266 | ||
267 | ||
268 | /* | |
269 | Problems: | |
270 | 1. ???? fCollection.IsOnline("") | |
271 | ||
272 | #12 0x04e9a03f in XrdClientAdmin::IsFileOnline (this=0x94a91b0, vs=@0xbfe07f60, vb=@0xbfe07f80) at ../../src/XrdClient/XrdClientVector.hh:55 | |
273 | #13 0x0423a975 in TXNetSystem::IsOnline (this=0x9652800, | |
274 | path=0x9661488 "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91") | |
275 | at netx/src/TXNetSystem.cxx:476 | |
276 | #14 0x04237c6e in TXNetFileStager::IsStaged (this=0x96526b8, | |
277 | path=0x9661400 "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91") | |
278 | at netx/src/TXNetFileStager.cxx:63 | |
279 | #15 0x00551117 in TFileStager::GetStaged (this=0x96526b8, pathlist=0x9649820) at net/src/TFileStager.cxx:58 | |
280 | #16 0x010e5d96 in TAlienCollection::CheckIfOnline (this=0x95da8d8, bulk=true) at alien/src/TAlienCollection.cxx:1143 | |
281 | ||
282 | ||
283 | 2. id xrootd on voalice04.cern.ch ? | |
284 | "root://voalice04.cern.ch:1094//castor/cern.ch/alice/2005_castor2/15/23612/9e787bec-b010-11dc-a80c-000e0c3e6d91" | |
285 | ||
286 | 3. How to check if the file exist in redirector ? | |
287 | ||
288 | 4. Ho | |
289 | ||
290 | */ | |
291 |