]> git.uio.no Git - u/mrichter/AliRoot.git/blobdiff - SHUTTLE/AliShuttle.cxx
Moved getters and setters of Shuttle's main OCDB/Reference, local
[u/mrichter/AliRoot.git] / SHUTTLE / AliShuttle.cxx
index afa94a777f5bb3594ee883093f9f418214a2ea9a..bb3f99097061a0bb70b5bd5e27208fb70717c552 100644 (file)
 
 /*
 $Log$
+Revision 1.27  2007/01/30 17:52:42  jgrosseo
+adding monalisa monitoring
+
+Revision 1.26  2007/01/23 19:20:03  acolla
+Removed old ldif files, added TOF, MCH ldif files. Added some options in
+AliShuttleConfig::Print. Added in Ali Shuttle: SetShuttleTempDir and
+SetShuttleLogDir
+
+Revision 1.25  2007/01/15 19:13:52  acolla
+Moved some AliInfo to AliDebug in SendMail function
+
+Revision 1.21  2006/12/07 08:51:26  jgrosseo
+update (alberto):
+table, db names in ldap configuration
+added GRP preprocessor
+DCS data can also be retrieved by data point
+
+Revision 1.20  2006/11/16 16:16:48  jgrosseo
+introducing strict run ordering flag
+removed giving preprocessor name to preprocessor, they have to know their name themselves ;-)
+
 Revision 1.19  2006/11/06 14:23:04  jgrosseo
 major update (Alberto)
 o) reading of run parameters from the logbook
@@ -152,6 +173,8 @@ some docs added
 #include <TSQLRow.h>
 #include <TMutex.h>
 
+#include <TMonaLisaWriter.h>
+
 #include <fstream>
 
 #include <sys/types.h>
@@ -159,15 +182,7 @@ some docs added
 
 ClassImp(AliShuttle)
 
-TString AliShuttle::fgkMainCDB("alien://folder=ShuttleCDB");
-TString AliShuttle::fgkLocalCDB("local://LocalShuttleCDB");
-TString AliShuttle::fgkMainRefStorage("alien://folder=ShuttleReference");
-TString AliShuttle::fgkLocalRefStorage("local://LocalReferenceStorage");
-
-Bool_t AliShuttle::fgkProcessDCS(kTRUE); 
-
-const char* AliShuttle::fgkShuttleTempDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/temp");
-const char* AliShuttle::fgkShuttleLogDir = gSystem->ExpandPathName("$ALICE_ROOT/SHUTTLE/log");
+Bool_t AliShuttle::fgkProcessDCS(kTRUE);
 
 //______________________________________________________________________________________________
 AliShuttle::AliShuttle(const AliShuttleConfig* config,
@@ -181,7 +196,8 @@ fStatusEntry(0),
 fGridError(kFALSE),
 fMonitoringMutex(0),
 fLastActionTime(0),
-fLastAction()
+fLastAction(),
+fMonaLisa(0)
 {
        //
        // config: AliShuttleConfig used
@@ -193,7 +209,7 @@ fLastAction()
        for(int iSys=0;iSys<4;iSys++) {
                fServer[iSys]=0;
                if (iSys < 3)
-                       fFESlist[iSys].SetOwner(kTRUE);
+                       fFXSlist[iSys].SetOwner(kTRUE);
        }
        fPreprocessorMap.SetOwner(kTRUE);
 
@@ -314,7 +330,7 @@ UInt_t AliShuttle::WriteToCDB(const char* mainUri, const char* localUri,
 
        if(! dynamic_cast<TObjString*> (metaData->GetProperty("RunUsed(TObjString)"))){
                TObjString runUsed = Form("%d", GetCurrentRun());
-               metaData->SetProperty("RunUsed(TObjString)",&runUsed);
+               metaData->SetProperty("RunUsed(TObjString)", runUsed.Clone());
        }
 
        UInt_t result = 0;
@@ -360,7 +376,7 @@ AliShuttleStatus* AliShuttle::ReadShuttleStatus()
                fStatusEntry = 0;
        }
 
-       fStatusEntry = AliCDBManager::Instance()->GetStorage(AliShuttle::GetLocalCDB())
+       fStatusEntry = AliCDBManager::Instance()->GetStorage(GetLocalCDB())
                ->Get(Form("/SHUTTLE/STATUS/%s", fCurrentDetector.Data()), GetCurrentRun());
 
        if (!fStatusEntry) return 0;
@@ -398,6 +414,8 @@ Bool_t AliShuttle::WriteShuttleStatus(AliShuttleStatus* status)
                AliError(Form("WriteShuttleStatus for %s, run %d failed", fCurrentDetector.Data(), run));
                return kFALSE;
        }
+       
+       SendMLInfo();
 
        return kTRUE;
 }
@@ -419,9 +437,9 @@ void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t
                return;
        }
 
-       TString actionStr = Form("UpdateShuttleStatus - %s: Changing state from %s to %s", 
+       TString actionStr = Form("UpdateShuttleStatus - %s: Changing state from %s to %s",
                                fCurrentDetector.Data(),
-                               status->GetStatusName(), 
+                               status->GetStatusName(),
                                status->GetStatusName(newStatus));
        Log("SHUTTLE", actionStr);
        SetLastAction(actionStr);
@@ -430,7 +448,34 @@ void AliShuttle::UpdateShuttleStatus(AliShuttleStatus::Status newStatus, Bool_t
        if (increaseCount) status->IncreaseCount();
 
        AliCDBManager::Instance()->GetStorage(fgkLocalCDB)->Put(fStatusEntry);
+
+       SendMLInfo();
+}
+
+//______________________________________________________________________________________________
+void AliShuttle::SendMLInfo()
+{
+       //
+       // sends ML information about the current status of the current detector being processed
+       //
+       
+       AliShuttleStatus* status = dynamic_cast<AliShuttleStatus*> (fStatusEntry->GetObject());
+       
+       if (!status){
+               AliError("UNEXPECTED: status could not be read from current CDB entry");
+               return;
+       }
+       
+       TMonaLisaText  mlStatus(Form("%s_status", fCurrentDetector.Data()), status->GetStatusName());
+       TMonaLisaValue mlRetryCount(Form("%s_count", fCurrentDetector.Data()), status->GetCount());
+
+       TList mlList;
+       mlList.Add(&mlStatus);
+       mlList.Add(&mlRetryCount);
+
+       fMonaLisa->SendParameters(&mlList);
 }
+
 //______________________________________________________________________________________________
 Bool_t AliShuttle::ContinueProcessing()
 {
@@ -438,11 +483,21 @@ Bool_t AliShuttle::ContinueProcessing()
 // checks if the processing should be continued
 // if yes it returns kTRUE and updates the AliShuttleStatus with nextStatus
 
+       if (!fConfig->HostProcessDetector(fCurrentDetector)) return kFALSE;
+
+       AliPreprocessor* aPreprocessor =
+               dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
+       if (!aPreprocessor)
+       {
+               AliInfo(Form("%s: no preprocessor registered", fCurrentDetector.Data()));
+               return kFALSE;
+       }
+
        AliShuttleLogbookEntry::Status entryStatus =
                fLogbookEntry->GetDetectorStatus(fCurrentDetector);
 
        if(entryStatus != AliShuttleLogbookEntry::kUnprocessed) {
-               Log("SHUTTLE", Form("ContinueProcessing - %s is %s",
+               AliInfo(Form("ContinueProcessing - %s is %s",
                                fCurrentDetector.Data(),
                                fLogbookEntry->GetDetectorStatusName(entryStatus)));
                return kFALSE;
@@ -491,29 +546,37 @@ Bool_t AliShuttle::ContinueProcessing()
                        Log("SHUTTLE",
                                Form("ContinueProcessing - %s: Grid storage failed again",
                                        fCurrentDetector.Data()));
+                       // trigger ML information manually because we do not had a status change
+                       SendMLInfo();
                }
                return kFALSE;
        }
 
        // if we get here, there is a restart
+       Bool_t cont = kFALSE;
 
        // abort conditions
        if (status->GetCount() >= fConfig->GetMaxRetries()) {
-               Log("SHUTTLE",
-                       Form("ContinueProcessing - %s failed %d times in status %s - Updating Shuttle Logbook",
-                               fCurrentDetector.Data(),
+               Log("SHUTTLE", Form("ContinueProcessing - %s failed %d times in status %s - "
+                               "Updating Shuttle Logbook", fCurrentDetector.Data(),
                                status->GetCount(), status->GetStatusName()));
                UpdateShuttleLogbook(fCurrentDetector.Data(), "FAILED");
-               return kFALSE;
+               UpdateShuttleStatus(AliShuttleStatus::kFailed);
+       } else {
+               Log("SHUTTLE", Form("ContinueProcessing - %s: restarting. "
+                               "Aborted before with %s. Retry number %d.", fCurrentDetector.Data(),
+                               status->GetStatusName(), status->GetCount()));
+               UpdateShuttleStatus(AliShuttleStatus::kStarted, kTRUE);
+               cont = kTRUE;
        }
 
-       Log("SHUTTLE", Form("ContinueProcessing - %s: restarting. Aborted before with %s. Retry number %d.",
-                       fCurrentDetector.Data(),
-                       status->GetStatusName(), status->GetCount()));
-
-       UpdateShuttleStatus(AliShuttleStatus::kStarted, kTRUE);
+       // Send mail to detector expert!
+       AliInfo(Form("Sending mail to %s expert...", fCurrentDetector.Data()));
+       if (!SendMail())
+               Log("SHUTTLE", Form("ContinueProcessing - Could not send mail to %s expert",
+                               fCurrentDetector.Data()));
 
-       return kTRUE;
+       return cont;
 }
 
 //______________________________________________________________________________________________
@@ -530,22 +593,35 @@ Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
 
        fLogbookEntry = entry;
 
-       if(fLogbookEntry->IsDone()){
+       if (fLogbookEntry->IsDone())
+       {
                Log("SHUTTLE","Process - Shuttle is already DONE. Updating logbook");
                UpdateShuttleLogbook("shuttle_done");
                fLogbookEntry = 0;
                return kTRUE;
        }
 
+       // create ML instance that monitors this run
+       fMonaLisa = new TMonaLisaWriter(Form("%d", GetCurrentRun()), "SHUTTLE", "aliendb1.cern.ch");
+       // disable monitoring of other parameters that come e.g. from TFile
+       gMonitoringWriter = 0;
 
        AliInfo(Form("\n\n \t\t\t^*^*^*^*^*^*^*^*^*^*^*^* run %d: START ^*^*^*^*^*^*^*^*^*^*^*^* \n",
                                        GetCurrentRun()));
 
+
+       // Send the information to ML
+       TMonaLisaText  mlStatus("SHUTTLE_status", "Processing");
+
+       TList mlList;
+       mlList.Add(&mlStatus);
+
+       fMonaLisa->SendParameters(&mlList);
+                       
        fLogbookEntry->Print("all");
 
        // Initialization
        Bool_t hasError = kFALSE;
-       for(Int_t iSys=0;iSys<3;iSys++) fFESCalled[iSys]=kFALSE;
 
        AliCDBStorage *mainCDBSto = AliCDBManager::Instance()->GetStorage(fgkMainCDB);
        if(mainCDBSto) mainCDBSto->QueryCDB(GetCurrentRun());
@@ -560,22 +636,14 @@ Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
        {
                fCurrentDetector = aDetector->String();
 
-               if (!fConfig->HostProcessDetector(fCurrentDetector)) continue;
-
-               AliPreprocessor* aPreprocessor =
-                       dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
-               if (!aPreprocessor)
-               {
-                       Log("SHUTTLE",Form("Process - %s: no preprocessor registered. Skipping",
-                                                       fCurrentDetector.Data()));
-                       continue;
-               }
-
                if (ContinueProcessing() == kFALSE) continue;
 
                AliInfo(Form("\n\n \t\t\t****** run %d - %s: START  ******",
                                                GetCurrentRun(), aDetector->GetName()));
 
+               for(Int_t iSys=0;iSys<3;iSys++) fFXSCalled[iSys]=kFALSE;
+
+               Log(fCurrentDetector.Data(), "Starting processing");
 
                Int_t pid = fork();
 
@@ -656,23 +724,13 @@ Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
 
                        if (result > 0)
                        {
-                               // Process successful: Update time_processed field in FES logbooks!
-                               if (fFESCalled[kDAQ])
-                               {
-                                       if (UpdateDAQTable() == kFALSE)
-                                       returnCode = 1;
-                                       fFESlist[kDAQ].Clear();
-                               }
-                               //if(fFESCalled[kDCS]) {
-                               //  if (UpdateDCSTable(aDetector->GetName()) == kFALSE)
-                               //    returnCode = 1;
-                               //  fFESlist[kDCS].Clear();
-                               //}
-                               //if(fFESCalled[kHLT]) {
-                               //  if (UpdateHLTTable(aDetector->GetName()) == kFALSE)
-                               //    returnCode = 1;
-                               //      fFESlist[kHLT].Clear();
-                               //}
+                               // Process successful: Update time_processed field in FXS logbooks!
+                               if (UpdateTable() == kFALSE) returnCode = 1;
+                       }
+
+                       for (UInt_t iSys=0; iSys<3; iSys++)
+                       {
+                               if (fFXSCalled[iSys]) fFXSlist[iSys].Clear();
                        }
 
                        AliInfo(Form("Client process of %d - %s is exiting now with %d.",
@@ -691,34 +749,41 @@ Bool_t AliShuttle::Process(AliShuttleLogbookEntry* entry)
        //check if shuttle is done for this run, if so update logbook
        TObjArray checkEntryArray;
        checkEntryArray.SetOwner(1);
-       TString whereClause = Form("where run=%d",GetCurrentRun());
-       if (QueryShuttleLogbook(whereClause.Data(), checkEntryArray)) {
+       TString whereClause = Form("where run=%d", GetCurrentRun());
+       if (!QueryShuttleLogbook(whereClause.Data(), checkEntryArray) || checkEntryArray.GetEntries() == 0) {
+               Log("SHUTTLE", Form("Process - Warning: Cannot check status of run %d on Shuttle logbook!",
+                                               GetCurrentRun()));
+               return hasError == kFALSE;
+       }
 
-               AliShuttleLogbookEntry* checkEntry = dynamic_cast<AliShuttleLogbookEntry*>
-                                                       (checkEntryArray.At(0));
+       AliShuttleLogbookEntry* checkEntry = dynamic_cast<AliShuttleLogbookEntry*>
+                                               (checkEntryArray.At(0));
 
-               if (checkEntry)
+       if (checkEntry)
+       {
+               if (checkEntry->IsDone())
                {
-                       if (checkEntry->IsDone())
-                       {
-                               Log("SHUTTLE","Process - Shuttle is DONE. Updating logbook");
-                               UpdateShuttleLogbook("shuttle_done");
-                       }
-                       else
+                       Log("SHUTTLE","Process - Shuttle is DONE. Updating logbook");
+                       UpdateShuttleLogbook("shuttle_done");
+               }
+               else
+               {
+                       for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
                        {
-                               for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
+                               if (checkEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
                                {
-                                       if (checkEntry->GetDetectorStatus(iDet) == AliShuttleLogbookEntry::kUnprocessed)
-                                       {
-                                               AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
-                                                               checkEntry->GetRun(), GetDetName(iDet)));
-                                               fFirstUnprocessed[iDet] = kFALSE;
-                                       }
+                                       AliDebug(2, Form("Run %d: setting %s as \"not first time unprocessed\"",
+                                                       checkEntry->GetRun(), GetDetName(iDet)));
+                                       fFirstUnprocessed[iDet] = kFALSE;
                                }
                        }
                }
        }
 
+       // remove ML instance
+       delete fMonaLisa;
+       fMonaLisa = 0;
+
        fLogbookEntry = 0;
 
        return hasError == kFALSE;
@@ -735,37 +800,75 @@ UInt_t AliShuttle::ProcessCurrentDetector()
 
        UpdateShuttleStatus(AliShuttleStatus::kDCSStarted);
 
-       TString host(fConfig->GetDCSHost(fCurrentDetector));
-       Int_t port = fConfig->GetDCSPort(fCurrentDetector);
-
-       TIter iter(fConfig->GetDCSAliases(fCurrentDetector));
-       TObjString* anAlias;
-       TMap aliasMap;
-       aliasMap.SetOwner(1);
+       TMap dcsMap;
+       dcsMap.SetOwner(1);
 
        Bool_t aDCSError = kFALSE;
        fGridError = kFALSE;
 
-       while ((anAlias = (TObjString*) iter.Next())) {
-               TObjArray *valueSet = new TObjArray();
-               valueSet->SetOwner(1);
-               // TODO Test only... I've added a flag that allows to
-               // exclude DCS archive DB query
-               if(fgkProcessDCS){
-                       AliInfo("Querying DCS archive DB data...");
-                       aDCSError = (GetValueSet(host, port, anAlias->String(), valueSet) == 0);
-               } else {
-                       AliInfo(Form("Skipping DCS processing. Port = %d",port));
-                       aDCSError = kFALSE;
+       // TODO Test only... I've added a flag that allows to
+       // exclude DCS archive DB query
+       if (!fgkProcessDCS)
+       {
+               AliInfo("Skipping DCS processing!");
+               aDCSError = kFALSE;
+       } else {
+               TString host(fConfig->GetDCSHost(fCurrentDetector));
+               Int_t port = fConfig->GetDCSPort(fCurrentDetector);
+
+               // Retrieval of Aliases
+               TObjString* anAlias = 0;
+               Int_t iAlias = 1;
+               Int_t nTotAliases= ((TMap*)fConfig->GetDCSAliases(fCurrentDetector))->GetEntries();
+               TIter iterAliases(fConfig->GetDCSAliases(fCurrentDetector));
+               while ((anAlias = (TObjString*) iterAliases.Next()))
+               {
+                       TObjArray *valueSet = new TObjArray();
+                       valueSet->SetOwner(1);
+
+                       if (((iAlias-1) % 500) == 0 || iAlias == nTotAliases)
+                               AliInfo(Form("Querying DCS archive: alias %s (%d of %d)",
+                                               anAlias->GetName(), iAlias++, nTotAliases));
+                       aDCSError = (GetValueSet(host, port, anAlias->String(), valueSet, kAlias) == 0);
+
+                       if(!aDCSError)
+                       {
+                               dcsMap.Add(anAlias->Clone(), valueSet);
+                       } else {
+                               Log(fCurrentDetector,
+                                       Form("ProcessCurrentDetector - Error while retrieving alias %s",
+                                               anAlias->GetName()));
+                               UpdateShuttleStatus(AliShuttleStatus::kDCSError);
+                               dcsMap.DeleteAll();
+                               return 0;
+                       }
                }
-               if(!aDCSError) {
-                       aliasMap.Add(anAlias->Clone(), valueSet);
-               }else{
-                       Log(fCurrentDetector, Form("ProcessCurrentDetector - Error while retrieving alias %s",
-                                       anAlias->GetName()));
-                       UpdateShuttleStatus(AliShuttleStatus::kDCSError, kTRUE);
-                       aliasMap.DeleteAll();
-                       return 0;
+
+               // Retrieval of Data Points
+               TObjString* aDP = 0;
+               Int_t iDP = 0;
+               Int_t nTotDPs= ((TMap*)fConfig->GetDCSDataPoints(fCurrentDetector))->GetEntries();
+               TIter iterDP(fConfig->GetDCSDataPoints(fCurrentDetector));
+               while ((aDP = (TObjString*) iterDP.Next()))
+               {
+                       TObjArray *valueSet = new TObjArray();
+                       valueSet->SetOwner(1);
+                       if (((iDP-1) % 500) == 0 || iDP == nTotDPs)
+                               AliInfo(Form("Querying DCS archive: DP %s (%d of %d)",
+                                               aDP->GetName(), iDP++, nTotDPs));
+                       aDCSError = (GetValueSet(host, port, aDP->String(), valueSet, kDP) == 0);
+
+                       if(!aDCSError)
+                       {
+                               dcsMap.Add(aDP->Clone(), valueSet);
+                       } else {
+                               Log(fCurrentDetector,
+                                       Form("ProcessCurrentDetector - Error while retrieving data point %s",
+                                               aDP->GetName()));
+                               UpdateShuttleStatus(AliShuttleStatus::kDCSError);
+                               dcsMap.DeleteAll();
+                               return 0;
+                       }
                }
        }
 
@@ -776,7 +879,7 @@ UInt_t AliShuttle::ProcessCurrentDetector()
                dynamic_cast<AliPreprocessor*> (fPreprocessorMap.GetValue(fCurrentDetector));
 
        aPreprocessor->Initialize(GetCurrentRun(), GetCurrentStartTime(), GetCurrentEndTime());
-       UInt_t aPPResult = aPreprocessor->Process(&aliasMap);
+       UInt_t aPPResult = aPreprocessor->Process(&dcsMap);
 
        UInt_t returnValue = 0;
        if (aPPResult == 0) { // Preprocessor error
@@ -793,7 +896,7 @@ UInt_t AliShuttle::ProcessCurrentDetector()
                returnValue = 2;
        }
 
-       aliasMap.DeleteAll();
+       dcsMap.DeleteAll();
 
        return returnValue;
 }
@@ -805,6 +908,8 @@ Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
 // Query DAQ's Shuttle logbook and fills detector status object.
 // Call QueryRunParameters to query DAQ logbook for run parameters.
 
+       entries.SetOwner(1);
+
        // check connection, in case connect
        if(!Connect(3)) return kFALSE;
 
@@ -817,28 +922,28 @@ Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
                return kFALSE;
        }
 
+       AliDebug(2,Form("Query = %s", sqlQuery.Data()));
+
        if(aResult->GetRowCount() == 0) {
-               if(sqlQuery.Contains("where shuttle_done=0")){
-                       Log("SHUTTLE", "QueryShuttleLogbook - All runs in Shuttle Logbook are already DONE");
+//             if(sqlQuery.EndsWith("where shuttle_done=0 order by run")){
+//                     Log("SHUTTLE", "QueryShuttleLogbook - All runs in Shuttle Logbook are already DONE");
+//                     delete aResult;
+//                     return kTRUE;
+//             } else {
+                       AliInfo("No entries in Shuttle Logbook match request");
                        delete aResult;
                        return kTRUE;
-               } else {
-                       AliError("No entries in Shuttle Logbook match request");
-                       delete aResult;
-                       return kFALSE;
-               }
+//             }
        }
 
        // TODO Check field count!
-       const UInt_t nCols = 24;
+       const UInt_t nCols = 22;
        if (aResult->GetFieldCount() != (Int_t) nCols) {
                AliError("Invalid SQL result field number!");
                delete aResult;
                return kFALSE;
        }
 
-       entries.SetOwner(1);
-
        TSQLRow* aRow;
        while ((aRow = aResult->Next())) {
                TString runString(aRow->GetField(0), aRow->GetFieldLength(0));
@@ -856,9 +961,9 @@ Bool_t AliShuttle::QueryShuttleLogbook(const char* whereClause,
                delete aRow;
        }
 
-       if(sqlQuery.Contains("where shuttle_done=0"))
-               Log("SHUTTLE", Form("QueryShuttleLogbook - Found %d unprocessed runs in Shuttle Logbook",
-                                                       entries.GetEntriesFast()));
+//     if(sqlQuery.EndsWith("where shuttle_done=0 order by run"))
+//             Log("SHUTTLE", Form("QueryShuttleLogbook - Found %d unprocessed runs in Shuttle Logbook",
+//                                                     entries.GetEntriesFast()));
        delete aResult;
        return kTRUE;
 }
@@ -875,7 +980,7 @@ AliShuttleLogbookEntry* AliShuttle::QueryRunParameters(Int_t run)
                return 0;
 
        TString sqlQuery;
-       sqlQuery.Form("select * from logbook where run=%d", run);
+       sqlQuery.Form("select * from %s where run=%d", fConfig->GetDAQlbTable(), run);
 
        TSQLResult* aResult = fServer[3]->Query(sqlQuery);
        if (!aResult) {
@@ -1004,7 +1109,8 @@ Bool_t AliShuttle::TryToStoreAgain(TString& gridURI)
                                if (!fFirstUnprocessed[GetDetPos(fCurrentDetector)])
                                {
                                        Log(fCurrentDetector.Data(),
-                                               ("TryToStoreAgain - This object has validity infinite but there are previous unprocessed runs!"));
+                                               ("TryToStoreAgain - This object has validity infinite but "
+                                                "there are previous unprocessed runs!"));
                                        continue;
                                } else {
                                        break;
@@ -1057,27 +1163,41 @@ Bool_t AliShuttle::TryToStoreAgain(TString& gridURI)
 }
 
 //______________________________________________________________________________________________
-Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* alias,
-                               TObjArray* valueSet)
+Bool_t AliShuttle::GetValueSet(const char* host, Int_t port, const char* entry,
+                               TObjArray* valueSet, DCSType type)
 {
-// Retrieve all "alias" data points from the DCS server
+// Retrieve all "entry" data points from the DCS server
 // host, port: TSocket connection parameters
-// alias: name of the alias
+// entry: name of the alias or data point
 // valueSet: array of retrieved AliDCSValue's
+// type: kAlias or kDP
 
        AliDCSClient client(host, port, fTimeout, fRetries);
-       if (!client.IsConnected()) {
+       if (!client.IsConnected())
+       {
                return kFALSE;
        }
 
-       Int_t result = client.GetAliasValues(alias,
-               GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
+       Int_t result=0;
+
+       if (type == kAlias)
+       {
+               result = client.GetAliasValues(entry,
+                       GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
+       } else
+       if (type == kDP)
+       {
+               result = client.GetDPValues(entry,
+                       GetCurrentStartTime(), GetCurrentEndTime(), valueSet);
+       }
 
-       if (result < 0) {
+       if (result < 0)
+       {
                Log(fCurrentDetector.Data(), Form("GetValueSet - Can't get '%s'! Reason: %s",
-                       alias, AliDCSClient::GetErrorString(result)));
+                       entry, AliDCSClient::GetErrorString(result)));
 
-               if (result == AliDCSClient::fgkServerError) {
+               if (result == AliDCSClient::fgkServerError)
+               {
                        Log(fCurrentDetector.Data(), Form("GetValueSet - Server error: %s",
                                client.GetServerError().Data()));
                }
@@ -1093,154 +1213,84 @@ const char* AliShuttle::GetFile(Int_t system, const char* detector,
                const char* id, const char* source)
 {
 // Get calibration file from file exchange servers
-// calls specific getter according to system index (kDAQ, kDCS, kHLT)
-
-       switch(system){
-               case kDAQ:
-                       return GetDAQFileName(detector, id, source);
-                       break;
-               case kDCS:
-                       return GetDCSFileName(detector, id, source);
-                       break;
-               case kHLT:
-                       return GetHLTFileName(detector, id, source);
-                       break;
-               default:
-                       AliError(Form("No valid system index: %d",system));
-       }
-
-       return 0;
-}
-
-//______________________________________________________________________________________________
-TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
-{
-// Get sources producing the condition file Id from file exchange servers
-// calls specific getter according to system index (kDAQ, kDCS, kHLT)
-
-       switch(system){
-               case kDAQ:
-                       return GetDAQFileSources(detector, id);
-                       break;
-               case kDCS:
-                       return GetDCSFileSources(detector, id);
-                       break;
-               case kHLT:
-                       return GetHLTFileSources(detector, id);
-                       break;
-               default:
-                       AliError(Form("No valid system index: %d",system));
-       }
-
-       return NULL;
-}
-
-//______________________________________________________________________________________________
-Bool_t AliShuttle::Connect(Int_t system)
-{
-// Connect to MySQL Server of the system's FES logbook
-// DAQ Logbook, Shuttle Logbook and DAQ FES Logbook are on the same host
-
-       // check connection: if already connected return
-       if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
-
-       TString lbHost, lbUser, lbPass;
-
-       if (system < 3) // FES logbook servers
-       {
-               lbHost = Form("mysql://%s", fConfig->GetFESlbHost(system));
-               lbUser = fConfig->GetFESlbUser(system);
-               lbPass = fConfig->GetFESlbPass(system);
-       } else { // Run & Shuttle logbook servers
-       // TODO Will the Shuttle logbook server be the same as the Run logbook server ???
-               lbHost = Form("mysql://%s", fConfig->GetDAQlbHost());
-               lbUser = fConfig->GetDAQlbUser();
-               lbPass = fConfig->GetDAQlbPass();
-       }
-
-       fServer[system] = TSQLServer::Connect(lbHost.Data(), lbUser.Data(), lbPass.Data());
-       if (!fServer[system] || !fServer[system]->IsConnected()) {
-               if(system < 3)
-               {
-               AliError(Form("Can't establish connection to FES logbook for %s",
-                                       AliShuttleInterface::GetSystemName(system)));
-               } else {
-               AliError("Can't establish connection to Run logbook.");
-               }
-               if(fServer[system]) delete fServer[system];
-               return kFALSE;
-       }
-
-       // Get tables
-       // TODO in the configuration should the table name be there too?
-       TSQLResult* aResult=0;
-       switch(system){
-               case kDAQ:
-                       aResult = fServer[kDAQ]->GetTables("REFSYSLOG");
-                       break;
-               case kDCS:
-                       //aResult = fServer[kDCS]->GetTables("REFSYSLOG");
-                       break;
-               case kHLT:
-                       //aResult = fServer[kHLT]->GetTables("REFSYSLOG");
-                       break;
-               default:
-                       aResult = fServer[3]->GetTables("REFSYSLOG");
-                       break;
-       }
-
-       delete aResult;
-       return kTRUE;
-}
-
-//______________________________________________________________________________________________
-const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, const char* source)
-{
-// Retrieves a file from the DAQ FES.
-// First queris the DAQ logbook_fs for the DAQ file name, using the run, detector, id and source info
-// then calls RetrieveDAQFile(DAQfilename) for actual copy to local disk
+// First queris the FXS database for the file name, using the run, detector, id and source info
+// then calls RetrieveFile(filename) for actual copy to local disk
 // run: current run being processed (given by Logbook entry fLogbookEntry)
 // detector: the Preprocessor name
 // id: provided as a parameter by the Preprocessor
 // source: provided by the Preprocessor through GetFileSources function
 
        // check connection, in case connect
-       if (!Connect(kDAQ))
+       if (!Connect(system))
        {
-               Log(detector, "GetDAQFileName - Couldn't connect to DAQ Logbook");
+               Log(detector, Form("GetFile - Couldn't connect to %s FXS database", GetSystemName(system)));
                return 0;
        }
 
        // Query preparation
-       TString sqlQueryStart = "select filePath from logbook_fs where";
-       TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
+       TString sqlQueryStart;
+       TString whereClause;
+       TString sourceName(source);
+       Int_t nFields = 0;
+       if (system == kDAQ)
+       {
+               sqlQueryStart = Form("select filePath,size from %s where", fConfig->GetFXSdbTable(system));
+               whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\"",
+                               GetCurrentRun(), detector, id, source);
+               nFields = 2;
+
+       }
+       else if (system == kDCS)
+       {
+               sqlQueryStart = Form("select filePath,size from %s where", fConfig->GetFXSdbTable(system));
+               whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
+                               GetCurrentRun(), detector, id);
+               nFields = 2;
+               sourceName="none";
+       }
+       else if (system == kHLT)
+       {
+               sqlQueryStart = Form("select filePath,fileSize,fileChecksum from %s where",
+                                                                               fConfig->GetFXSdbTable(system));
+               whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\" and DDLnumbers=\"%s\"",
                                GetCurrentRun(), detector, id, source);
+               nFields = 3;
+       }
+
        TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
 
        AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
 
        // Query execution
        TSQLResult* aResult = 0;
-       aResult = dynamic_cast<TSQLResult*> (fServer[kDAQ]->Query(sqlQuery));
+       aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
        if (!aResult) {
-               Log(detector, Form("GetDAQFileName - Can't execute SQL query for: id = %s, source = %s",
-                               id, source));
+               Log(detector, Form("GetFileName - Can't execute SQL query to %s database for: id = %s, source = %s",
+                               GetSystemName(system), id, sourceName.Data()));
                return 0;
        }
 
        if(aResult->GetRowCount() == 0)
        {
                Log(detector,
-                       Form("GetDAQFileName - No entry in FES table for: id = %s, source = %s",
-                               id, source));
+                       Form("GetFileName - No entry in %s FXS db for: id = %s, source = %s",
+                               GetSystemName(system), id, sourceName.Data()));
                delete aResult;
                return 0;
        }
 
        if (aResult->GetRowCount() > 1) {
                Log(detector,
-                       Form("GetDAQFileName - More than one entry in FES table for: id = %s, source = %s",
-                               id, source));
+                       Form("GetFileName - More than one entry in %s FXS db for: id = %s, source = %s",
+                               GetSystemName(system), id, sourceName.Data()));
+               delete aResult;
+               return 0;
+       }
+
+       if (aResult->GetFieldCount() != nFields) {
+               Log(detector,
+                       Form("GetFileName - Wrong field count in %s FXS db for: id = %s, source = %s",
+                               GetSystemName(system), id, sourceName.Data()));
                delete aResult;
                return 0;
        }
@@ -1248,13 +1298,16 @@ const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, con
        TSQLRow* aRow = dynamic_cast<TSQLRow*> (aResult->Next());
 
        if (!aRow){
-               Log(detector, Form("GetDAQFileName - Empty set result from query: id = %s, source = %s",
-                               id, source));
+               Log(detector, Form("GetFileName - Empty set result in %s FXS db from query: id = %s, source = %s",
+                               GetSystemName(system), id, sourceName.Data()));
                delete aResult;
                return 0;
        }
 
        TString filePath(aRow->GetField(0), aRow->GetFieldLength(0));
+       TString fileSize(aRow->GetField(1), aRow->GetFieldLength(1));
+       TString fileMd5Sum;
+       if(system == kHLT) fileMd5Sum = aRow->GetField(2);
 
        delete aResult;
        delete aRow;
@@ -1262,40 +1315,75 @@ const char* AliShuttle::GetDAQFileName(const char* detector, const char* id, con
        AliDebug(2, Form("filePath = %s",filePath.Data()));
 
        // retrieved file is renamed to make it unique
-       TString localFileName = Form("%s_%d_%s_%s.shuttle",
-                                       detector, GetCurrentRun(), id, source);
+       TString localFileName = Form("%s_%s_%d_%s_%s.shuttle",
+                                       GetSystemName(system), detector, GetCurrentRun(), id, sourceName.Data());
 
-       // file retrieval from DAQ FES
-       Bool_t result = RetrieveDAQFile(filePath.Data(), localFileName.Data());
-       if(!result) {
-               Log(detector, Form("GetDAQFileName - Copy of file %s from DAQ FES failed", filePath.Data()));
-               return 0;
-       } else {
-               AliInfo(Form("File %s copied from DAQ FES into %s/%s",
-                       filePath.Data(), fgkShuttleTempDir, localFileName.Data()));
+
+       // file retrieval from FXS
+       UInt_t nRetries = 0;
+       UInt_t maxRetries = 3;
+       Bool_t result = kFALSE;
+
+       // copy!! if successful TSystem::Exec returns 0
+       while(nRetries++ < maxRetries) {
+               AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
+               result = RetrieveFile(system, filePath.Data(), localFileName.Data());
+               if(!result)
+               {
+                       Log(detector, Form("GetFileName - Copy of file %s from %s FXS failed",
+                                       filePath.Data(), GetSystemName(system)));
+                       continue;
+               } else {
+                       AliInfo(Form("File %s copied from %s FXS into %s/%s",
+                                               filePath.Data(), GetSystemName(system),
+                                               GetShuttleTempDir(), localFileName.Data()));
+               }
+
+               if (system == kHLT)
+               {
+                       // compare md5sum of local file with the one stored in the FXS DB
+                       Int_t md5Comp = gSystem->Exec(Form("md5sum %s/%s |grep %s 2>&1 > /dev/null",
+                                               GetShuttleTempDir(), localFileName.Data(), fileMd5Sum.Data()));
+
+                       if (md5Comp != 0)
+                       {
+                               Log(detector, Form("GetFileName - md5sum of file %s does not match with local copy!",
+                                                       filePath.Data()));
+                               result = kFALSE;
+                               continue;
+                       }
+               }
+               if (result) break;
        }
 
+       if(!result) return 0;
+
+       fFXSCalled[system]=kTRUE;
+       TObjString *fileParams = new TObjString(Form("%s#!?!#%s", id, sourceName.Data()));
+       fFXSlist[system].Add(fileParams);
+
+       static TString fullLocalFileName;
+       fullLocalFileName = TString::Format("%s/%s", GetShuttleTempDir(), localFileName.Data());
 
-       fFESCalled[kDAQ]=kTRUE;
-       TObjString *fileParams = new TObjString(Form("%s_!?!_%s", id, source));
-       fFESlist[kDAQ].Add(fileParams);
+       AliInfo(Form("fullLocalFileName = %s", fullLocalFileName.Data()));
 
-       return localFileName.Data();
+       return fullLocalFileName.Data();
 
 }
 
 //______________________________________________________________________________________________
-Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFileName)
+Bool_t AliShuttle::RetrieveFile(UInt_t system, const char* fxsFileName, const char* localFileName)
 {
+// Copies file from FXS to local Shuttle machine
 
        // check temp directory: trying to cd to temp; if it does not exist, create it
-       AliDebug(2, Form("Copy file %s from DAQ FES into folder %s and rename it as %s",
-                       daqFileName,fgkShuttleTempDir, localFileName));
+       AliDebug(2, Form("Copy file %s from %s FXS into %s/%s",
+                       GetSystemName(system), fxsFileName, GetShuttleTempDir(), localFileName));
 
-       void* dir = gSystem->OpenDirectory(fgkShuttleTempDir);
+       void* dir = gSystem->OpenDirectory(GetShuttleTempDir());
        if (dir == NULL) {
-               if (gSystem->mkdir(fgkShuttleTempDir, kTRUE)) {
-                       AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
+               if (gSystem->mkdir(GetShuttleTempDir(), kTRUE)) {
+                       AliError(Form("Can't open directory <%s>", GetShuttleTempDir()));
                        return kFALSE;
                }
 
@@ -1303,43 +1391,66 @@ Bool_t AliShuttle::RetrieveDAQFile(const char* daqFileName, const char* localFil
                gSystem->FreeDirectory(dir);
        }
 
-       TString baseDAQFESFolder = "DAQ";
-       TString command = Form("scp %s@%s:%s/%s %s/%s",
-               fConfig->GetFESUser(kDAQ),
-               fConfig->GetFESHost(kDAQ),
-               baseDAQFESFolder.Data(),
-               daqFileName,
-               fgkShuttleTempDir,
-               localFileName);
+       TString baseFXSFolder;
+       if (system == kDAQ)
+       {
+               baseFXSFolder = "FES/";
+       }
+       else if (system == kDCS)
+       {
+               baseFXSFolder = "";
+       }
+       else if (system == kHLT)
+       {
+               baseFXSFolder = "~/";
+       }
 
-       AliDebug(2, Form("%s",command.Data()));
 
-       UInt_t nRetries = 0;
-       UInt_t maxRetries = 3;
+       TString command = Form("scp -oPort=%d -2 %s@%s:%s%s %s/%s",
+               fConfig->GetFXSPort(system),
+               fConfig->GetFXSUser(system),
+               fConfig->GetFXSHost(system),
+               baseFXSFolder.Data(),
+               fxsFileName,
+               GetShuttleTempDir(),
+               localFileName);
 
-       // copy!! if successful TSystem::Exec returns 0
-       while(nRetries++ < maxRetries) {
-               AliDebug(2, Form("Trying to copy file. Retry # %d", nRetries));
-               if(gSystem->Exec(command.Data()) == 0) return kTRUE;
-       }
+       AliDebug(2, Form("%s",command.Data()));
 
-       return kFALSE;
+       Bool_t result = (gSystem->Exec(command.Data()) == 0);
 
+       return result;
 }
 
 //______________________________________________________________________________________________
-TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id)
+TList* AliShuttle::GetFileSources(Int_t system, const char* detector, const char* id)
 {
-// Retrieves a file from the DCS FES.
+// Get sources producing the condition file Id from file exchange servers
+
+       if (system == kDCS)
+       {
+               AliError("DCS system has only one source of data!");
+               return NULL;
+
+       }
 
        // check connection, in case connect
-       if(!Connect(kDAQ)){
-               Log(detector, "GetDAQFileSources - Couldn't connect to DAQ Logbook");
-               return 0;
+       if (!Connect(system))
+       {
+               Log(detector, Form("GetFile - Couldn't connect to %s FXS database", GetSystemName(system)));
+               return NULL;
        }
 
-       // Query preparation
-       TString sqlQueryStart = "select DAQsource from logbook_fs where";
+       TString sourceName = 0;
+       if (system == kDAQ)
+       {
+               sourceName = "DAQsource";
+       } else if (system == kHLT)
+       {
+               sourceName = "DDLnumbers";
+       }
+
+       TString sqlQueryStart = Form("select %s from %s where", sourceName.Data(), fConfig->GetFXSdbTable(kDAQ));
        TString whereClause = Form("run=%d and detector=\"%s\" and fileId=\"%s\"",
                                GetCurrentRun(), detector, id);
        TString sqlQuery = Form("%s %s", sqlQueryStart.Data(), whereClause.Data());
@@ -1348,15 +1459,17 @@ TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id)
 
        // Query execution
        TSQLResult* aResult;
-       aResult = fServer[kDAQ]->Query(sqlQuery);
+       aResult = fServer[system]->Query(sqlQuery);
        if (!aResult) {
-               Log(detector, Form("GetDAQFileSources - Can't execute SQL query for id: %s", id));
+               Log(detector, Form("GetFileSources - Can't execute SQL query to %s database for id: %s",
+                               GetSystemName(system), id));
                return 0;
        }
 
-       if (aResult->GetRowCount() == 0) {
+       if (aResult->GetRowCount() == 0)
+       {
                Log(detector,
-                       Form("GetDAQFileSources - No entry in FES table for id: %s", id));
+                       Form("GetFileSources - No entry in %s FXS table for id: %s", GetSystemName(system), id));
                delete aResult;
                return 0;
        }
@@ -1365,106 +1478,166 @@ TList* AliShuttle::GetDAQFileSources(const char* detector, const char* id)
        TList *list = new TList();
        list->SetOwner(1);
 
-       while((aRow = aResult->Next())){
+       while ((aRow = aResult->Next()))
+       {
 
-               TString daqSource(aRow->GetField(0), aRow->GetFieldLength(0));
-               AliDebug(2, Form("daqSource = %s", daqSource.Data()));
-               list->Add(new TObjString(daqSource));
+               TString source(aRow->GetField(0), aRow->GetFieldLength(0));
+               AliDebug(2, Form("%s = %s", sourceName.Data(), source.Data()));
+               list->Add(new TObjString(source));
                delete aRow;
        }
+
        delete aResult;
 
        return list;
-
 }
 
 //______________________________________________________________________________________________
-const char* AliShuttle::GetDCSFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
-// Retrieves a file from the DCS FES.
-
-return "You're in DCS";
-
-}
+Bool_t AliShuttle::Connect(Int_t system)
+{
+// Connect to MySQL Server of the system's FXS MySQL databases
+// DAQ Logbook, Shuttle Logbook and DAQ FXS db are on the same host
 
-//______________________________________________________________________________________________
-TList* AliShuttle::GetDCSFileSources(const char* /*detector*/, const char* /*id*/){
-// Retrieves a file from the DCS FES.
+       // check connection: if already connected return
+       if(fServer[system] && fServer[system]->IsConnected()) return kTRUE;
 
-return NULL;
+       TString dbHost, dbUser, dbPass, dbName;
 
-}
+       if (system < 3) // FXS db servers
+       {
+               dbHost = Form("mysql://%s:%d", fConfig->GetFXSdbHost(system), fConfig->GetFXSdbPort(system));
+               dbUser = fConfig->GetFXSdbUser(system);
+               dbPass = fConfig->GetFXSdbPass(system);
+               dbName =   fConfig->GetFXSdbName(system);
+       } else { // Run & Shuttle logbook servers
+       // TODO Will the Shuttle logbook server be the same as the Run logbook server ???
+               dbHost = Form("mysql://%s:%d", fConfig->GetDAQlbHost(), fConfig->GetDAQlbPort());
+               dbUser = fConfig->GetDAQlbUser();
+               dbPass = fConfig->GetDAQlbPass();
+               dbName =   fConfig->GetDAQlbDB();
+       }
 
-//______________________________________________________________________________________________
-const char* AliShuttle::GetHLTFileName(const char* /*detector*/, const char* /*id*/, const char* /*source*/){
-// Retrieves a file from the HLT FES.
+       fServer[system] = TSQLServer::Connect(dbHost.Data(), dbUser.Data(), dbPass.Data());
+       if (!fServer[system] || !fServer[system]->IsConnected()) {
+               if(system < 3)
+               {
+               AliError(Form("Can't establish connection to FXS database for %s",
+                                       AliShuttleInterface::GetSystemName(system)));
+               } else {
+               AliError("Can't establish connection to Run logbook.");
+               }
+               if(fServer[system]) delete fServer[system];
+               return kFALSE;
+       }
 
-return "You're in HLT";
+       // Get tables
+       TSQLResult* aResult=0;
+       switch(system){
+               case kDAQ:
+                       aResult = fServer[kDAQ]->GetTables(dbName.Data());
+                       break;
+               case kDCS:
+                       aResult = fServer[kDCS]->GetTables(dbName.Data());
+                       break;
+               case kHLT:
+                       aResult = fServer[kHLT]->GetTables(dbName.Data());
+                       break;
+               default:
+                       aResult = fServer[3]->GetTables(dbName.Data());
+                       break;
+       }
 
+       delete aResult;
+       return kTRUE;
 }
 
 //______________________________________________________________________________________________
-TList* AliShuttle::GetHLTFileSources(const char* /*detector*/, const char* /*id*/){
-// Retrieves a file from the HLT FES.
+Bool_t AliShuttle::UpdateTable()
+{
+// Update FXS table filling time_processed field in all rows corresponding to current run and detector
 
-return NULL;
+       Bool_t result = kTRUE;
 
-}
+       for (UInt_t system=0; system<3; system++)
+       {
+               if(!fFXSCalled[system]) continue;
 
-//______________________________________________________________________________________________
-Bool_t AliShuttle::UpdateDAQTable()
-{
-// Update DAQ table filling time_processed field in all rows corresponding to current run and detector
+               // check connection, in case connect
+               if (!Connect(system))
+               {
+                       Log(fCurrentDetector, Form("UpdateTable - Couldn't connect to %s FXS database", GetSystemName(system)));
+                       result = kFALSE;
+                       continue;
+               }
 
-       // check connection, in case connect
-       if(!Connect(kDAQ)){
-               Log(fCurrentDetector, "UpdateDAQTable - Couldn't connect to DAQ Logbook");
-               return kFALSE;
-       }
+               TTimeStamp now; // now
 
-       TTimeStamp now; // now
+               // Loop on FXS list entries
+               TIter iter(&fFXSlist[system]);
+               TObjString *aFXSentry=0;
+               while ((aFXSentry = dynamic_cast<TObjString*> (iter.Next())))
+               {
+                       TString aFXSentrystr = aFXSentry->String();
+                       TObjArray *aFXSarray = aFXSentrystr.Tokenize("#!?!#");
+                       if (!aFXSarray || aFXSarray->GetEntries() != 2 )
+                       {
+                               Log(fCurrentDetector, Form("UpdateTable - error updating %s FXS entry. Check string: <%s>",
+                                       GetSystemName(system), aFXSentrystr.Data()));
+                               if(aFXSarray) delete aFXSarray;
+                               result = kFALSE;
+                               continue;
+                       }
+                       const char* fileId = ((TObjString*) aFXSarray->At(0))->GetName();
+                       const char* source = ((TObjString*) aFXSarray->At(1))->GetName();
 
-       // Loop on FES list entries
-       TIter iter(&fFESlist[kDAQ]);
-       TObjString *aFESentry=0;
-       while((aFESentry = dynamic_cast<TObjString*> (iter.Next()))){
-               TString aFESentrystr = aFESentry->String();
-               TObjArray *aFESarray = aFESentrystr.Tokenize("_!?!_");
-               if(!aFESarray || aFESarray->GetEntries() != 2 ) {
-                       Log(fCurrentDetector, Form("UpdateDAQTable - error updating FES entry. Check string: <%s>",
-                               aFESentrystr.Data()));
-                       if(aFESarray) delete aFESarray;
-                       return kFALSE;
-               }
-               const char* fileId = ((TObjString*) aFESarray->At(0))->GetName();
-               const char* daqSource = ((TObjString*) aFESarray->At(1))->GetName();
-               TString whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
-                       GetCurrentRun(), fCurrentDetector.Data(), fileId, daqSource);
+                       TString whereClause;
+                       if (system == kDAQ)
+                       {
+                               whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DAQsource=\"%s\";",
+                                                       GetCurrentRun(), fCurrentDetector.Data(), fileId, source);
+                       }
+                       else if (system == kDCS)
+                       {
+                               whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\";",
+                                                       GetCurrentRun(), fCurrentDetector.Data(), fileId);
+                       }
+                       else if (system == kHLT)
+                       {
+                               whereClause = Form("where run=%d and detector=\"%s\" and fileId=\"%s\" and DDLnumbers=\"%s\";",
+                                                       GetCurrentRun(), fCurrentDetector.Data(), fileId, source);
+                       }
 
-               delete aFESarray;
+                       delete aFXSarray;
 
-               TString sqlQuery = Form("update logbook_fs set time_processed=%d %s", now.GetSec(), whereClause.Data());
+                       TString sqlQuery = Form("update %s set time_processed=%d %s", fConfig->GetFXSdbTable(system),
+                                                               now.GetSec(), whereClause.Data());
 
-               AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
+                       AliDebug(2, Form("SQL query: \n%s",sqlQuery.Data()));
 
-               // Query execution
-               TSQLResult* aResult;
-               aResult = dynamic_cast<TSQLResult*> (fServer[3]->Query(sqlQuery));
-               if (!aResult) {
-                       Log(fCurrentDetector, Form("UpdateDAQTable - Can't execute SQL query <%s>", sqlQuery.Data()));
-                       return kFALSE;
+                       // Query execution
+                       TSQLResult* aResult;
+                       aResult = dynamic_cast<TSQLResult*> (fServer[system]->Query(sqlQuery));
+                       if (!aResult)
+                       {
+                               Log(fCurrentDetector, Form("UpdateTable - %s db: can't execute SQL query <%s>",
+                                                               GetSystemName(system), sqlQuery.Data()));
+                               result = kFALSE;
+                               continue;
+                       }
+                       delete aResult;
                }
-               delete aResult;
        }
 
-       return kTRUE;
+       return result;
 }
 
-
 //______________________________________________________________________________________________
 Bool_t AliShuttle::UpdateShuttleLogbook(const char* detector, const char* status)
 {
-// Update Shuttle logbook filling detector or shuttle_done column
-// ex. of usage: UpdateShuttleLogbook("PHOS", "DONE") or UpdateShuttleLogbook("shuttle_done")
+       //
+       // Update Shuttle logbook filling detector or shuttle_done column
+       // ex. of usage: UpdateShuttleLogbook("PHOS", "DONE") or UpdateShuttleLogbook("shuttle_done")
+       //
 
        // check connection, in case connect
        if(!Connect(3)){
@@ -1474,8 +1647,17 @@ Bool_t AliShuttle::UpdateShuttleLogbook(const char* detector, const char* status
 
        TString detName(detector);
        TString setClause;
-       if(detName == "shuttle_done") {
+       if(detName == "shuttle_done")
+       {
                setClause = "set shuttle_done=1";
+
+               // Send the information to ML
+               TMonaLisaText  mlStatus("SHUTTLE_status", "Done");
+
+               TList mlList;
+               mlList.Add(&mlStatus);
+
+               fMonaLisa->SendParameters(&mlList);
        } else {
                TString statusStr(status);
                if(statusStr.Contains("done", TString::kIgnoreCase) ||
@@ -1537,10 +1719,10 @@ void AliShuttle::Log(const char* detector, const char* message)
 {
 // Fill log string with a message
 
-       void* dir = gSystem->OpenDirectory(fgkShuttleLogDir);
+       void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
        if (dir == NULL) {
-               if (gSystem->mkdir(fgkShuttleLogDir, kTRUE)) {
-                       AliError(Form("Can't open directory <%s>", fgkShuttleTempDir));
+               if (gSystem->mkdir(GetShuttleLogDir(), kTRUE)) {
+                       AliError(Form("Can't open directory <%s>", GetShuttleLogDir()));
                        return;
                }
 
@@ -1549,13 +1731,18 @@ void AliShuttle::Log(const char* detector, const char* message)
        }
 
        TString toLog = Form("%s (%d): %s - ", TTimeStamp(time(0)).AsString("s"), getpid(), detector);
-       if(GetCurrentRun()>=0 ) toLog += Form("run %d - ", GetCurrentRun());
+       if (GetCurrentRun() >= 0) 
+               toLog += Form("run %d - ", GetCurrentRun());
        toLog += Form("%s", message);
 
        AliInfo(toLog.Data());
 
        TString fileName;
-       fileName.Form("%s/%s.log", fgkShuttleLogDir, detector);
+       if (GetCurrentRun() >= 0) 
+               fileName.Form("%s/%s_%d.log", GetShuttleLogDir(), detector, GetCurrentRun());
+       else
+               fileName.Form("%s/%s.log", GetShuttleLogDir(), detector);
+       
        gSystem->ExpandPathName(fileName);
 
        ofstream logFile;
@@ -1599,10 +1786,20 @@ Bool_t AliShuttle::Collect(Int_t run)
                return kFALSE;
        }
 
+       if (shuttleLogbookEntries.GetEntries() == 0)
+       {
+               if (run == -1)
+                       Log("SHUTTLE","Collect - Found no UNPROCESSED runs in Shuttle logbook");
+               else
+                       Log("SHUTTLE", Form("Collect - Run %d is already DONE "
+                                               "or it does not exist in Shuttle logbook", run));
+               return kTRUE;
+       }
+
        for (UInt_t iDet=0; iDet<NDetectors(); iDet++)
                fFirstUnprocessed[iDet] = kTRUE;
 
-       if (run != 1)
+       if (run != -1)
        {
                // query Shuttle logbook for earlier runs, check if some detectors are unprocessed,
                // flag them into fFirstUnprocessed array
@@ -1638,6 +1835,7 @@ Bool_t AliShuttle::Collect(Int_t run)
                return kFALSE;
        }
 
+       Log("SHUTTLE", "Collect - Requested run(s) successfully processed");
        return kTRUE;
 }
 
@@ -1655,6 +1853,10 @@ Bool_t AliShuttle::RetrieveConditionsData(const TObjArray& dateEntries)
                if (!Process(anEntry)){
                        hasError = kTRUE;
                }
+
+               // clean SHUTTLE temp directory
+               TString command = Form("rm -f %s/*.shuttle", GetShuttleTempDir());
+               gSystem->Exec(command.Data());
        }
 
        return hasError == kFALSE;
@@ -1664,13 +1866,13 @@ Bool_t AliShuttle::RetrieveConditionsData(const TObjArray& dateEntries)
 ULong_t AliShuttle::GetTimeOfLastAction() const
 {
        ULong_t tmp;
-       
+
        fMonitoringMutex->Lock();
 
        tmp = fLastActionTime;
-       
+
        fMonitoringMutex->UnLock();
-       
+
        return tmp;
 }
 
@@ -1680,23 +1882,23 @@ const TString AliShuttle::GetLastAction() const
        // returns a string description of the last action
 
        TString tmp;
-       
+
        fMonitoringMutex->Lock();
        
        tmp = fLastAction;
        
        fMonitoringMutex->UnLock();
 
-       return tmp;     
+       return tmp;
 }
 
 //______________________________________________________________________________________________
 void AliShuttle::SetLastAction(const char* action)
 {
        // updates the monitoring variables
-       
+
        fMonitoringMutex->Lock();
-       
+
        fLastAction = action;
        fLastActionTime = time(0);
        
@@ -1715,3 +1917,99 @@ const char* AliShuttle::GetRunParameter(const char* param)
 
        return fLogbookEntry->GetRunParameter(param);
 }
+
+//______________________________________________________________________________________________
+Bool_t AliShuttle::SendMail()
+{
+// sends a mail to the subdetector expert in case of preprocessor error
+
+       void* dir = gSystem->OpenDirectory(GetShuttleLogDir());
+       if (dir == NULL)
+       {
+               if (gSystem->mkdir(GetShuttleLogDir(), kTRUE))
+               {
+                       AliError(Form("Can't open directory <%s>", GetShuttleLogDir()));
+                       return kFALSE;
+               }
+
+       } else {
+               gSystem->FreeDirectory(dir);
+       }
+
+       TString bodyFileName;
+       bodyFileName.Form("%s/mail.body", GetShuttleLogDir());
+       gSystem->ExpandPathName(bodyFileName);
+
+       ofstream mailBody;
+       mailBody.open(bodyFileName, ofstream::out);
+
+       if (!mailBody.is_open())
+       {
+               AliError(Form("Could not open mail body file %s", bodyFileName.Data()));
+               return kFALSE;
+       }
+
+       TString to="";
+       TIter iterExperts(fConfig->GetResponsibles(fCurrentDetector));
+       TObjString *anExpert=0;
+       while ((anExpert = (TObjString*) iterExperts.Next()))
+       {
+               to += Form("%s,", anExpert->GetName());
+       }
+       to.Remove(to.Length()-1);
+       AliDebug(2, Form("to: %s",to.Data()));
+
+       // TODO this will be removed...
+       if (to.Contains("not_yet_set")) {
+               AliInfo("List of detector responsibles not yet set!");
+               return kFALSE;
+       }
+
+       TString cc="alberto.colla@cern.ch";
+
+       TString subject = Form("%s Shuttle preprocessor error in run %d !",
+                               fCurrentDetector.Data(), GetCurrentRun());
+       AliDebug(2, Form("subject: %s", subject.Data()));
+
+       TString body = Form("Dear %s expert(s), \n\n", fCurrentDetector.Data());
+       body += Form("SHUTTLE just detected that your preprocessor "
+                       "exited with ERROR state in run %d!!\n\n", GetCurrentRun());
+       body += Form("Please check %s status on the web page asap!\n\n", fCurrentDetector.Data());
+       body += Form("The last 10 lines of %s log file are following:\n\n");
+
+       AliDebug(2, Form("Body begin: %s", body.Data()));
+
+       mailBody << body.Data();
+       mailBody.close();
+       mailBody.open(bodyFileName, ofstream::out | ofstream::app);
+
+       TString logFileName = Form("%s/%s_%d.log", GetShuttleLogDir(), fCurrentDetector.Data(), GetCurrentRun());
+       TString tailCommand = Form("tail -n 10 %s >> %s", logFileName.Data(), bodyFileName.Data());
+       if (gSystem->Exec(tailCommand.Data()))
+       {
+               mailBody << Form("%s log file not found ...\n\n", fCurrentDetector.Data());
+       }
+
+       TString endBody = Form("------------------------------------------------------\n\n");
+       endBody += Form("In case of problems please contact the SHUTTLE core team.\n\n");
+       endBody += "Please do not answer this message directly, it is automatically generated.\n\n";
+       endBody += "Sincerely yours,\n\n \t\t\tthe SHUTTLE\n";
+
+       AliDebug(2, Form("Body end: %s", endBody.Data()));
+
+       mailBody << endBody.Data();
+
+       mailBody.close();
+
+       // send mail!
+       TString mailCommand = Form("mail -s \"%s\" -c %s %s < %s",
+                                               subject.Data(),
+                                               cc.Data(),
+                                               to.Data(),
+                                               bodyFileName.Data());
+       AliDebug(2, Form("mail command: %s", mailCommand.Data()));
+
+       Bool_t result = gSystem->Exec(mailCommand.Data());
+
+       return result == 0;
+}