]> git.uio.no Git - u/mrichter/AliRoot.git/blobdiff - HLT/BASE/AliHLTDataBuffer.cxx
(very old)bugfix: -concatenate-event option did not work as expected
[u/mrichter/AliRoot.git] / HLT / BASE / AliHLTDataBuffer.cxx
index d0ecff5b6410089fbec56c9bccf836cfeb9264f6..5825dd55fd46c5b8b28a0ba359559728014fe63a 100644 (file)
@@ -1,10 +1,11 @@
 // $Id$
 
 /**************************************************************************
- * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
+ * This file is property of and copyright by the ALICE HLT Project        * 
+ * ALICE Experiment at CERN, All rights reserved.                         *
  *                                                                        *
- * Authors: Matthias Richter <Matthias.Richter@ift.uib.no>                *
- *          for The ALICE Off-line Project.                               *
+ * Primary Authors: Matthias Richter <Matthias.Richter@ift.uib.no>        *
+ *                  for The ALICE HLT Project.                            *
  *                                                                        *
  * Permission to use, copy, modify and distribute this software and its   *
  * documentation strictly for non-commercial purposes is hereby granted   *
     @brief  Handling of Data Buffers for HLT components.
 */
 
+// see header file for class documentation
+// or
+// refer to README to build package
+// or
+// visit http://web.ift.uib.no/~kjeks/doc/alice-hlt
+
 #if __GNUC__>= 3
 using namespace std;
 #endif
@@ -28,10 +35,15 @@ using namespace std;
 #include "AliHLTDataBuffer.h"
 #include "AliHLTConsumerDescriptor.h"
 #include "AliHLTComponent.h"
+#include "AliHLTTask.h"
 #include <cerrno>
+#include <cassert>
 //#include <string>
 //#include "AliHLTSystem.h"
 
+typedef vector<AliHLTDataBuffer::AliHLTDataSegment> AliHLTDataSegmentList;
+typedef vector<AliHLTDataBuffer::AliHLTRawBuffer*>  AliHLTRawBufferPList;
+
 /** ROOT macro for the implementation of ROOT specific class methods */
 ClassImp(AliHLTDataBuffer)
 
@@ -42,7 +54,9 @@ AliHLTDataBuffer::AliHLTDataBuffer()
   fActiveConsumers(),
   fReleasedConsumers(),
   fpBuffer(NULL),
-  fFlags(0)
+  fFlags(0),
+  fForwardedSegmentSources(),
+  fForwardedSegments()
 {
   // see header file for class documentation
   // or
@@ -56,31 +70,9 @@ AliHLTDataBuffer::AliHLTDataBuffer()
   fgNofInstances++;
 }
 
-AliHLTDataBuffer::AliHLTDataBuffer(const AliHLTDataBuffer&)
-  :
-  TObject(),
-  AliHLTLogging(),
-  fSegments(),
-  fConsumers(),
-  fActiveConsumers(),
-  fReleasedConsumers(),
-  fpBuffer(NULL),
-  fFlags(0)
-{
-  // see header file for function documentation
-  HLTFatal("copy constructor untested");
-}
-
-AliHLTDataBuffer& AliHLTDataBuffer::operator=(const AliHLTDataBuffer&)
-{ 
-  // see header file for function documentation
-  HLTFatal("assignment operator untested");
-  return *this;
-}
-
 int AliHLTDataBuffer::fgNofInstances=0;
-vector<AliHLTDataBuffer::AliHLTRawBuffer*> AliHLTDataBuffer::fgFreeBuffers;
-vector<AliHLTDataBuffer::AliHLTRawBuffer*> AliHLTDataBuffer::fgActiveBuffers;
+AliHLTRawBufferPList AliHLTDataBuffer::fgFreeBuffers;
+AliHLTRawBufferPList AliHLTDataBuffer::fgActiveBuffers;
 AliHLTUInt32_t AliHLTDataBuffer::fgMargin=1024;
 AliHLTLogging AliHLTDataBuffer::fgLogging;
 const Int_t AliHLTDataBuffer::fgkSafetyPatternSize=16;
@@ -118,15 +110,15 @@ int AliHLTDataBuffer::SetConsumer(AliHLTComponent* pConsumer)
   return iResult;
 }
 
-int AliHLTDataBuffer::FindMatchingDataBlocks(const AliHLTComponent* pConsumer, vector<AliHLTComponentDataType>* tgtList)
+int AliHLTDataBuffer::FindMatchingDataBlocks(const AliHLTComponent* pConsumer, AliHLTComponentDataTypeList* tgtList)
 {
   // see header file for function documentation
   int iResult=0;
   if (pConsumer) {
-    vector<AliHLTDataBuffer::AliHLTDataSegment> segments;
+    AliHLTDataSegmentList segments;
     if ((iResult=FindMatchingDataSegments(pConsumer, segments))>=0) {
       if (tgtList) {
-       vector<AliHLTDataBuffer::AliHLTDataSegment>::iterator segment=segments.begin();
+       AliHLTDataSegmentList::iterator segment=segments.begin();
        while (segment!=segments.end()) {
          tgtList->push_back((*segment).fDataType);
          segment++;
@@ -144,12 +136,23 @@ int AliHLTDataBuffer::FindMatchingDataSegments(const AliHLTComponent* pConsumer,
 {
   // see header file for function documentation
   int iResult=0;
+
+  // Matthias 26.09.2007 relax the restriction to matching data blocks
+  // all blocks are passed to the consumer, which is the policy also in
+  // PubSub
+  tgtList.assign(fSegments.begin(), fSegments.end());
+
+  // add all forwarded blocks
+  tgtList.insert(tgtList.begin(), fForwardedSegments.begin(), fForwardedSegments.end());
+  iResult=tgtList.size();
+  return iResult;
+  
   if (pConsumer) {
-    vector<AliHLTComponentDataType> dtlist;
+    AliHLTComponentDataTypeList dtlist;
     ((AliHLTComponent*)pConsumer)->GetInputDataTypes(dtlist);
-    vector<AliHLTDataBuffer::AliHLTDataSegment>::iterator segment=fSegments.begin();
+    AliHLTDataSegmentList::iterator segment=fSegments.begin();
     while (segment!=fSegments.end()) {
-      vector<AliHLTComponentDataType>::iterator type=dtlist.begin();
+      AliHLTComponentDataTypeList::iterator type=dtlist.begin();
       while (type!=dtlist.end()) {
        if ((*segment).fDataType==(*type) ||
            (*type)==kAliHLTAnyDataType) {
@@ -167,45 +170,55 @@ int AliHLTDataBuffer::FindMatchingDataSegments(const AliHLTComponent* pConsumer,
   return iResult;
 }
 
-int AliHLTDataBuffer::Subscribe(const AliHLTComponent* pConsumer, AliHLTComponentBlockData* arrayBlockDesc, int iArraySize)
+int AliHLTDataBuffer::Subscribe(const AliHLTComponent* pConsumer, AliHLTComponentBlockDataList& blockDescList)
 {
   // see header file for function documentation
   int iResult=0;
-  if (pConsumer && arrayBlockDesc) {
-    if (fpBuffer) {
+  if (pConsumer) {
+    if (1/*fpBuffer*/) {
       AliHLTConsumerDescriptor* pDesc=FindConsumer(pConsumer, fConsumers);
       if (pDesc) {
-       vector<AliHLTDataBuffer::AliHLTDataSegment> tgtList;
-       /* TODO: think about a good policy for this check
-        * is it enough that at least one segment is available, or have all to be available?
-        * or is it possible to have optional segments?
-        */
-       if ((iResult=FindMatchingDataSegments(pConsumer, tgtList))>0) {
-         int i =0;
-         vector<AliHLTDataBuffer::AliHLTDataSegment>::iterator segment=tgtList.begin();
-         while (segment!=tgtList.end() && i<iArraySize) {
+       AliHLTDataSegmentList tgtList;
+       // Matthias 26.07.2007 AliHLTSystem should behave the same way as PubSub
+       // so it does not matter if there are matching data types or not, unless
+       // we implement such a check in PubSub
+       if ((iResult=FindMatchingDataSegments(pConsumer, tgtList))>=0) {
+         AliHLTDataSegmentList::iterator segment=tgtList.begin();
+         while (segment!=tgtList.end()) {
            // fill the block data descriptor
-           arrayBlockDesc[i].fStructSize=sizeof(AliHLTComponentBlockData);
-           // the shared memory key is not used in AliRoot
-           arrayBlockDesc[i].fShmKey.fStructSize=sizeof(AliHLTComponentShmData);
-           arrayBlockDesc[i].fShmKey.fShmType=gkAliHLTComponentInvalidShmType;
-           arrayBlockDesc[i].fShmKey.fShmID=gkAliHLTComponentInvalidShmID;
-           arrayBlockDesc[i].fOffset=(*segment).fSegmentOffset;
-           arrayBlockDesc[i].fPtr=fpBuffer->fPtr;
-           arrayBlockDesc[i].fSize=(*segment).fSegmentSize;
-           arrayBlockDesc[i].fDataType=(*segment).fDataType;
-           arrayBlockDesc[i].fSpecification=(*segment).fSpecification;
-           pDesc->SetActiveDataSegment(arrayBlockDesc[i].fOffset, arrayBlockDesc[i].fSize);
-           HLTDebug("component %p (%s) subscribed to segment #%d offset %d", pConsumer, ((AliHLTComponent*)pConsumer)->GetComponentID(), i, arrayBlockDesc[i].fOffset);
-           i++;
+           AliHLTComponentBlockData bd;
+           AliHLTComponent::FillBlockData(bd);
+           // This models the behavior of PubSub.
+           // For incoming data blocks, fOffset must be ignored by the
+           // processing component. It is set for bookkeeping in the framework.
+           // fPtr always points to the beginning of the data.
+           bd.fOffset=0;
+           AliHLTUInt8_t* pTgt=*segment;
+           bd.fPtr=reinterpret_cast<void*>(pTgt);
+           bd.fSize=(*segment).fSegmentSize;
+           bd.fDataType=(*segment).fDataType;
+           bd.fSpecification=(*segment).fSpecification;
+           blockDescList.push_back(bd);
+           pDesc->SetActiveDataSegment(*segment);
+           HLTDebug("component %p (%s) subscribed to segment offset %d size %d data type %s %#x", 
+                    pConsumer, ((AliHLTComponent*)pConsumer)->GetComponentID(), bd.fOffset,
+                    bd.fSize, (AliHLTComponent::DataType2Text(bd.fDataType)).c_str(), 
+                    bd.fSpecification);
            segment++;
          }
          // move this consumer to the active list
-         if (ChangeConsumerState(pDesc, fConsumers, fActiveConsumers)>=0) {
+         if (tgtList.size()==0) {
+           ChangeConsumerState(pDesc, fConsumers, fReleasedConsumers);
+           HLTDebug("no input data for component %p (%s) available", pConsumer, ((AliHLTComponent*)pConsumer)->GetComponentID());
+         } else if (ChangeConsumerState(pDesc, fConsumers, fActiveConsumers)>=0) {
            HLTDebug("component %p (%s) subscribed to data buffer %p", pConsumer, ((AliHLTComponent*)pConsumer)->GetComponentID(), this);
          } else {
            // TODO: cleanup the consumer descriptor correctly
-           memset(arrayBlockDesc, 0, iArraySize*sizeof(AliHLTComponentBlockData));
+           segment=tgtList.begin();
+           while (segment!=tgtList.end()) {
+             blockDescList.pop_back();
+             segment++;
+           }
            HLTError("can not activate consumer %p for data buffer %p", pConsumer, this);
            iResult=-EACCES;
          }
@@ -218,8 +231,11 @@ int AliHLTDataBuffer::Subscribe(const AliHLTComponent* pConsumer, AliHLTComponen
        iResult=-ENOENT;
       }
     } else {
-      HLTError("data buffer %p is empty", this);
-      iResult=-ENODATA;
+      // Matthias 26.07.2007 until now, data had to be present for successful subscription
+      // in order to be consistent with the PubSub framework, this restiction has been
+      // removed
+      //HLTError("data buffer %p is empty", this);
+      //iResult=-ENODATA;
     }
   } else {
     HLTError("invalid parameter");
@@ -228,26 +244,57 @@ int AliHLTDataBuffer::Subscribe(const AliHLTComponent* pConsumer, AliHLTComponen
   return iResult;
 }
 
-int AliHLTDataBuffer::Release(AliHLTComponentBlockData* pBlockDesc, const AliHLTComponent* pConsumer)
+int AliHLTDataBuffer::Release(AliHLTComponentBlockData* pBlockDesc,
+                             const AliHLTComponent* pConsumer,
+                             const AliHLTTask* pOwnerTask)
 {
   // see header file for function documentation
   int iResult=0;
   if (pBlockDesc && pConsumer) {
     AliHLTConsumerDescriptor* pDesc=FindConsumer(pConsumer, fActiveConsumers);
     if (pDesc) {
-      if ((iResult=pDesc->CheckActiveDataSegment(pBlockDesc->fOffset, pBlockDesc->fSize))!=1) {
+      if ((iResult=pDesc->CheckActiveDataSegment(AliHLTDataSegment(pBlockDesc->fPtr, pBlockDesc->fOffset, pBlockDesc->fSize)))!=1) {
        HLTWarning("data segment missmatch, component %p has not subscribed to a segment with offset %#x and size %d", pConsumer, pBlockDesc->fOffset, pBlockDesc->fSize);
        // TODO: appropriate error handling, but so far optional
        iResult=0;
       } else {
-       pDesc->ReleaseActiveDataSegment(pBlockDesc->fOffset, pBlockDesc->fSize);
-       pBlockDesc->fOffset=0;
-       pBlockDesc->fPtr=NULL;
-       pBlockDesc->fSize=0;
+       pDesc->ReleaseActiveDataSegment(AliHLTDataSegment(pBlockDesc->fPtr, pBlockDesc->fOffset, pBlockDesc->fSize));
       }
+      if (GetNofPendingConsumers()==0 && fForwardedSegments.size()>0) {
+       // last consumer, release forwarded segments
+       assert(fForwardedSegments.size()==fForwardedSegmentSources.size());
+       AliHLTDataSegmentList::iterator segment=fForwardedSegments.begin();
+       AliHLTTaskPList::iterator src=fForwardedSegmentSources.begin();
+       //HLTDebug("%p checking forwarded segments", this);
+       for (; segment!=fForwardedSegments.end(); segment++, src++) {
+         //HLTDebug("segment ptr=%p offset=%d size=%d\n"
+         //   "block ptr=%p offset=%d size=%d", (*segment).fPtr, (*segment).fSegmentOffset, (*segment).fSegmentSize, pBlockDesc->fPtr, pBlockDesc->fOffset, pBlockDesc->fSize);
+         if ((*segment)==AliHLTDataSegment(pBlockDesc->fPtr, pBlockDesc->fOffset, pBlockDesc->fSize)) {
+           //HLTDebug("release segment of task %p", *src);
+           assert((*src)!=NULL);
+           if ((*src)!=NULL) {
+             if ((*src)->Release(pBlockDesc, pOwnerTask)>=0) {
+               HLTDebug("task %s (%p) released forwarded segment %p size %d of task %s (%p)",
+                        pOwnerTask->GetName(), pOwnerTask, (*segment).GetPtr(), (*segment).GetSize(),
+                        (*src)->GetName(), *src);
+             } else {
+               HLTError("task %s (%p) failed releasing forwarded segment %p size %d of task %s (%p)",
+                        pOwnerTask->GetName(), pOwnerTask, (*segment).GetPtr(), (*segment).GetSize(),
+                        (*src)->GetName(), *src);
+             }
+           }
+           fForwardedSegments.erase(segment);
+           fForwardedSegmentSources.erase(src);
+           break;
+         }
+       }
+      }
+      pBlockDesc->fOffset=0;
+      pBlockDesc->fPtr=NULL;
+      pBlockDesc->fSize=0;
       if (pDesc->GetNofActiveSegments()==0) {
        if ((iResult=ChangeConsumerState(pDesc, fActiveConsumers, fReleasedConsumers))>=0) {
-         if (GetNofActiveConsumers()==0) {
+         if (GetNofActiveConsumers()==0 && GetNofPendingConsumers()==0) {
            // this is the last consumer, reset the consumer list and release the raw buffer
            ResetDataBuffer();
          }
@@ -267,13 +314,27 @@ int AliHLTDataBuffer::Release(AliHLTComponentBlockData* pBlockDesc, const AliHLT
   return iResult;
 }
 
+int AliHLTDataBuffer::Forward(AliHLTTask* pSrcTask, AliHLTComponentBlockData* pBlockDesc)
+{
+  // see header file for function documentation
+  if (pSrcTask==NULL || pBlockDesc==NULL) return -EINVAL;
+  assert(fForwardedSegments.size()==fForwardedSegmentSources.size());
+  if (fForwardedSegments.size()!=fForwardedSegmentSources.size()) return -EFAULT;
+  fForwardedSegmentSources.push_back(pSrcTask);
+  fForwardedSegments.push_back(AliHLTDataSegment(pBlockDesc->fPtr, pBlockDesc->fOffset, pBlockDesc->fSize, pBlockDesc->fDataType, pBlockDesc->fSpecification));
+  return 0;
+}
+
 AliHLTUInt8_t* AliHLTDataBuffer::GetTargetBuffer(int iMinSize)
 {
   // see header file for function documentation
   AliHLTUInt8_t* pTargetBuffer=NULL;
+  if (fpBuffer!=NULL) {
+    HLTWarning("data buffer not properly reset, possible memory leak\n");
+  }
   fpBuffer=CreateRawBuffer(iMinSize);
   if (fpBuffer) {
-    pTargetBuffer=(AliHLTUInt8_t*)fpBuffer->fPtr;
+    pTargetBuffer=*fpBuffer;
   } else {
     HLTError("can not create raw buffer");
   }
@@ -286,25 +347,39 @@ int AliHLTDataBuffer::SetSegments(AliHLTUInt8_t* pTgt, AliHLTComponentBlockData*
   int iResult=0;
   if (pTgt && arrayBlockData && iSize>=0) {
     if (fpBuffer) {
-      if (fpBuffer->fPtr==(void*)pTgt) {
+      if (*fpBuffer==pTgt) {
        AliHLTDataBuffer::AliHLTDataSegment segment;
-       memset(&segment, 0, sizeof(AliHLTDataBuffer::AliHLTDataSegment));
        for (int i=0; i<iSize; i++) {
-         if (arrayBlockData[i].fOffset+arrayBlockData[i].fSize<=fpBuffer->fSize) {
-           segment.fSegmentOffset=arrayBlockData[i].fOffset;
-           segment.fSegmentSize=arrayBlockData[i].fSize;
-           segment.fDataType=arrayBlockData[i].fDataType;
-           segment.fSpecification=arrayBlockData[i].fSpecification;
-           fSegments.push_back(segment);
-           HLTDebug("set segment %s with size %d at offset %d", AliHLTComponent::DataType2Text(segment.fDataType).data(), segment.fSegmentSize, segment.fSegmentOffset);
+         // This function has to model the behavior of PubSub
+         // For output blocks only the fOffset value is used, this must be the offset
+         // relative to the output pointer. fPtr must be either NULL or the output
+         // pointer. In either case it is 'ignored' and set to the beginning of the
+         // data buffer
+         if (arrayBlockData[i].fPtr==NULL ||
+             arrayBlockData[i].fPtr==*fpBuffer) {
+           arrayBlockData[i].fPtr=*fpBuffer;
+           if (arrayBlockData[i].fOffset+arrayBlockData[i].fSize<=fpBuffer->GetUsedSize()) {
+             segment.fSegmentOffset=arrayBlockData[i].fOffset;
+             segment.fPtr=(AliHLTUInt8_t*)arrayBlockData[i].fPtr;
+             segment.fSegmentSize=arrayBlockData[i].fSize;
+             segment.fDataType=arrayBlockData[i].fDataType;
+             segment.fSpecification=arrayBlockData[i].fSpecification;
+             fSegments.push_back(segment);
+             HLTDebug("set segment %s with size %d at offset %d", AliHLTComponent::DataType2Text(segment.fDataType).data(), segment.fSegmentSize, segment.fSegmentOffset);
+           } else {
+             HLTError("block data specification %#d (%s) exceeds size of data buffer", i, AliHLTComponent::DataType2Text(arrayBlockData[i].fDataType).data());
+             HLTError("block offset=%d, block size=%d, buffer size=%d", arrayBlockData[i].fOffset, arrayBlockData[i].fSize, fpBuffer->GetUsedSize());
+             iResult=-E2BIG;
+           }
          } else {
-           HLTError("block data specification %#d (%s) exceeds size of data buffer", i, AliHLTComponent::DataType2Text(arrayBlockData[i].fDataType).data());
-           HLTError("block offset=%d, block size=%d, buffer size=%d", arrayBlockData[i].fOffset, arrayBlockData[i].fSize, fpBuffer->fSize);
-           iResult=-E2BIG;
+           HLTError("invalid pointer (%p) in block data specification (buffer %p size %d)."
+                    "please note: for output blocks only the fOffset value is valid and must "
+                    "be relative to the output buffer", arrayBlockData[i].fPtr, fpBuffer->GetPointer(), fpBuffer->GetUsedSize());
+           iResult=-ERANGE;
          }
        }
       } else {
-       HLTError("this data buffer (%p) does not match the internal data buffer %p of raw buffer %p", pTgt, fpBuffer->fPtr, fpBuffer);
+       HLTError("this data buffer (%p) does not match the internal data buffer %p of raw buffer %p", pTgt, fpBuffer->GetPointer(), fpBuffer);
        iResult=-EINVAL;
       }
     } else {
@@ -339,6 +414,13 @@ int AliHLTDataBuffer::GetNofConsumers()
   return iResult;
 }
 
+int AliHLTDataBuffer::GetNofPendingConsumers()
+{
+  // see header file for function documentation
+  int iResult=fConsumers.size();
+  return iResult;
+}
+
 int AliHLTDataBuffer::GetNofActiveConsumers()
 {
   // see header file for function documentation
@@ -351,14 +433,14 @@ AliHLTDataBuffer::AliHLTRawBuffer* AliHLTDataBuffer::CreateRawBuffer(AliHLTUInt3
   // see header file for function documentation
   AliHLTRawBuffer* pRawBuffer=NULL;
   unsigned int reqSize=size+fgkSafetyPatternSize;
-  vector<AliHLTRawBuffer*>::iterator buffer=fgFreeBuffers.begin();
+  AliHLTRawBufferPList::iterator buffer=fgFreeBuffers.begin();
   while (buffer!=fgFreeBuffers.end() && pRawBuffer==NULL) {
-    if ((*buffer)->fTotalSize>=reqSize && ((*buffer)->fTotalSize-reqSize)<fgMargin) {
+    if ((*buffer)->CheckSize(reqSize)) {
       // assign this element
       pRawBuffer=*buffer;
-      pRawBuffer->fSize=size;
+      pRawBuffer->UseBuffer(size);
       fgFreeBuffers.erase(buffer);
-      fgLogging.Logging(kHLTLogDebug, "AliHLTDataBuffer::CreateRawBuffer", "data buffer handling", "raw buffer container %p provided for request of %d bytes (total %d available in buffer %p)", pRawBuffer, size, pRawBuffer->fTotalSize, pRawBuffer->fPtr);
+      fgLogging.Logging(kHLTLogDebug, "AliHLTDataBuffer::CreateRawBuffer", "data buffer handling", "raw buffer container %p provided for request of %d bytes (total %d available in buffer %p)", pRawBuffer, size, pRawBuffer->GetTotalSize(), pRawBuffer->GetPointer());
       fgActiveBuffers.push_back(pRawBuffer);
       break;
     }
@@ -366,18 +448,12 @@ AliHLTDataBuffer::AliHLTRawBuffer* AliHLTDataBuffer::CreateRawBuffer(AliHLTUInt3
   }
   if (pRawBuffer==NULL) {
     // no buffer found, create a new one
-    pRawBuffer=new AliHLTRawBuffer;
+    pRawBuffer=new AliHLTRawBuffer(reqSize);
     if (pRawBuffer) {
-      memset(pRawBuffer, 0, sizeof(AliHLTRawBuffer));
-      pRawBuffer->fPtr=malloc(reqSize);
-      if (pRawBuffer->fPtr) {
-       pRawBuffer->fSize=size;
-       pRawBuffer->fTotalSize=reqSize;
+      if (pRawBuffer->GetPointer()) {
+       pRawBuffer->UseBuffer(size);
        fgActiveBuffers.push_back(pRawBuffer);
-       fgLogging.Logging(kHLTLogDebug, "AliHLTDataBuffer::CreateRawBuffer", "data buffer handling", "new raw buffer %p of size %d created (container %p)", pRawBuffer->fPtr, pRawBuffer->fTotalSize, pRawBuffer);
-       if (fgkSafetyPatternSize>0) {
-         memcpy(((char*)pRawBuffer->fPtr)+pRawBuffer->fSize, fgkSafetyPattern, fgkSafetyPatternSize);
-       }
+       fgLogging.Logging(kHLTLogDebug, "AliHLTDataBuffer::CreateRawBuffer", "data buffer handling", "new raw buffer %p of size %d created (container %p)", pRawBuffer->GetPointer(), pRawBuffer->GetTotalSize(), pRawBuffer);
       } else {
        delete pRawBuffer;
        pRawBuffer=NULL;
@@ -387,6 +463,10 @@ AliHLTDataBuffer::AliHLTRawBuffer* AliHLTDataBuffer::CreateRawBuffer(AliHLTUInt3
       fgLogging.Logging(kHLTLogError, "AliHLTDataBuffer::CreateRawBuffer", "data buffer handling", "memory allocation failed");
     }
   }
+  if (pRawBuffer!=NULL && fgkSafetyPatternSize>0) {
+    //fgLogging.Logging(kHLTLogDebug, "AliHLTDataBuffer::CreateRawBuffer", "data buffer handling", "writing safety pattern to %p offset %d", (*buffer)->GetPointer(), (*buffer)->GetUsedSize());
+    pRawBuffer->WritePattern(fgkSafetyPattern, fgkSafetyPatternSize);
+  }
   return pRawBuffer;
 }
 
@@ -395,17 +475,18 @@ int AliHLTDataBuffer::ReleaseRawBuffer(AliHLTRawBuffer* pBuffer)
   // see header file for function documentation
   int iResult=0;
   if (pBuffer) {
-    vector<AliHLTRawBuffer*>::iterator buffer=fgActiveBuffers.begin();
+    AliHLTRawBufferPList::iterator buffer=fgActiveBuffers.begin();
     while (buffer!=fgActiveBuffers.end() && (*buffer)!=pBuffer) {
       buffer++;
     }
     if (buffer!=fgActiveBuffers.end()) {
       if (fgkSafetyPatternSize>0) {
-       if (memcmp(((char*)(*buffer)->fPtr)+(*buffer)->fSize, fgkSafetyPattern, fgkSafetyPatternSize)!=0) {
-         fgLogging.Logging(kHLTLogFatal, "AliHLTDataBuffer::ReleaseRawBuffer", "data buffer handling", "component has written beyond end of data buffer %p size %d", (*buffer)->fPtr, (*buffer)->fSize);
+       //fgLogging.Logging(kHLTLogDebug, "AliHLTDataBuffer::ReleaseRawBuffer", "data buffer handling", "comparing safety pattern at %p offset %d", (*buffer)->GetPointer(), reinterpret_cast<AliHLTUInt32_t>(*buffer));
+       if ((*buffer)->CheckPattern(fgkSafetyPattern, fgkSafetyPatternSize)) {
+         fgLogging.Logging(kHLTLogFatal, "AliHLTDataBuffer::ReleaseRawBuffer", "data buffer handling", "component has written beyond end of data buffer %p size %d", (*buffer)->GetPointer(), (*buffer)->GetUsedSize());
        }
       }
-      (*buffer)->fSize=0;
+      (*buffer)->Reset();
       fgFreeBuffers.push_back(*buffer);
       fgActiveBuffers.erase(buffer);
     } else {
@@ -424,29 +505,29 @@ int AliHLTDataBuffer::DeleteRawBuffers()
 {
   // see header file for function documentation
   int iResult=0;
-  vector<AliHLTRawBuffer*>::iterator buffer=fgFreeBuffers.begin();
-  while (buffer!=fgFreeBuffers.end()) {
-    free((*buffer)->fPtr);
+//   int iTotalSize=0;
+//   int iCount=fgFreeBuffers.size()+fgActiveBuffers.size();
+  AliHLTRawBufferPList::iterator buffer;;
+  while ((buffer=fgFreeBuffers.begin())!=fgFreeBuffers.end()) {
+//     iTotalSize+=(*buffer)->GetTotalSize();
     delete *buffer;
     fgFreeBuffers.erase(buffer);
-    buffer=fgFreeBuffers.begin();
   }
-  buffer=fgActiveBuffers.begin();
-  while (buffer!=fgActiveBuffers.end()) {
-    fgLogging.Logging(kHLTLogWarning, "AliHLTDataBuffer::ReleaseRawBuffer", "data buffer handling", "request to delete active raw buffer container (raw buffer %p, size %d)", (*buffer)->fPtr, (*buffer)->fTotalSize);
-    free((*buffer)->fPtr);
+  while ((buffer=fgActiveBuffers.begin())!=fgActiveBuffers.end()) {
+//     iTotalSize+=(*buffer)->GetTotalSize();
+    fgLogging.Logging(kHLTLogWarning, "AliHLTDataBuffer::ReleaseRawBuffer", "data buffer handling", "request to delete active raw buffer container (raw buffer %p, size %d)", (*buffer)->GetPointer(), (*buffer)->GetTotalSize());
     delete *buffer;
     fgActiveBuffers.erase(buffer);
-    buffer=fgActiveBuffers.begin();
   }
+//   fgLogging.Logging(kHLTLogInfo, "AliHLTDataBuffer::ReleaseRawBuffer", "data buffer handling", "Total memory allocation: %d byte in %d buffers", iTotalSize, iCount);
   return iResult;
 }
 
-AliHLTConsumerDescriptor* AliHLTDataBuffer::FindConsumer(const AliHLTComponent* pConsumer, vector<AliHLTConsumerDescriptor*> &list) const
+AliHLTConsumerDescriptor* AliHLTDataBuffer::FindConsumer(const AliHLTComponent* pConsumer, AliHLTConsumerDescriptorPList &list) const
 {
   // see header file for function documentation
   AliHLTConsumerDescriptor* pDesc=NULL;
-  vector<AliHLTConsumerDescriptor*>::iterator desc=list.begin();
+  AliHLTConsumerDescriptorPList::iterator desc=list.begin();
   while (desc!=list.end() && pDesc==NULL) {
     if ((pConsumer==NULL || (*desc)->GetComponent()==pConsumer)) {
       pDesc=*desc;
@@ -463,8 +544,22 @@ int AliHLTDataBuffer::ResetDataBuffer()
   AliHLTRawBuffer* pBuffer=fpBuffer;
   fpBuffer=NULL;
 
+  // cleanup forwarded segment lists
+  assert(fForwardedSegments.size()==0);
+  fForwardedSegments.clear();
+  fForwardedSegmentSources.clear();
+
   // cleanup consumer states
-  vector<AliHLTConsumerDescriptor*>::iterator desc=fReleasedConsumers.begin();
+  AliHLTConsumerDescriptorPList::iterator desc;
+//   if (GetNofPendingConsumers()>0) {
+//     desc=fConsumers.begin();
+//     while (desc!=fConsumers.end()) {
+//       AliHLTComponent* pComp=(*desc)->GetComponent();
+//       HLTError("internal error: consumer %p (%s %p) did not get data from data buffer %p", *desc, pComp?pComp->GetComponentID():"", pComp, this);
+//       desc++;
+//     }
+//   }
+  desc=fReleasedConsumers.begin();
   while (desc!=fReleasedConsumers.end()) {
     AliHLTConsumerDescriptor* pDesc=*desc;
     fReleasedConsumers.erase(desc);
@@ -481,7 +576,7 @@ int AliHLTDataBuffer::ResetDataBuffer()
   }
 
   // cleanup segments
-  vector<AliHLTDataBuffer::AliHLTDataSegment>::iterator segment=fSegments.begin();
+  AliHLTDataSegmentList::iterator segment=fSegments.begin();
   while (segment!=fSegments.end()) {
     fSegments.erase(segment);
     segment=fSegments.begin();
@@ -501,11 +596,11 @@ int AliHLTDataBuffer::Reset()
 }
 
 // this is the version which works on lists of components instead of consumer descriptors
-// int AliHLTDataBuffer::ChangeConsumerState(AliHLTComponent* pConsumer, vector<AliHLTComponent*> &srcList, vector<AliHLTComponent*> &tgtList)
+// int AliHLTDataBuffer::ChangeConsumerState(AliHLTComponent* pConsumer, AliHLTComponentPList &srcList, AliHLTComponentPList &tgtList)
 // {
 //   int iResult=0;
 //   if (pDesc) {
-//     vector<AliHLTComponent*>::iterator desc=srcList.begin();
+//     AliHLTComponentPList::iterator desc=srcList.begin();
 //     while (desc!=srcList.end()) {
 //       if ((*desc)==pConsumer) {
 //     srcList.erase(desc);
@@ -525,12 +620,12 @@ int AliHLTDataBuffer::Reset()
 //   return iResult;
 // }
 
-int AliHLTDataBuffer::ChangeConsumerState(AliHLTConsumerDescriptor* pDesc, vector<AliHLTConsumerDescriptor*> &srcList, vector<AliHLTConsumerDescriptor*> &tgtList)
+int AliHLTDataBuffer::ChangeConsumerState(AliHLTConsumerDescriptor* pDesc, AliHLTConsumerDescriptorPList &srcList, AliHLTConsumerDescriptorPList &tgtList)
 {
   // see header file for function documentation
   int iResult=-ENOENT;
   if (pDesc) {
-    vector<AliHLTConsumerDescriptor*>::iterator desc=srcList.begin();
+    AliHLTConsumerDescriptorPList::iterator desc=srcList.begin();
     while (desc!=srcList.end()) {
       if ((*desc)==pDesc) {
        srcList.erase(desc);
@@ -555,7 +650,7 @@ int AliHLTDataBuffer::CleanupConsumerList()
   // see header file for function documentation
   int iResult=0;
   ResetDataBuffer();
-  vector<AliHLTConsumerDescriptor*>::iterator desc=fConsumers.begin();
+  AliHLTConsumerDescriptorPList::iterator desc=fConsumers.begin();
   while (desc!=fConsumers.end()) {
     delete *desc;
     fConsumers.erase(desc);
@@ -567,7 +662,7 @@ int AliHLTDataBuffer::CleanupConsumerList()
 int AliHLTDataBuffer::FindConsumer(AliHLTComponent* pConsumer, int bAllLists)
 {
   // see header file for function documentation
-  vector<AliHLTConsumerDescriptor*>::iterator desc=fConsumers.begin();
+  AliHLTConsumerDescriptorPList::iterator desc=fConsumers.begin();
   while (desc!=fConsumers.end()) {
     if ((*desc)->GetComponent()==pConsumer)
       return 1;
@@ -589,3 +684,110 @@ int AliHLTDataBuffer::FindConsumer(AliHLTComponent* pConsumer, int bAllLists)
   }
   return 0;
 }
+
+AliHLTDataBuffer::AliHLTRawBuffer::AliHLTRawBuffer(AliHLTUInt32_t size)
+  :
+  fSize(0),
+  fTotalSize(size),
+  fPtr(static_cast<AliHLTUInt8_t*>(malloc(size)))
+{
+  // see header file for class documentation
+  // or
+  // refer to README to build package
+  // or
+  // visit http://web.ift.uib.no/~kjeks/doc/alice-hlt
+  if (fPtr==NULL) {
+    fSize=0;
+    fTotalSize=0;
+  }
+}
+
+AliHLTDataBuffer::AliHLTRawBuffer::~AliHLTRawBuffer()
+{
+  if (fPtr) {
+    free(fPtr);
+  }
+  fPtr=NULL;
+  fSize=0;
+  fTotalSize=0;
+}
+
+int AliHLTDataBuffer::AliHLTRawBuffer::operator==(void* ptr) const
+{
+  // see header file for function documentation
+  return fPtr == static_cast<AliHLTUInt8_t*>(ptr);
+}
+
+int AliHLTDataBuffer::AliHLTRawBuffer::operator<=(void* ptr) const
+{
+  // see header file for function documentation
+  int iResult=fPtr <= static_cast<AliHLTUInt8_t*>(ptr);
+  //printf("%p: %p <= %p (%d)\n", this, fPtr, ptr, iResult);
+  return iResult;
+}
+
+int AliHLTDataBuffer::AliHLTRawBuffer::operator>(void* ptr) const
+{
+  // see header file for function documentation
+  int iResult=fPtr+fSize > static_cast<AliHLTUInt8_t*>(ptr);
+  //printf("%p: %p + %d > %p (%d)\n", this, fPtr, fSize, ptr, iResult);
+  return iResult;
+}
+
+int AliHLTDataBuffer::AliHLTRawBuffer::operator-(void* ptr) const
+{
+  // see header file for function documentation
+  return static_cast<int>(static_cast<AliHLTUInt8_t*>(ptr)-fPtr);
+}
+
+AliHLTUInt8_t* AliHLTDataBuffer::AliHLTRawBuffer::UseBuffer(AliHLTUInt32_t size)
+{
+  // see header file for function documentation
+  if (size>0 && fTotalSize>=size) {
+    fSize=size;
+    return fPtr;
+  }
+  return NULL;
+}
+
+int AliHLTDataBuffer::AliHLTRawBuffer::CheckSize(AliHLTUInt32_t size) const
+{
+  // see header file for function documentation
+  return fTotalSize>=size && ((fTotalSize-size)<fgMargin);
+}
+
+int AliHLTDataBuffer::AliHLTRawBuffer::Reset()
+{
+  // see header file for function documentation
+  fSize=0;
+  return 0;
+}
+
+int AliHLTDataBuffer::AliHLTRawBuffer::WritePattern(const char* pattern, int size)
+{
+  // see header file for function documentation
+  int iResult=0;
+  if (pattern!=NULL && size>0) {
+    if (fSize+size<=fTotalSize) {
+      memcpy(((char*)fPtr)+fSize, pattern, size);
+      iResult=size;
+    } else {
+      iResult=-ENOSPC;
+    }
+  }
+  return iResult;
+}
+
+int AliHLTDataBuffer::AliHLTRawBuffer::CheckPattern(const char* pattern, int size) const
+{
+  // see header file for function documentation
+  int iResult=0;
+  if (pattern!=NULL && size>0) {
+    if (fSize+size<=fTotalSize) {
+      iResult=memcmp(((char*)fPtr)+fSize, pattern, size)!=0;
+    } else {
+      iResult=-ENOSPC;
+    }
+  }
+  return iResult;
+}