3 //**************************************************************************
4 //* This file is property of and copyright by the *
5 //* ALICE Experiment at CERN, All rights reserved. *
7 //* Primary Authors: Matthias Richter <Matthias.Richter@ift.uib.no> *
8 //* for The ALICE HLT Project. *
10 //* Permission to use, copy, modify and distribute this software and its *
11 //* documentation strictly for non-commercial purposes is hereby granted *
12 //* without fee, provided that the above copyright notice appears in all *
13 //* copies and that both the copyright notice and this permission notice *
14 //* appear in the supporting documentation. The authors make no claims *
15 //* about the suitability of this software for any purpose. It is *
16 //* provided "as is" without express or implied warranty. *
17 //**************************************************************************
19 // @file AliHLTDataBuffer.cxx
20 // @author Matthias Richter
22 // @brief Handling of Data Buffers for HLT components.
23 // @note Only used in the AliRoot framework
25 #include "AliHLTDataBuffer.h"
26 #include "AliHLTConsumerDescriptor.h"
27 #include "AliHLTComponent.h"
28 #include "AliHLTTask.h"
32 //#include "AliHLTSystem.h"
36 /** ROOT macro for the implementation of ROOT specific class methods */
37 ClassImp(AliHLTDataBuffer)
39 AliHLTDataBuffer::AliHLTDataBuffer()
47 fForwardedSegmentSources(),
50 // see header file for class documentation
52 // refer to README to build package
54 // visit http://web.ift.uib.no/~kjeks/doc/alice-hlt
57 fActiveConsumers.empty();
58 fReleasedConsumers.empty();
62 int AliHLTDataBuffer::fgNofInstances=0;
63 AliHLTDataBuffer::AliHLTRawBufferPList AliHLTDataBuffer::fgFreeBuffers;
64 AliHLTDataBuffer::AliHLTRawBufferPList AliHLTDataBuffer::fgActiveBuffers;
65 AliHLTUInt32_t AliHLTDataBuffer::fgMargin=1024;
66 AliHLTLogging AliHLTDataBuffer::fgLogging;
67 const Int_t AliHLTDataBuffer::fgkSafetyPatternSize=16;
68 const char AliHLTDataBuffer::fgkSafetyPattern[]={0x28, 0x63, 0x29, 0x4d, 0x52, 0x49, 0x43, 0x48, 0x54, 0x45, 0x52, 0x20, 0x32, 0x30, 0x30, 0x37};
69 AliHLTUInt32_t AliHLTDataBuffer::fgEventCount=0;
71 AliHLTDataBuffer::~AliHLTDataBuffer()
73 // see header file for function documentation
74 CleanupConsumerList();
76 if (--fgNofInstances<=0) {
81 int AliHLTDataBuffer::SetConsumer(AliHLTComponent* pConsumer)
83 // see header file for function documentation
86 if (FindConsumer(pConsumer)) {
87 HLTWarning("consumer %s (%p) already set to data buffer %p", pConsumer->GetComponentID(), pConsumer, this);
89 AliHLTConsumerDescriptor* pDesc=new AliHLTConsumerDescriptor(pConsumer);
91 fConsumers.push_back(pDesc);
92 HLTDebug("set consumer %s (%p) to data buffer %p", pConsumer->GetComponentID(), pConsumer, this);
94 HLTError("memory allocation failed");
98 HLTError("invalid parameter: consumer component (nil)");
104 int AliHLTDataBuffer::FindMatchingDataBlocks(const AliHLTComponent* pConsumer, AliHLTComponentDataTypeList* tgtList)
106 // see header file for function documentation
109 AliHLTDataSegmentList segments;
110 if ((iResult=FindMatchingDataSegments(pConsumer, segments))>=0) {
112 AliHLTDataSegmentList::iterator segment=segments.begin();
113 while (segment!=segments.end()) {
114 tgtList->push_back((*segment).fDataType);
118 iResult=segments.size();
126 int AliHLTDataBuffer::FindMatchingDataSegments(const AliHLTComponent* /*pConsumer*/,
127 AliHLTDataSegmentList& tgtList)
129 // see header file for function documentation
132 // Matthias 26.09.2007 relax the restriction to matching data blocks
133 // all blocks are passed to the consumer, which is the policy also in
135 tgtList.assign(fSegments.begin(), fSegments.end());
137 // add all forwarded blocks
138 tgtList.insert(tgtList.begin(), fForwardedSegments.begin(), fForwardedSegments.end());
139 iResult=tgtList.size();
143 int AliHLTDataBuffer::Subscribe(const AliHLTComponent* pConsumer, AliHLTComponentBlockDataList& blockDescList)
145 // see header file for function documentation
149 AliHLTConsumerDescriptor* pDesc=FindConsumer(pConsumer, fConsumers);
151 AliHLTDataSegmentList tgtList;
152 // Matthias 26.07.2007 AliHLTSystem should behave the same way as PubSub
153 // so it does not matter if there are matching data types or not, unless
154 // we implement such a check in PubSub
155 if ((iResult=FindMatchingDataSegments(pConsumer, tgtList))>=0) {
156 AliHLTDataSegmentList::iterator segment=tgtList.begin();
157 while (segment!=tgtList.end()) {
158 // fill the block data descriptor
159 AliHLTComponentBlockData bd;
160 AliHLTComponent::FillBlockData(bd);
161 // This models the behavior of PubSub.
162 // For incoming data blocks, fOffset must be ignored by the
163 // processing component. It is set for bookkeeping in the framework.
164 // fPtr always points to the beginning of the data.
166 AliHLTUInt8_t* pTgt=*segment;
167 bd.fPtr=reinterpret_cast<void*>(pTgt);
168 bd.fSize=(*segment).fSegmentSize;
169 bd.fDataType=(*segment).fDataType;
170 bd.fSpecification=(*segment).fSpecification;
171 blockDescList.push_back(bd);
172 pDesc->SetActiveDataSegment(*segment);
173 HLTDebug("component %p (%s) subscribed to segment offset %d size %d data type %s %#x",
174 pConsumer, ((AliHLTComponent*)pConsumer)->GetComponentID(), bd.fOffset,
175 bd.fSize, (AliHLTComponent::DataType2Text(bd.fDataType)).c_str(),
179 // move this consumer to the active list
180 if (tgtList.size()==0) {
181 ChangeConsumerState(pDesc, fConsumers, fReleasedConsumers);
182 HLTDebug("no input data for component %p (%s) available", pConsumer, ((AliHLTComponent*)pConsumer)->GetComponentID());
183 } else if (ChangeConsumerState(pDesc, fConsumers, fActiveConsumers)>=0) {
184 HLTDebug("component %p (%s) subscribed to data buffer %p", pConsumer, ((AliHLTComponent*)pConsumer)->GetComponentID(), this);
186 // TODO: cleanup the consumer descriptor correctly
187 segment=tgtList.begin();
188 while (segment!=tgtList.end()) {
189 blockDescList.pop_back();
192 HLTError("can not activate consumer %p for data buffer %p", pConsumer, this);
196 HLTError("unresolved data segment(s) for component %p (%s)", pConsumer, ((AliHLTComponent*)pConsumer)->GetComponentID());
200 if (!FindConsumer(pConsumer)) {
201 HLTError("component %p is not a data consumer of data buffer %p", pConsumer, this);
203 HLTError("component %p is a valid data consumer of data buffer %p, but did not release it's buffer subscription", pConsumer, this);
208 // Matthias 26.07.2007 until now, data had to be present for successful subscription
209 // in order to be consistent with the PubSub framework, this restiction has been
211 //HLTError("data buffer %p is empty", this);
215 HLTError("invalid parameter");
221 int AliHLTDataBuffer::Release(AliHLTComponentBlockData* pBlockDesc,
222 const AliHLTComponent* pConsumer,
223 const AliHLTTask* pOwnerTask)
225 // see header file for function documentation
227 if (pBlockDesc && pConsumer) {
228 AliHLTConsumerDescriptor* pDesc=FindConsumer(pConsumer, fActiveConsumers);
230 if ((iResult=pDesc->CheckActiveDataSegment(AliHLTDataSegment(pBlockDesc->fPtr, pBlockDesc->fOffset, pBlockDesc->fSize)))!=1) {
231 HLTWarning("data segment mismatch, component %p has not subscribed to a segment with offset %#x and size %d", pConsumer, pBlockDesc->fOffset, pBlockDesc->fSize);
232 // TODO: appropriate error handling, but so far optional
235 pDesc->ReleaseActiveDataSegment(AliHLTDataSegment(pBlockDesc->fPtr, pBlockDesc->fOffset, pBlockDesc->fSize));
237 if (GetNofPendingConsumers()==0 && fForwardedSegments.size()>0) {
238 // last consumer, release forwarded segments
239 ReleaseForwardedBlock(pBlockDesc, pOwnerTask);
241 pBlockDesc->fOffset=0;
242 pBlockDesc->fPtr=NULL;
244 if (pDesc->GetNofActiveSegments()==0) {
245 if ((iResult=ChangeConsumerState(pDesc, fActiveConsumers, fReleasedConsumers))>=0) {
246 if (GetNofActiveConsumers()==0 && GetNofPendingConsumers()==0) {
247 // this is the last consumer, reset the consumer list and release the raw buffer
251 HLTError("can not deactivate consumer %p for data buffer %p", pConsumer, this);
256 HLTWarning("component %p has currently not subscribed to the data buffer %p", pConsumer, this);
260 HLTError("inavalid parameter: pBlockDesc=%p pConsumer=%p", pBlockDesc, pConsumer);
266 int AliHLTDataBuffer::ReleaseForwardedBlock(AliHLTComponentBlockData* pBlockDesc,
267 const AliHLTTask* pOwnerTask)
269 // see header file for function documentation
271 if (pBlockDesc && pOwnerTask) {
272 assert(fForwardedSegments.size()==fForwardedSegmentSources.size());
273 AliHLTDataSegmentList::iterator segment=fForwardedSegments.begin();
274 AliHLTTaskPList::iterator src=fForwardedSegmentSources.begin();
275 //HLTDebug("%p checking forwarded segments", this);
276 for (; segment!=fForwardedSegments.end(); segment++, src++) {
277 //HLTDebug("segment ptr=%p offset=%d size=%d\n"
278 // "block ptr=%p offset=%d size=%d", (*segment).fPtr, (*segment).fSegmentOffset, (*segment).fSegmentSize, pBlockDesc->fPtr, pBlockDesc->fOffset, pBlockDesc->fSize);
279 if ((*segment)==AliHLTDataSegment(pBlockDesc->fPtr, pBlockDesc->fOffset, pBlockDesc->fSize)) {
280 //HLTDebug("release segment of task %p", *src);
281 assert((*src)!=NULL);
283 if ((*src)->Release(pBlockDesc, pOwnerTask)>=0) {
284 HLTDebug("task %s (%p) released forwarded segment %p size %d of task %s (%p)",
285 pOwnerTask->GetName(), pOwnerTask, (*segment).GetPtr(), (*segment).GetSize(),
286 (*src)->GetName(), *src);
288 HLTError("task %s (%p) failed releasing forwarded segment %p size %d of task %s (%p)",
289 pOwnerTask->GetName(), pOwnerTask, (*segment).GetPtr(), (*segment).GetSize(),
290 (*src)->GetName(), *src);
293 fForwardedSegments.erase(segment);
294 fForwardedSegmentSources.erase(src);
299 HLTError("inavalid parameter: pBlockDesc=%p pOwnerTask=%p", pBlockDesc, pOwnerTask);
305 int AliHLTDataBuffer::Forward(AliHLTTask* pSrcTask, AliHLTComponentBlockData* pBlockDesc)
307 // see header file for function documentation
308 if (pSrcTask==NULL || pBlockDesc==NULL) return -EINVAL;
309 assert(fForwardedSegments.size()==fForwardedSegmentSources.size());
310 if (fForwardedSegments.size()!=fForwardedSegmentSources.size()) return -EFAULT;
311 fForwardedSegmentSources.push_back(pSrcTask);
312 fForwardedSegments.push_back(AliHLTDataSegment(pBlockDesc->fPtr, pBlockDesc->fOffset, pBlockDesc->fSize, pBlockDesc->fDataType, pBlockDesc->fSpecification));
316 AliHLTUInt8_t* AliHLTDataBuffer::GetTargetBuffer(int iMinSize)
318 // see header file for function documentation
319 AliHLTUInt8_t* pTargetBuffer=NULL;
320 if (fpBuffer!=NULL) {
321 HLTWarning("data buffer not properly reset, possible memory leak\n");
323 fpBuffer=CreateRawBuffer(iMinSize);
325 pTargetBuffer=*fpBuffer;
327 HLTError("can not create raw buffer");
329 return pTargetBuffer;
332 int AliHLTDataBuffer::SetSegments(AliHLTUInt8_t* pTgt, AliHLTComponentBlockData* arrayBlockData, int iSize)
334 // see header file for function documentation
336 if (pTgt && arrayBlockData && iSize>=0) {
338 if (*fpBuffer==pTgt) {
339 AliHLTDataBuffer::AliHLTDataSegment segment;
340 AliHLTUInt32_t maxSize=0;
341 for (int i=0; i<iSize; i++) {
342 // This function has to model the behavior of PubSub
343 // For output blocks only the fOffset value is used, this must be the offset
344 // relative to the output pointer. fPtr must be either NULL or the output
345 // pointer. In either case it is 'ignored' and set to the beginning of the
347 if (arrayBlockData[i].fPtr==NULL ||
348 arrayBlockData[i].fPtr==*fpBuffer) {
349 arrayBlockData[i].fPtr=*fpBuffer;
350 if ((arrayBlockData[i].fOffset+arrayBlockData[i].fSize<=fpBuffer->GetUsedSize()) ||
351 ((arrayBlockData[i].fOffset==~(AliHLTUInt32_t)0) && arrayBlockData[i].fSize==0)) {
352 segment.fSegmentOffset=arrayBlockData[i].fOffset;
353 segment.fPtr=(AliHLTUInt8_t*)arrayBlockData[i].fPtr;
354 segment.fSegmentSize=arrayBlockData[i].fSize;
355 segment.fDataType=arrayBlockData[i].fDataType;
356 segment.fSpecification=arrayBlockData[i].fSpecification;
357 fSegments.push_back(segment);
358 HLTDebug("set segment %s with size %d at offset %d", AliHLTComponent::DataType2Text(segment.fDataType).data(), segment.fSegmentSize, segment.fSegmentOffset);
360 // find the actual size of the data
361 if ((arrayBlockData[i].fOffset!=~(AliHLTUInt32_t)0) &&
362 arrayBlockData[i].fOffset+arrayBlockData[i].fSize>maxSize) {
363 maxSize=arrayBlockData[i].fOffset+arrayBlockData[i].fSize;
366 HLTError("block data specification %#d (%s) exceeds size of data buffer", i, AliHLTComponent::DataType2Text(arrayBlockData[i].fDataType).data());
367 HLTError("block offset=%d, block size=%d, buffer size=%d", arrayBlockData[i].fOffset, arrayBlockData[i].fSize, fpBuffer->GetUsedSize());
371 HLTError("invalid pointer (%p) in block data specification (buffer %p size %d)."
372 "please note: for output blocks only the fOffset value is valid and must "
373 "be relative to the output buffer", arrayBlockData[i].fPtr, fpBuffer->GetPointer(), fpBuffer->GetUsedSize());
377 // to be enabled if unit test is ready
378 iResult=SetRawBufferDataSize(fpBuffer, maxSize);
380 HLTError("this data buffer (%p) does not match the internal data buffer %p of raw buffer %p", pTgt, fpBuffer->GetPointer(), fpBuffer);
384 HLTFatal("internal data structur mismatch");
388 HLTError("invalid parameter: pTgtBuffer=%p arrayBlockData=%p", pTgt, arrayBlockData);
394 int AliHLTDataBuffer::IsEmpty()
396 // see header file for function documentation
397 int iResult=(fpBuffer==NULL && fForwardedSegments.size()==0) || GetNofSegments()==0;
401 int AliHLTDataBuffer::GetNofSegments() const
403 // see header file for function documentation
404 int iResult=fSegments.size() + fForwardedSegments.size();
408 int AliHLTDataBuffer::GetNofConsumers() const
410 // see header file for function documentation
411 int iResult=fConsumers.size() + GetNofActiveConsumers() + fReleasedConsumers.size();
415 int AliHLTDataBuffer::GetNofPendingConsumers() const
417 // see header file for function documentation
418 int iResult=fConsumers.size();
422 int AliHLTDataBuffer::GetNofActiveConsumers() const
424 // see header file for function documentation
425 int iResult=fActiveConsumers.size();
429 AliHLTDataBuffer::AliHLTRawBuffer* AliHLTDataBuffer::CreateRawBuffer(AliHLTUInt32_t size)
431 // see header file for function documentation
432 AliHLTRawBuffer* pRawBuffer=NULL;
433 unsigned int reqSize=size+fgkSafetyPatternSize;
434 pRawBuffer=AliHLTDataBuffer::AliHLTRawPage::GlobalAlloc(reqSize);
436 pRawBuffer->UseBuffer(size);
438 if (pRawBuffer!=NULL && fgkSafetyPatternSize>0) {
439 //fgLogging.Logging(kHLTLogDebug, "AliHLTDataBuffer::CreateRawBuffer", "data buffer handling", "writing safety pattern to %p offset %d", (*buffer)->GetPointer(), (*buffer)->GetUsedSize());
440 pRawBuffer->WritePattern(fgkSafetyPattern, fgkSafetyPatternSize);
445 int AliHLTDataBuffer::SetRawBufferDataSize(AliHLTRawBuffer* pBuffer, AliHLTUInt32_t size) const
447 // see header file for function documentation
449 if (!pBuffer) return -EINVAL;
450 if (size>pBuffer->GetUsedSize()) {
451 HLTError("indicated data size %d exceeds data buffer %p (%d)", size, pBuffer->GetPointer(), pBuffer->GetUsedSize());
454 if (fgkSafetyPatternSize>0) {
455 if (pBuffer->CheckPattern(fgkSafetyPattern, fgkSafetyPatternSize)) {
456 HLTError("potential memory corruption: component has written beyond end of data buffer %p size %d", pBuffer->GetPointer(), pBuffer->GetUsedSize());
459 // shrink the buffer and write new pattern at the end
460 AliHLTDataBuffer::AliHLTRawPage* rawpage=AliHLTDataBuffer::AliHLTRawPage::FindPage(pBuffer);
462 pBuffer->UseBuffer(size);
463 if (rawpage->SetSize(pBuffer, size+fgkSafetyPatternSize)==0) {
466 fgLogging.Logging(kHLTLogError, "AliHLTDataBuffer::SetRawBufferDataSize", "data buffer handling", "failed to set size for raw buffer %p", pBuffer);
470 fgLogging.Logging(kHLTLogError, "AliHLTDataBuffer::SetRawBufferDataSize", "data buffer handling", "can not find raw page for buffer %p", pBuffer);
473 if (fgkSafetyPatternSize>0) {
474 pBuffer->WritePattern(fgkSafetyPattern, fgkSafetyPatternSize);
479 int AliHLTDataBuffer::ReleaseRawBuffer(AliHLTRawBuffer* pBuffer)
481 // see header file for function documentation
484 AliHLTDataBuffer::AliHLTRawPage* rawpage=AliHLTDataBuffer::AliHLTRawPage::FindPage(pBuffer);
487 if (fgkSafetyPatternSize>0) {
488 //fgLogging.Logging(kHLTLogDebug, "AliHLTDataBuffer::ReleaseRawBuffer", "data buffer handling", "comparing safety pattern at %p offset %d", pBuffer->GetPointer(), reinterpret_cast<AliHLTUInt32_t>(pBuffer));
489 if ((pBuffer)->CheckPattern(fgkSafetyPattern, fgkSafetyPatternSize)) {
490 fgLogging.Logging(kHLTLogFatal, "AliHLTDataBuffer::ReleaseRawBuffer", "data buffer handling", "component has written beyond end of data buffer %p size %d", pBuffer->GetPointer(), pBuffer->GetUsedSize());
494 if (rawpage->Free(pBuffer)==0) {
496 fgLogging.Logging(kHLTLogError, "AliHLTDataBuffer::ReleaseRawBuffer", "data buffer handling", "failed to release raw buffer %p", pBuffer);
499 fgLogging.Logging(kHLTLogError, "AliHLTDataBuffer::ReleaseRawBuffer", "data buffer handling", "can not find raw page for buffer %p", pBuffer);
503 fgLogging.Logging(kHLTLogError, "AliHLTDataBuffer::ReleaseRawBuffer", "data buffer handling", "invalid parameter");
510 int AliHLTDataBuffer::DeleteRawBuffers()
512 // see header file for function documentation
514 #ifdef ALIHLTSYSTEM_PROFILING
516 int iCount=fgFreeBuffers.size()+fgActiveBuffers.size();
517 #endif //ALIHLTSYSTEM_PROFILING
518 AliHLTRawBufferPList::iterator buffer;;
519 while ((buffer=fgFreeBuffers.begin())!=fgFreeBuffers.end()) {
520 #ifdef ALIHLTSYSTEM_PROFILING
521 iTotalSize+=(*buffer)->GetTotalSize();
522 #endif //ALIHLTSYSTEM_PROFILING
524 fgFreeBuffers.erase(buffer);
526 while ((buffer=fgActiveBuffers.begin())!=fgActiveBuffers.end()) {
527 #ifdef ALIHLTSYSTEM_PROFILING
528 iTotalSize+=(*buffer)->GetTotalSize();
529 #endif //ALIHLTSYSTEM_PROFILING
530 fgLogging.Logging(kHLTLogWarning, "AliHLTDataBuffer::DeleteRawBuffer", "data buffer handling", "request to delete active raw buffer container (raw buffer %p, size %d)", (*buffer)->GetPointer(), (*buffer)->GetTotalSize());
532 fgActiveBuffers.erase(buffer);
534 #ifdef ALIHLTSYSTEM_PROFILING
535 fgLogging.Logging(kHLTLogImportant, "AliHLTDataBuffer::DeleteRawBuffer", "data buffer handling", "Total memory allocation: %d byte in %d buffers", iTotalSize, iCount);
536 #endif //ALIHLTSYSTEM_PROFILING
540 int AliHLTDataBuffer::PrintStatistics()
542 // see header file for function documentation
545 AliHLTUInt32_t totalSize=0;
546 for (AliHLTDataBuffer::AliHLTRawPage* rawpage=AliHLTDataBuffer::AliHLTRawPage::NextPage(NULL);
548 rawpage=AliHLTDataBuffer::AliHLTRawPage::NextPage(rawpage)) {
550 totalSize+=rawpage->Size();
551 if (fgLogging.CheckFilter(kHLTLogDebug)) rawpage->Print("");
553 //if (rawpage) rawpage->Print("global");
554 fgLogging.Logging(kHLTLogInfo, "AliHLTDataBuffer::PrintStatistics", "data buffer handling", "total number of memory pages: %d total size %d", nofPages, totalSize);
559 AliHLTConsumerDescriptor* AliHLTDataBuffer::FindConsumer(const AliHLTComponent* pConsumer, AliHLTConsumerDescriptorPList &list) const
561 // see header file for function documentation
562 AliHLTConsumerDescriptor* pDesc=NULL;
563 AliHLTConsumerDescriptorPList::iterator desc=list.begin();
564 while (desc!=list.end() && pDesc==NULL) {
565 if ((pConsumer==NULL || (*desc)->GetComponent()==pConsumer)) {
573 int AliHLTDataBuffer::ResetDataBuffer()
575 // see header file for function documentation
577 AliHLTRawBuffer* pBuffer=fpBuffer;
580 // cleanup forwarded segment lists
581 assert(fForwardedSegments.size()==0);
582 fForwardedSegments.clear();
583 fForwardedSegmentSources.clear();
585 // cleanup consumer states
586 AliHLTConsumerDescriptorPList::iterator desc;
587 // if (GetNofPendingConsumers()>0) {
588 // desc=fConsumers.begin();
589 // while (desc!=fConsumers.end()) {
590 // AliHLTComponent* pComp=(*desc)->GetComponent();
591 // HLTError("internal error: consumer %p (%s %p) did not get data from data buffer %p", *desc, pComp?pComp->GetComponentID():"", pComp, this);
595 desc=fReleasedConsumers.begin();
596 while (desc!=fReleasedConsumers.end()) {
597 AliHLTConsumerDescriptor* pDesc=*desc;
598 fReleasedConsumers.erase(desc);
599 desc=fReleasedConsumers.begin();
600 fConsumers.push_back(pDesc);
602 desc=fActiveConsumers.begin();
603 while (desc!=fActiveConsumers.end()) {
604 AliHLTConsumerDescriptor* pDesc=*desc;
605 HLTWarning("consumer %p (%s) was not released", pDesc, pDesc->GetComponent()?pDesc->GetComponent()->GetComponentID():"### invalid component ###");
606 fActiveConsumers.erase(desc);
607 desc=fActiveConsumers.begin();
608 fConsumers.push_back(pDesc);
612 AliHLTDataSegmentList::iterator segment=fSegments.begin();
613 while (segment!=fSegments.end()) {
614 fSegments.erase(segment);
615 segment=fSegments.begin();
618 // cleanup raw buffer
620 ReleaseRawBuffer(pBuffer);
625 int AliHLTDataBuffer::Reset()
627 // see header file for function documentation
628 return ResetDataBuffer();
631 // this is the version which works on lists of components instead of consumer descriptors
632 // int AliHLTDataBuffer::ChangeConsumerState(AliHLTComponent* pConsumer, AliHLTComponentPList &srcList, AliHLTComponentPList &tgtList)
636 // AliHLTComponentPList::iterator desc=srcList.begin();
637 // while (desc!=srcList.end()) {
638 // if ((*desc)==pConsumer) {
639 // srcList.erase(desc);
640 // tgtList.push_back(pConsumer);
645 // if (desc==srcList.end()) {
646 // HLTError("can not find consumer component %p in list", pConsumer);
650 // HLTError("invalid parameter");
656 int AliHLTDataBuffer::ChangeConsumerState(AliHLTConsumerDescriptor* pDesc, AliHLTConsumerDescriptorPList &srcList, AliHLTConsumerDescriptorPList &tgtList)
658 // see header file for function documentation
661 AliHLTConsumerDescriptorPList::iterator desc=srcList.begin();
662 while (desc!=srcList.end()) {
663 if ((*desc)==pDesc) {
665 tgtList.push_back(pDesc);
672 HLTError("can not find consumer descriptor %p in list", pDesc);
675 HLTError("invalid parameter");
681 int AliHLTDataBuffer::CleanupConsumerList()
683 // see header file for function documentation
686 AliHLTConsumerDescriptorPList::iterator desc=fConsumers.begin();
687 while (desc!=fConsumers.end()) {
689 fConsumers.erase(desc);
690 desc=fConsumers.begin();
695 int AliHLTDataBuffer::FindConsumer(const AliHLTComponent* pConsumer, int bAllLists)
697 // see header file for function documentation
698 AliHLTConsumerDescriptorPList::iterator desc=fConsumers.begin();
699 while (desc!=fConsumers.end()) {
700 if ((*desc)->GetComponent()==pConsumer)
704 if (bAllLists==0) return 0;
706 desc=fActiveConsumers.begin();
707 while (desc!=fActiveConsumers.end()) {
708 if ((*desc)->GetComponent()==pConsumer)
712 desc=fReleasedConsumers.begin();
713 while (desc!=fReleasedConsumers.end()) {
714 if ((*desc)->GetComponent()==pConsumer)
721 AliHLTDataBuffer::AliHLTRawBuffer::AliHLTRawBuffer(AliHLTUInt32_t size)
725 , fPtr(static_cast<AliHLTUInt8_t*>(malloc(size)))
728 // see header file for class documentation
730 // refer to README to build package
732 // visit http://web.ift.uib.no/~kjeks/doc/alice-hlt
739 AliHLTDataBuffer::AliHLTRawBuffer::AliHLTRawBuffer(AliHLTUInt32_t size, AliHLTUInt8_t* buffer)
742 , fExternalPtr(buffer)
746 // see header file for class documentation
749 AliHLTDataBuffer::AliHLTRawBuffer::~AliHLTRawBuffer()
751 // see header file for class documentation
752 if (fExternalPtr==NULL && fPtr) {
760 int AliHLTDataBuffer::AliHLTRawBuffer::operator==(void* ptr) const
762 // see header file for function documentation
763 return fPtr == static_cast<AliHLTUInt8_t*>(ptr);
766 int AliHLTDataBuffer::AliHLTRawBuffer::operator<(void* ptr) const
768 // see header file for function documentation
769 int iResult=fPtr < static_cast<AliHLTUInt8_t*>(ptr);
770 //printf("%p: %p <= %p (%d)\n", this, fPtr, ptr, iResult);
774 int AliHLTDataBuffer::AliHLTRawBuffer::operator<=(void* ptr) const
776 // see header file for function documentation
777 int iResult=fPtr <= static_cast<AliHLTUInt8_t*>(ptr);
778 //printf("%p: %p <= %p (%d)\n", this, fPtr, ptr, iResult);
782 int AliHLTDataBuffer::AliHLTRawBuffer::operator>(void* ptr) const
784 // see header file for function documentation
785 int iResult=fPtr+fSize > static_cast<AliHLTUInt8_t*>(ptr);
786 //printf("%p: %p + %d > %p (%d)\n", this, fPtr, fSize, ptr, iResult);
790 int AliHLTDataBuffer::AliHLTRawBuffer::operator-(void* ptr) const
792 // see header file for function documentation
793 return static_cast<int>(static_cast<AliHLTUInt8_t*>(ptr)-fPtr);
796 int AliHLTDataBuffer::AliHLTRawBuffer::operator<(const AliHLTRawBuffer& op) const
798 // see header file for function documentation
799 return (fPtr+fSize < op.fPtr);
802 int AliHLTDataBuffer::AliHLTRawBuffer::operator<=(const AliHLTRawBuffer& op) const
804 // see header file for function documentation
805 return (fPtr+fSize <= op.fPtr);
808 int AliHLTDataBuffer::AliHLTRawBuffer::operator>(const AliHLTRawBuffer& op) const
810 // see header file for function documentation
811 return (fPtr >= op.fPtr+op.fSize);
814 AliHLTUInt8_t* AliHLTDataBuffer::AliHLTRawBuffer::UseBuffer(AliHLTUInt32_t size)
816 // mark a portion of the buffer as used
817 if (fTotalSize>=size) {
819 fLastEventCount=AliHLTDataBuffer::fgEventCount;
820 // only return pointer if there is a portion of the buffer used
821 if (size>0) return fPtr;
826 AliHLTDataBuffer::AliHLTRawBuffer* AliHLTDataBuffer::AliHLTRawBuffer::Split(AliHLTUInt32_t size)
828 // split a buffer at specified size
829 // only possible for buffers with external memory
830 if (fTotalSize>size &&
831 (fSize==0 || fSize<=size) && // used size must fit into the first part
832 fExternalPtr!=NULL) {
833 AliHLTRawBuffer* part2=new AliHLTRawBuffer(fTotalSize-size, fPtr+size);
842 int AliHLTDataBuffer::AliHLTRawBuffer::CheckSize(AliHLTUInt32_t size) const
844 // see header file for function documentation
845 if (fTotalSize<size) return 0;
847 if (fLastEventCount+1<AliHLTDataBuffer::fgEventCount) {
848 adjust=AliHLTDataBuffer::fgEventCount-fLastEventCount;
850 return (adjust>2) || ((fTotalSize-size)<(fgMargin<<adjust));
853 int AliHLTDataBuffer::AliHLTRawBuffer::Reset()
855 // see header file for function documentation
860 int AliHLTDataBuffer::AliHLTRawBuffer::WritePattern(const char* pattern, int size)
862 // see header file for function documentation
864 if (pattern!=NULL && size>0) {
865 if (fSize+size<=fTotalSize) {
866 memcpy(((char*)fPtr)+fSize, pattern, size);
875 int AliHLTDataBuffer::AliHLTRawBuffer::CheckPattern(const char* pattern, int size) const
877 // see header file for function documentation
879 if (pattern!=NULL && size>0) {
880 if (fSize+size<=fTotalSize) {
881 iResult=memcmp(((char*)fPtr)+fSize, pattern, size)!=0;
889 int AliHLTDataBuffer::AliHLTRawBuffer::Merge(const AliHLTDataBuffer::AliHLTRawBuffer& neighbor)
891 // Merge buffer with neighboring buffer.
892 // Only possible if the buffers are consecutive with out any gap.
894 if (!fExternalPtr || !neighbor.fExternalPtr) return -EPERM;
896 if (neighbor.fTotalSize==0 &&
897 fPtr < neighbor.fPtr &&
898 fPtr+fTotalSize > neighbor.fPtr) {
899 // special case for a buffer of zero size embedded into this buffer
904 neighbor.fPtr < fPtr &&
905 neighbor.fPtr+neighbor.fTotalSize > fPtr) {
906 // special case for this buffer of size zero embedded into another buffer
909 fTotalSize+=neighbor.fTotalSize;
913 if (fPtr+fTotalSize == neighbor.fPtr) {
914 fTotalSize+=neighbor.fTotalSize;
918 if (fPtr == neighbor.fPtr+neighbor.fTotalSize) {
921 fTotalSize+=neighbor.fTotalSize;
928 void AliHLTDataBuffer::AliHLTRawBuffer::Print(const char* option) const
930 /// print buffer information
931 if (strcmp(option, "min")!=0) {
932 cout << "************* AliHLTRawBuffer status ***********" << endl;
934 printf(" %p: buffer %p%s size %d used %d\n", this, fPtr, fExternalPtr?" (external)":"", fTotalSize, fSize); fflush(stdout);
937 AliHLTDataBuffer::AliHLTRawPage::AliHLTRawPage(AliHLTUInt32_t pagesize)
939 , fPtr(static_cast<AliHLTUInt8_t*>(malloc(pagesize)))
945 fFreeBuffers.push_back(new AliHLTRawBuffer(fSize, fPtr));
951 AliHLTDataBuffer::AliHLTRawPage::~AliHLTRawPage()
955 // do not free if the resources have not been completely freed
956 HLTError("memory mismatch: not all allocated intances have been released");
958 if (IsFragmented()) {
959 HLTWarning("page still fragmented");
961 AliHLTRawBufferPList::iterator element=fFreeBuffers.begin();
962 while (element!=fFreeBuffers.end()) {
963 if (*element) delete *element;
964 element=fFreeBuffers.erase(element);
974 AliHLTDataBuffer::AliHLTRawBuffer* AliHLTDataBuffer::AliHLTRawPage::Alloc(AliHLTUInt32_t size)
976 /// alloc a buffer of specified size
977 if (fFreeBuffers.size()==0) return NULL;
979 for (AliHLTRawBufferPList::iterator iter=fFreeBuffers.begin();
980 iter!=fFreeBuffers.end();
982 if ((*iter)->GetTotalSize()==size) {
983 AliHLTRawBuffer* thisbuffer=*iter;
984 fFreeBuffers.erase(iter);
985 fUsedBuffers.push_back(thisbuffer);
987 } else if ((*iter)->GetTotalSize()>size) {
988 AliHLTRawBuffer* thisbuffer=*iter;
989 AliHLTRawBuffer* newbuffer=thisbuffer->Split(size);
992 fUsedBuffers.push_back(thisbuffer);
995 HLTWarning("failed to alloc raw buffer: cannot split raw buffer %p of size %d (used %d) at size %d", *iter, (*iter)->GetTotalSize(), (*iter)->GetUsedSize(), size);
1002 int AliHLTDataBuffer::AliHLTRawPage::Free(AliHLTRawBuffer* pBuffer)
1004 /// free a buffer and merge consecutive free buffers
1006 for (AliHLTRawBufferPList::iterator iter=fUsedBuffers.begin();
1007 iter!=fUsedBuffers.end() && iResult>=0;
1009 if ((*iter)==pBuffer) {
1010 fUsedBuffers.erase(iter);
1011 AliHLTRawBufferPList::iterator prev=fFreeBuffers.begin();
1012 for (; prev!=fFreeBuffers.end() && iResult>=0; prev++) {
1013 if ((*pBuffer)<(*(*prev)) ||
1014 ((*prev)->GetTotalSize()==0 && pBuffer->GetPointer()<=(*prev)->GetPointer() && (*prev)->GetPointer()<=pBuffer->GetPointer()+pBuffer->GetTotalSize())) {
1015 // check consecutive buffers
1016 if ((*(*prev)) == (pBuffer->GetPointer()+pBuffer->GetTotalSize()) ||
1017 ((*prev)->GetTotalSize()==0 && pBuffer->GetPointer()<=(*prev)->GetPointer() && (*prev)->GetPointer()<=pBuffer->GetPointer()+pBuffer->GetTotalSize())) {
1018 // the buffer to be released has a consecutive free buffer -> merge them
1019 if ((iResult=pBuffer->Merge(*(*prev)))>=0) {
1023 HLTError("failed to merge consecutive/overlapping buffers %p and %p", pBuffer, (*prev));
1029 fFreeBuffers.insert(prev, pBuffer);
1032 if ((*pBuffer)>(*(*prev)) ||
1033 (pBuffer->GetTotalSize()==0 && (*prev)->GetPointer()<=pBuffer->GetPointer() && pBuffer->GetPointer()<=(*prev)->GetPointer()+(*prev)->GetTotalSize())) {
1034 // check consecutive buffers
1035 if ((*pBuffer) == ((*prev)->GetPointer()+(*prev)->GetTotalSize())||
1036 (pBuffer->GetTotalSize()==0 && (*prev)->GetPointer()<=pBuffer->GetPointer() && pBuffer->GetPointer()<=(*prev)->GetPointer()+(*prev)->GetTotalSize())) {
1037 // the buffer to be released is consecutive to a free buffer -> merge them
1038 if ((iResult=pBuffer->Merge(*(*prev)))>=0) {
1039 AliHLTRawBufferPList::iterator succ=prev+1;
1042 // check if the buffer and the following one are consecutive
1043 if (succ!=fFreeBuffers.end() &&
1044 (*(*succ)) == (pBuffer->GetPointer()+pBuffer->GetTotalSize())) {
1045 if ((iResult=pBuffer->Merge(*(*succ)))>=0) {
1047 fFreeBuffers.erase(succ);
1055 if (prev==fFreeBuffers.end()) {
1056 fFreeBuffers.push_back(pBuffer);
1059 // merge consecutive free buffers
1060 prev=fFreeBuffers.begin();
1061 for (AliHLTRawBufferPList::iterator current=prev+1; current!=fFreeBuffers.end() && iResult>=0; ) {
1062 // check if the buffer is embedded into the previous one
1063 if ((*current)->GetTotalSize()==0 && (*prev)->GetPointer()<=(*current)->GetPointer() && (*current)->GetPointer()<(*prev)->GetPointer()+(*prev)->GetTotalSize()) {
1064 if ((iResult=(*prev)->Merge(*(*current)))>=0) {
1065 current=fFreeBuffers.erase(current);
1068 HLTError("failed to merge embedded zero length buffer into preceeding buffer");
1072 // check if the buffer is consecutive to the previous one
1073 if ((*(*current)) == ((*prev)->GetPointer()+(*prev)->GetTotalSize())) {
1074 if ((iResult=(*prev)->Merge(*(*current)))>=0) {
1075 current=fFreeBuffers.erase(current);
1078 HLTError("failed to merge consecutive free buffers");
1085 // buffer was part of this page
1089 // buffer not found in this page
1093 int AliHLTDataBuffer::AliHLTRawPage::SetSize(const AliHLTDataBuffer::AliHLTRawBuffer* pBuffer, AliHLTUInt32_t size)
1095 /// set the size of a raw buffer and release the remaining part
1097 for (AliHLTRawBufferPList::iterator iter=fUsedBuffers.begin();
1098 iter!=fUsedBuffers.end() && iResult>=0;
1100 if ((*iter)==pBuffer) { // buffer was part of this page
1101 if ((*iter)->GetTotalSize()==size) return 0;
1102 if ((*iter)->GetTotalSize()<size) {
1103 HLTError("%d exceeds total size of buffer %p (%d used %d)\n", size, *iter, (*iter)->GetTotalSize(), (*iter)->GetUsedSize());
1106 AliHLTDataBuffer::AliHLTRawBuffer* freespace=(*iter)->Split(size);
1108 fUsedBuffers.push_back(freespace);
1111 HLTWarning("failed to relase unused memory: cannot split raw buffer %p of size %d (used %d) at size %d", *iter, (*iter)->GetTotalSize(), (*iter)->GetUsedSize(), size);
1116 // buffer not found in this page
1120 bool AliHLTDataBuffer::AliHLTRawPage::HasBuffer(const AliHLTDataBuffer::AliHLTRawBuffer* pBuffer)
1122 /// check if the buffer is in this page
1123 for (AliHLTRawBufferPList::iterator iter=fUsedBuffers.begin();
1124 iter!=fUsedBuffers.end();
1126 if ((*iter)==pBuffer) { // buffer was part of this page
1130 // buffer not found in this page
1134 AliHLTUInt32_t AliHLTDataBuffer::AliHLTRawPage::Capacity() const
1136 /// get max available contiguous buffer
1137 AliHLTUInt32_t capacity=0;
1138 for (unsigned i=0; i<fFreeBuffers.size(); i++) {
1139 if (fFreeBuffers[i]->GetTotalSize()>capacity)
1140 capacity=fFreeBuffers[i]->GetTotalSize();
1145 void AliHLTDataBuffer::AliHLTRawPage::Print(const char* option)
1147 /// print page information
1148 if (strcmp(option, "global")==0) {
1149 cout << "number of global pages: " << fgGlobalPages.size() << endl;
1150 for (AliHLTRawPage* rawpage=NextPage(NULL);
1152 rawpage=NextPage(rawpage)) {
1157 cout << "************* AliHLTRawPage status ***********" << endl;
1158 cout << " instance " << this << endl;
1159 printf(" buffer %p size %d", fPtr, fSize);
1160 cout << " used buffers: " << fUsedBuffers.size() << endl;
1161 AliHLTRawBufferPList::iterator iter=fUsedBuffers.begin();
1162 for (; iter!=fUsedBuffers.end(); iter++) {
1163 cout << " "; (*iter)->Print("min");
1165 cout << " free buffers: " << fFreeBuffers.size() << endl;
1166 iter=fFreeBuffers.begin();
1167 for (; iter!=fFreeBuffers.end(); iter++) {
1168 cout << " "; (*iter)->Print("min");
1173 vector<AliHLTDataBuffer::AliHLTRawPage*> AliHLTDataBuffer::AliHLTRawPage::fgGlobalPages;
1175 AliHLTUInt32_t AliHLTDataBuffer::AliHLTRawPage::fgGlobalPageSize=30*1024*1024;
1177 AliHLTDataBuffer::AliHLTRawBuffer* AliHLTDataBuffer::AliHLTRawPage::GlobalAlloc(AliHLTUInt32_t size, int verbosity)
1179 // alloc a buffer of specified size from the global pages
1180 AliHLTDataBuffer::AliHLTRawBuffer* rawbuffer=NULL;
1181 vector<AliHLTDataBuffer::AliHLTRawPage*>::iterator page=fgGlobalPages.begin();
1183 for (page=fgGlobalPages.begin();page!=fgGlobalPages.end(); page++) {
1184 if ((rawbuffer=(*page)->Alloc(size))!=NULL) {
1186 log.Logging(kHLTLogInfo, "AliHLTDataBuffer::AliHLTRawPage::GlobalAlloc", "data buffer handling", "allocated raw buffer %p from page %p\n", rawbuffer, *page);
1187 rawbuffer->Print("min");
1193 AliHLTUInt32_t rawPageSize=fgGlobalPageSize;
1194 if (rawPageSize<size) {
1195 if (rawPageSize*10<size) {
1196 log.Logging(kHLTLogError, "AliHLTDataBuffer::AliHLTRawPage::GlobalAlloc", "data buffer handling", "refusing to allocate buffer of size %d", size);
1201 AliHLTDataBuffer::AliHLTRawPage* rawpage=new AliHLTDataBuffer::AliHLTRawPage(rawPageSize);
1203 log.Logging(kHLTLogError, "AliHLTDataBuffer::AliHLTRawPage::GlobalAlloc", "data buffer handling", "can not create raw page");
1207 // check is there is at least one unused page which can be replaced by the newly created one
1208 for (page=fgGlobalPages.begin(); page!=fgGlobalPages.end(); page++) {
1209 if ((*page)->IsUsed()) continue;
1211 fgGlobalPages.erase(page);
1212 break; // delete only one page to be replaced by the new page
1214 fgGlobalPages.push_back(rawpage);
1215 if ((rawbuffer=rawpage->Alloc(size))!=NULL) {
1217 log.Logging(kHLTLogInfo, "AliHLTDataBuffer::AliHLTRawPage::GlobalAlloc", "data buffer handling", "allocated raw buffer %p from page %p\n", rawbuffer, rawpage);
1218 rawbuffer->Print("min");
1226 AliHLTDataBuffer::AliHLTRawPage* AliHLTDataBuffer::AliHLTRawPage::FindPage(AliHLTDataBuffer::AliHLTRawBuffer* buffer)
1228 // find buffer in the global pages
1229 vector<AliHLTDataBuffer::AliHLTRawPage*>::iterator page=fgGlobalPages.begin();
1230 for (; page!=fgGlobalPages.end(); page++) {
1231 if ((*page)->HasBuffer(buffer)) {
1239 int AliHLTDataBuffer::AliHLTRawPage::GlobalClean()
1241 // cleanup the global pages */
1242 vector<AliHLTDataBuffer::AliHLTRawPage*>::iterator page=fgGlobalPages.begin();
1243 while (page!=fgGlobalPages.end()) {
1244 if (!(*page)->IsUsed()) {
1246 page=fgGlobalPages.erase(page);
1250 log.Logging(kHLTLogError, "AliHLTDataBuffer::AliHLTRawPage::GlobalClean", "data buffer handling", "HLT memory page still in use, skipping cleanup, potential memory leak");
1258 AliHLTDataBuffer::AliHLTRawPage* AliHLTDataBuffer::AliHLTRawPage::NextPage(const AliHLTDataBuffer::AliHLTRawPage* prev)
1260 // get next global page
1261 vector<AliHLTDataBuffer::AliHLTRawPage*>::iterator page=fgGlobalPages.begin();
1262 for (; page!=fgGlobalPages.end(); page++) {
1263 if (prev==NULL) return *page;
1264 if (*page!=prev) continue;
1265 if (++page!=fgGlobalPages.end()) return *page;
1271 void AliHLTDataBuffer::AliHLTDataSegment::Print(const char* /*option*/) const
1273 // print info for data segment
1274 cout << "AliHLTDataSegment " << this
1275 << " " << AliHLTComponent::DataType2Text(fDataType)
1276 << " " << hex << fSpecification << dec
1277 << " Ptr " << (void*)fPtr
1278 << " offset " << fSegmentOffset
1279 << " size " << fSegmentSize
1283 void AliHLTDataBuffer::AliHLTForwardedDataSegment::Print(const char* option) const
1285 // print info for data segment
1286 cout << "AliHLTForwardeDataSegment " << this << endl;
1287 cout << " my : "; AliHLTDataSegment::Print(option);
1288 cout << " parent: "; fParentSegment.Print(option);
1290 if (fParentTask) fParentTask->Print("");
1291 else cout << "nil" << endl;
1294 void AliHLTDataBuffer::Print(const char* option) const
1296 // print info for data buffer
1298 cout << "AliHLTDataBuffer " << this << endl;
1299 cout << " raw buffer " << fpBuffer << endl;
1302 fpBuffer->Print(option);
1305 cout << " total segments: " << GetNofSegments() << endl;
1306 cout << " data segments: " << fSegments.size() << endl;
1307 for (i=0; i<fSegments.size(); i++) {
1309 fSegments[i].Print(option);
1312 cout << " forwarded segments: " << fForwardedSegments.size() << endl;
1313 for (i=0; i<fForwardedSegments.size(); i++) {
1315 fForwardedSegments[i].Print(option);
1318 cout << " consumers: " << GetNofConsumers() << endl;
1319 for (i=0; i<fConsumers.size(); i++) {
1321 fConsumers[i]->Print(option);
1324 cout << " active consumers: " << GetNofActiveConsumers() << endl;
1325 for (i=0; i<fActiveConsumers.size(); i++) {
1327 fActiveConsumers[i]->Print(option);
1330 cout << " released consumers: " << fReleasedConsumers.size() << endl;
1331 for (i=0; i<fReleasedConsumers.size(); i++) {
1333 fReleasedConsumers[i]->Print(option);