if (fpBuffer) {
if (*fpBuffer==pTgt) {
AliHLTDataBuffer::AliHLTDataSegment segment;
+ AliHLTUInt32_t maxSize=0;
for (int i=0; i<iSize; i++) {
// This function has to model the behavior of PubSub
// For output blocks only the fOffset value is used, this must be the offset
segment.fSpecification=arrayBlockData[i].fSpecification;
fSegments.push_back(segment);
HLTDebug("set segment %s with size %d at offset %d", AliHLTComponent::DataType2Text(segment.fDataType).data(), segment.fSegmentSize, segment.fSegmentOffset);
+
+ // find the actual size of the data
+ if ((arrayBlockData[i].fOffset!=~(AliHLTUInt32_t)0) &&
+ arrayBlockData[i].fOffset+arrayBlockData[i].fSize>maxSize) {
+ maxSize=arrayBlockData[i].fOffset+arrayBlockData[i].fSize;
+ }
} else {
HLTError("block data specification %#d (%s) exceeds size of data buffer", i, AliHLTComponent::DataType2Text(arrayBlockData[i].fDataType).data());
HLTError("block offset=%d, block size=%d, buffer size=%d", arrayBlockData[i].fOffset, arrayBlockData[i].fSize, fpBuffer->GetUsedSize());
iResult=-ERANGE;
}
}
+ // to be enabled if unit test is ready
+ //iResult=SetRawBufferDataSize(fpBuffer, maxSize);
} else {
HLTError("this data buffer (%p) does not match the internal data buffer %p of raw buffer %p", pTgt, fpBuffer->GetPointer(), fpBuffer);
iResult=-EINVAL;
return pRawBuffer;
}
+int AliHLTDataBuffer::SetRawBufferDataSize(AliHLTRawBuffer* pBuffer, AliHLTUInt32_t size) const
+{
+ // see header file for function documentation
+ int iResult=0;
+ if (!pBuffer) return -EINVAL;
+ if (size>pBuffer->GetUsedSize()) {
+ HLTError("indicated data size %d exceeds data buffer %p (%d)", size, pBuffer->GetPointer(), pBuffer->GetUsedSize());
+ return -ENOSPC;
+ }
+ if (fgkSafetyPatternSize>0) {
+ if (pBuffer->CheckPattern(fgkSafetyPattern, fgkSafetyPatternSize)) {
+ HLTError("potential memory corruption: component has written beyond end of data buffer %p size %d", pBuffer->GetPointer(), pBuffer->GetUsedSize());
+ }
+ }
+ // shrink the buffer and write new pattern at the end
+ pBuffer->UseBuffer(size);
+ if (fgkSafetyPatternSize>0) {
+ pBuffer->WritePattern(fgkSafetyPattern, fgkSafetyPatternSize);
+ }
+ return iResult;
+}
+
int AliHLTDataBuffer::ReleaseRawBuffer(AliHLTRawBuffer* pBuffer)
{
// see header file for function documentation
{
// split a buffer at specified size
// only possible for buffers with external memory
- if (size>0 && fTotalSize>size &&
+ if (fTotalSize>size &&
(fSize==0 || fSize<size) &&
fExternalPtr!=NULL) {
AliHLTRawBuffer* part2=new AliHLTRawBuffer(fTotalSize-size, fPtr+size);
if (!fExternalPtr || !neighbor.fExternalPtr) return -EPERM;
+ if (neighbor.fTotalSize==0 &&
+ fPtr < neighbor.fPtr &&
+ fPtr+fTotalSize > neighbor.fPtr) {
+ // special case for a buffer of zero size embedded into this buffer
+ // nothing to do
+ return 0;
+ }
+ if (fTotalSize==0 &&
+ neighbor.fPtr < fPtr &&
+ neighbor.fPtr+neighbor.fTotalSize > fPtr) {
+ // special case for this buffer of size zero embedded into another buffer
+ fPtr=neighbor.fPtr;
+ fExternalPtr=fPtr;
+ fTotalSize+=neighbor.fTotalSize;
+ fSize=0;
+ return 0;
+ }
if (fPtr+fTotalSize == neighbor.fPtr) {
fTotalSize+=neighbor.fTotalSize;
fSize=0;
AliHLTDataBuffer::AliHLTRawPage::~AliHLTRawPage()
{
// destructor
- if (fUsedBuffers.size()>0) {
+ if (IsUsed()) {
// do not free if the resources have not been completely freed
HLTError("memory mismatch: not all allocated intances have been released");
} else {
- if (fFreeBuffers.size()>0) {
+ if (IsFragmented()) {
HLTWarning("page still fragmented");
}
AliHLTRawBufferPList::iterator element=fFreeBuffers.begin();
for (AliHLTRawBufferPList::iterator iter=fFreeBuffers.begin();
iter!=fFreeBuffers.end();
iter++) {
- if ((*iter)->GetTotalSize()>=size) {
+ if ((*iter)->GetTotalSize()==size) {
+ AliHLTRawBuffer* thisbuffer=*iter;
+ fFreeBuffers.erase(iter);
+ fUsedBuffers.push_back(thisbuffer);
+ return thisbuffer;
+ } else if ((*iter)->GetTotalSize()>size) {
AliHLTRawBuffer* thisbuffer=*iter;
AliHLTRawBuffer* newbuffer=thisbuffer->Split(size);
if (newbuffer) {
fUsedBuffers.push_back(thisbuffer);
return thisbuffer;
} else {
- HLTWarning("failed to split raw buffer %p", *iter);
+ HLTWarning("failed to alloc raw buffer: cannot split raw buffer %p of size %d (used %d) at size %d", *iter, (*iter)->GetTotalSize(), (*iter)->GetUsedSize(), size);
}
}
}
iter!=fUsedBuffers.end() && iResult>=0;
iter++) {
if ((*iter)==pBuffer) {
+ if (false/*pBuffer->GetTotalSize()==0*/) {
+ HLTInfo("release zero lenght buffer %p %d (used %d)", pBuffer, pBuffer->GetTotalSize(), pBuffer->GetUsedSize());
+ pBuffer->Print("");
+ Print("");
+ }
fUsedBuffers.erase(iter);
AliHLTRawBufferPList::iterator prev=fFreeBuffers.begin();
for (; prev!=fFreeBuffers.end() && iResult>=0; prev++) {
- if ((*pBuffer)<(*(*prev))) {
+ if (false/*pBuffer->GetTotalSize()==0*/) {
+ HLTInfo("iterator buffer %p %d (used %d)", (*prev), (*prev)->GetTotalSize(), (*prev)->GetUsedSize());
+ (*prev)->Print("");
+ }
+ if ((*pBuffer)<(*(*prev)) ||
+ ((*prev)->GetTotalSize()==0 && pBuffer->GetPointer()<=(*prev)->GetPointer() && (*prev)->GetPointer()<=pBuffer->GetPointer()+pBuffer->GetTotalSize())) {
// check consecutive buffers
- if ((*(*prev)) == (pBuffer->GetPointer()+pBuffer->GetTotalSize())) {
+ if ((*(*prev)) == (pBuffer->GetPointer()+pBuffer->GetTotalSize()) ||
+ ((*prev)->GetTotalSize()==0 && pBuffer->GetPointer()<=(*prev)->GetPointer() && (*prev)->GetPointer()<=pBuffer->GetPointer()+pBuffer->GetTotalSize())) {
// the buffer to be released has a consecutive free buffer -> merge them
if ((iResult=pBuffer->Merge(*(*prev)))>=0) {
delete *prev;
*prev=pBuffer;
+ } else {
+ HLTError("failed to merge consecutive/overlapping buffers %p and %p", pBuffer, (*prev));
+ pBuffer->Print("");
+ (*prev)->Print("");
}
break;
}
fFreeBuffers.insert(prev, pBuffer);
break;
}
- if ((*pBuffer)>(*(*prev))) {
+ if ((*pBuffer)>(*(*prev)) ||
+ (pBuffer->GetTotalSize()==0 && (*prev)->GetPointer()<=pBuffer->GetPointer() && pBuffer->GetPointer()<=(*prev)->GetPointer()+(*prev)->GetTotalSize())) {
// check consecutive buffers
- if ((*pBuffer) == ((*prev)->GetPointer()+(*prev)->GetTotalSize())) {
+ if ((*pBuffer) == ((*prev)->GetPointer()+(*prev)->GetTotalSize())||
+ (pBuffer->GetTotalSize()==0 && (*prev)->GetPointer()<=pBuffer->GetPointer() && pBuffer->GetPointer()<=(*prev)->GetPointer()+(*prev)->GetTotalSize())) {
+ if (false/*pBuffer->GetTotalSize()==0*/) {
+ HLTInfo("merge with buffer %p %d (used %d)", (*prev), (*prev)->GetTotalSize(), (*prev)->GetUsedSize());
+ }
// the buffer to be released is consecutive to a free buffer -> merge them
if ((iResult=pBuffer->Merge(*(*prev)))>=0) {
AliHLTRawBufferPList::iterator succ=prev+1;
}
}
if (prev==fFreeBuffers.end()) {
+ if (false/*pBuffer->GetTotalSize()==0*/) {
+ HLTInfo("insert as free buffer");
+ Print("");
+ }
fFreeBuffers.push_back(pBuffer);
}
+
+ // merge consecutive free buffers
+ prev=fFreeBuffers.begin();
+ for (AliHLTRawBufferPList::iterator current=prev+1; current!=fFreeBuffers.end() && iResult>=0; ) {
+ // check if the buffer is embedded into the previous one
+ if ((*current)->GetTotalSize()==0 && (*prev)->GetPointer()<=(*current)->GetPointer() && (*current)->GetPointer()<(*prev)->GetPointer()+(*prev)->GetTotalSize()) {
+ if ((iResult=(*prev)->Merge(*(*current)))>=0) {
+ current=fFreeBuffers.erase(current);
+ continue;
+ } else {
+ HLTError("failed to merge embedded zero length buffer into preceeding buffer");
+ Print("");
+ }
+ }
+ // check if the buffer is consecutive to the previous one
+ if ((*(*current)) == ((*prev)->GetPointer()+(*prev)->GetTotalSize())) {
+ if ((iResult=(*prev)->Merge(*(*current)))>=0) {
+ current=fFreeBuffers.erase(current);
+ continue;
+ } else {
+ HLTError("failed to merge consecutive free buffers");
+ Print("");
+ }
+ }
+ prev=current++;
+ }
+
// buffer was part of this page
return 0;
}
iter!=fUsedBuffers.end() && iResult>=0;
iter++) {
if ((*iter)==pBuffer) { // buffer was part of this page
+ if ((*iter)->GetTotalSize()==size) return 0;
+ if ((*iter)->GetTotalSize()<size) {
+ HLTError("%d exceeds total size of buffer %p (%d used %d)\n", size, *iter, (*iter)->GetTotalSize(), (*iter)->GetUsedSize());
+ return -ENOSPC;
+ }
+ //HLTInfo("setting size %d of buffer %p (%d used %d)", size, *iter, (*iter)->GetTotalSize(), (*iter)->GetUsedSize());
AliHLTDataBuffer::AliHLTRawBuffer* freespace=(*iter)->Split(size);
+ //(*iter)->Print("");
if (freespace) {
+ //freespace->Print("");
fUsedBuffers.push_back(freespace);
Free(freespace);
} else {
- HLTWarning("failed to split raw buffer %p in order to relase unused memory", *iter);
+ HLTWarning("failed to relase unused memory: cannot split raw buffer %p of size %d (used %d) at size %d", *iter, (*iter)->GetTotalSize(), (*iter)->GetUsedSize(), size);
}
return 0;
}
return 1;
}
+bool AliHLTDataBuffer::AliHLTRawPage::HasBuffer(AliHLTDataBuffer::AliHLTRawBuffer* pBuffer)
+{
+ /// check if the buffer is in this page
+ for (AliHLTRawBufferPList::iterator iter=fUsedBuffers.begin();
+ iter!=fUsedBuffers.end();
+ iter++) {
+ if ((*iter)==pBuffer) { // buffer was part of this page
+ return true;
+ }
+ }
+ // buffer not found in this page
+ return false;
+}
+
AliHLTUInt32_t AliHLTDataBuffer::AliHLTRawPage::Capacity() const
{
/// get max available contiguous buffer
return capacity;
}
-void AliHLTDataBuffer::AliHLTRawPage::Print(const char* /*option*/)
+void AliHLTDataBuffer::AliHLTRawPage::Print(const char* option)
{
/// print page information
+ if (strcmp(option, "global")==0) {
+ cout << "number of global pages: " << fgGlobalPages.size() << endl;
+ for (AliHLTRawPage* rawpage=NextPage(NULL);
+ rawpage!=NULL;
+ rawpage=NextPage(rawpage)) {
+ rawpage->Print("");
+ }
+ return;
+ }
cout << "************* AliHLTRawPage status ***********" << endl;
+ cout << " instance " << this << endl;
printf(" buffer %p size %d", fPtr, fSize);
cout << " used buffers: " << fUsedBuffers.size() << endl;
AliHLTRawBufferPList::iterator iter=fUsedBuffers.begin();
cout << " "; (*iter)->Print("min");
}
}
+
+
+vector<AliHLTDataBuffer::AliHLTRawPage*> AliHLTDataBuffer::AliHLTRawPage::fgGlobalPages;
+
+AliHLTUInt32_t AliHLTDataBuffer::AliHLTRawPage::fgGlobalPageSize=1024*1024*10;
+
+AliHLTDataBuffer::AliHLTRawBuffer* AliHLTDataBuffer::AliHLTRawPage::GlobalAlloc(AliHLTUInt32_t size, int verbosity)
+{
+ // alloc a buffer of specified size from the global pages
+ AliHLTDataBuffer::AliHLTRawBuffer* rawbuffer=NULL;
+ vector<AliHLTDataBuffer::AliHLTRawPage*>::iterator page=fgGlobalPages.begin();
+ AliHLTLogging log;
+ for (page=fgGlobalPages.begin();page!=fgGlobalPages.end(); page++) {
+ if ((rawbuffer=(*page)->Alloc(size))!=NULL) {
+ if (verbosity>1) {
+ log.Logging(kHLTLogInfo, "AliHLTDataBuffer::AliHLTRawPage::GlobalAlloc", "data buffer handling", "allocated raw buffer %p from page %p\n", rawbuffer, *page);
+ rawbuffer->Print("min");
+ }
+ break;
+ }
+ }
+ if (!rawbuffer) {
+ AliHLTDataBuffer::AliHLTRawPage* rawpage=new AliHLTDataBuffer::AliHLTRawPage(fgGlobalPageSize);
+ if (!rawpage) {
+ log.Logging(kHLTLogError, "AliHLTDataBuffer::AliHLTRawPage::GlobalAlloc", "data buffer handling", "can not create raw page");
+ return NULL;
+ }
+ fgGlobalPages.push_back(rawpage);
+ if ((rawbuffer=rawpage->Alloc(size))!=NULL) {
+ if (verbosity>1) {
+ log.Logging(kHLTLogInfo, "AliHLTDataBuffer::AliHLTRawPage::GlobalAlloc", "data buffer handling", "allocated raw buffer %p from page %p\n", rawbuffer, rawpage);
+ rawbuffer->Print("min");
+ }
+ }
+ }
+
+ return rawbuffer;
+}
+
+AliHLTDataBuffer::AliHLTRawPage* AliHLTDataBuffer::AliHLTRawPage::FindPage(AliHLTDataBuffer::AliHLTRawBuffer* buffer)
+{
+ // find buffer in the global pages
+ vector<AliHLTDataBuffer::AliHLTRawPage*>::iterator page=fgGlobalPages.begin();
+ for (; page!=fgGlobalPages.end(); page++) {
+ if ((*page)->HasBuffer(buffer)) {
+ return *page;
+ }
+ }
+
+ return NULL;
+}
+
+int AliHLTDataBuffer::AliHLTRawPage::GlobalClean()
+{
+ // cleanup the global pages */
+ vector<AliHLTDataBuffer::AliHLTRawPage*>::iterator page=fgGlobalPages.begin();
+ while (page!=fgGlobalPages.end()) {
+ if (!(*page)->IsUsed()) {
+ delete *page;
+ page=fgGlobalPages.erase(page);
+ continue;
+ }
+ AliHLTLogging log;
+ log.Logging(kHLTLogError, "AliHLTDataBuffer::AliHLTRawPage::GlobalClean", "data buffer handling", "HLT memory page still in use, skipping cleanup, potential memory leak");
+
+ page++;
+ }
+
+ return 0;
+}
+
+AliHLTDataBuffer::AliHLTRawPage* AliHLTDataBuffer::AliHLTRawPage::NextPage(AliHLTDataBuffer::AliHLTRawPage* prev)
+{
+ // get next global page
+ vector<AliHLTDataBuffer::AliHLTRawPage*>::iterator page=fgGlobalPages.begin();
+ for (; page!=fgGlobalPages.end(); page++) {
+ if (prev==NULL) return *page;
+ if (*page!=prev) continue;
+ if (++page!=fgGlobalPages.end()) return *page;
+ break;
+ }
+ return NULL;
+}
int gVerbosity=0;
const int gPagesize=1024*1024*10;
-int testAliHLTRawPage();
+template<class T>
+int testLoop(vector<T>& descriptions, int nofLevels, int nofCycles);
int GetRandom(int min, int max);
-struct testProcessDescription {
+class AliHLTRandomBuffer {
+public:
+ AliHLTRandomBuffer(): fpBuffer(NULL), fBufferSize(0), fInstances() {}
+ AliHLTRandomBuffer(AliHLTUInt32_t size): fpBuffer(NULL), fBufferSize(0), fInstances() {Init(size);}
+ AliHLTRandomBuffer(const AliHLTRandomBuffer&): fpBuffer(NULL), fBufferSize(0), fInstances() {}
+ AliHLTRandomBuffer& operator=(const AliHLTRandomBuffer& src) {if (&src!=this) {fpBuffer=NULL; fBufferSize=0;} return *this;}
+ ~AliHLTRandomBuffer() {}
+
+ int Init(AliHLTUInt32_t size) {
+ if (fpBuffer) return -EINPROGRESS;
+
+ fpBuffer=new AliHLTUInt8_t[size];
+ if (!fpBuffer) return -ENOMEM;
+ fBufferSize=size;
+
+ int words=size/sizeof(AliHLTUInt32_t);
+ AliHLTUInt32_t* pTgt=reinterpret_cast<AliHLTUInt32_t*>(fpBuffer);
+ for (int i=0; i<words; i++, pTgt++) *pTgt=GetRandom(0, 0x7fffffff);
+ return 0;
+ }
+
+ struct AliHLTRandomBufferInstance {
+ int fId;
+ AliHLTUInt32_t fOffset;
+ AliHLTUInt32_t fSize;
+ };
+
+ int CreateInstance(AliHLTUInt32_t size) {
+ if (size>fBufferSize) return -ENOSPC;
+
+ AliHLTRandomBufferInstance instance;
+ instance.fId=fInstances.size()>0?fInstances.back().fId+1:0;
+ instance.fOffset=GetRandom(0, fBufferSize-size);
+ instance.fSize=size;
+ fInstances.push_back(instance);
+ return instance.fId;
+ }
+
+ int ReleaseInstance(int id) {
+ for (vector<AliHLTRandomBufferInstance>::iterator instance=fInstances.begin();
+ instance!=fInstances.end(); instance++) {
+ if (instance->fId==id) {
+ fInstances.erase(instance);
+ return 0;
+ }
+ }
+ return -ENOENT;
+ }
+
+ const AliHLTUInt8_t* GetBuffer(int id) {
+ for (vector<AliHLTRandomBufferInstance>::iterator instance=fInstances.begin();
+ instance!=fInstances.end(); instance++) {
+ if (instance->fId==id) {
+
+ return fpBuffer+instance->fOffset;
+ }
+ }
+ return NULL;
+ }
+
+ bool CompareBuffer(const AliHLTUInt8_t* buffer, AliHLTUInt32_t size, int id) {
+ if (!buffer) return false;
+ for (vector<AliHLTRandomBufferInstance>::iterator instance=fInstances.begin();
+ instance!=fInstances.end(); instance++) {
+ if (instance->fId==id) {
+ if (instance->fSize!=size) {
+ cout << "CompareBuffer ("<< instance->fId << "): size missmatch: " << size << ", expected " << instance->fSize << endl;
+ return false;
+ }
+ return memcmp(buffer, fpBuffer+instance->fOffset, size)==0;
+ }
+ }
+ return false;
+ }
+
+ void Print(const char* /*option*/) {
+ cout << "AliHLTRandomBufferInstance " << this << ": buffer " << (void*)fpBuffer << " size " << fBufferSize << endl;
+ if (!fpBuffer) return;
+
+ int words=fBufferSize/sizeof(AliHLTUInt32_t);
+ int lines=words/4;
+ AliHLTUInt32_t* pTgt=reinterpret_cast<AliHLTUInt32_t*>(fpBuffer);
+ for (int line=0; line<lines; line++) {
+ for (int i=0; i<4; i++, pTgt++) cout << hex << " 0x" << (*pTgt);
+ cout << endl;
+ }
+ if (words*sizeof(AliHLTUInt32_t)!=fBufferSize) cout << "some remaining bytes left due to alignment" << endl;
+ }
+
+private:
+ AliHLTUInt8_t* fpBuffer;
+ AliHLTUInt32_t fBufferSize;
+
+ vector<AliHLTRandomBufferInstance> fInstances;
+};
+
+AliHLTRandomBuffer gData(2*gPagesize);
+
+/**
+ * @class AliHLTTestRawPage
+ * Helper class for testing cycles of memory allocation, buffer modification, and
+ * cleanup with the AliHLTRawPage within the function testLoop.
+ */
+class AliHLTTestRawPage {
+public:
+ AliHLTTestRawPage() : fBuffer(NULL) {}
+ AliHLTTestRawPage(const AliHLTTestRawPage& src) : fBuffer(src.fBuffer) {}
+ AliHLTTestRawPage& operator=(const AliHLTTestRawPage& src) {
+ fBuffer=src.fBuffer;
+ return *this;
+ }
+
+ ~AliHLTTestRawPage() {}
+
+ bool IsAllocated() {return fBuffer!=NULL;}
+ bool Alloc(AliHLTUInt32_t size, int verbosity) {
+ fBuffer=AliHLTDataBuffer::AliHLTRawPage::GlobalAlloc(size);
+ if (!fBuffer) return false;
+ if (verbosity>1) {
+ printf("allocated raw buffer %p from page %p\n", fBuffer, AliHLTDataBuffer::AliHLTRawPage::FindPage(fBuffer));
+ fBuffer->Print("min");
+ }
+ return true;
+ }
+
+ AliHLTUInt8_t* GetPointer() {return fBuffer?fBuffer->GetPointer():NULL;}
+
+ bool SetSize(AliHLTUInt32_t size, int verbosity) {
+ AliHLTDataBuffer::AliHLTRawPage* rawpage=AliHLTDataBuffer::AliHLTRawPage::FindPage(fBuffer);
+ if (rawpage) {
+ if (rawpage->SetSize(fBuffer, size)==0) {
+ if (verbosity>1) {
+ cout << "setting size for raw buffer " << fBuffer << " of page " << rawpage << endl;
+ fBuffer->Print("min");
+ }
+ return true;
+ } else {
+ cerr << "failed to set size for raw buffer " << fBuffer << endl;
+ }
+ } else {
+ cerr << "can not find raw page for buffer " << fBuffer << endl;
+ }
+
+ return false;
+ }
+
+ bool Free(int verbosity) {
+ AliHLTDataBuffer::AliHLTRawPage* rawpage=AliHLTDataBuffer::AliHLTRawPage::FindPage(fBuffer);
+ if (rawpage) {
+ if (rawpage->Free(fBuffer)==0) {
+ if (verbosity>1) cout << "released raw buffer " << fBuffer << " from page " << rawpage << endl;
+ fBuffer=NULL;
+ return true;
+ } else {
+ cerr << "failed to release raw buffer " << fBuffer << endl;
+ }
+ } else {
+ cerr << "can not find raw page for buffer " << fBuffer << endl;
+ }
+ return false;
+ }
+
+ AliHLTUInt32_t GetTotalSize() {return fBuffer?fBuffer->GetTotalSize():0;}
+
+ bool GlobalClean(int verbosity) {
+ int nofPages=0;
+ for (AliHLTDataBuffer::AliHLTRawPage* rawpage=AliHLTDataBuffer::AliHLTRawPage::NextPage(NULL);
+ rawpage!=NULL;
+ rawpage=AliHLTDataBuffer::AliHLTRawPage::NextPage(rawpage)) {
+ nofPages++;
+ }
+ if (verbosity>=1) {
+ cout << "total number of pages: " << nofPages << endl;
+ }
+
+ AliHLTDataBuffer::AliHLTRawPage::GlobalClean();
+ return true;
+ }
+
+ void Print(const char* options) {
+ if (strcmp(options, "pages")==0 ||
+ strcmp(options, "global")==0) {
+ AliHLTDataBuffer::AliHLTRawPage::NextPage(NULL)->Print("global");
+ return;
+ }
+ }
+private:
+ AliHLTDataBuffer::AliHLTRawBuffer* fBuffer;
+};
+
+struct testRawPageDescription {
+ testRawPageDescription() : fLevel(0), fBufferSize(0), fDataId(-1), fTest() {};
int fLevel;
AliHLTUInt32_t fBufferSize;
- AliHLTDataBuffer::AliHLTRawBuffer* fRawBuffer;
+ int fDataId;
+ AliHLTTestRawPage fTest;
};
-int testAliHLTDataBuffer()
-{
- int iResult=0;
- if ((iResult=testAliHLTRawPage())<0) return iResult;
+class AliHLTDataBufferWrapper : public AliHLTDataBuffer {
+public:
+ int ResetDataBuffer() {
+ return AliHLTDataBuffer::ResetDataBuffer();
+ }
+};
+
+/**
+ * @class AliHLTTestDataBuffer
+ * Helper class for testing cycles of memory allocation, buffer modification, and
+ * cleanup with the AliHLTDataBuffer within the function testLoop.
+ */
+class AliHLTTestDataBuffer {
+public:
+ AliHLTTestDataBuffer() : fInstance(NULL), fBuffer(NULL), fSize(0) {}
+ AliHLTTestDataBuffer(const AliHLTTestDataBuffer&) : fInstance(NULL), fBuffer(NULL), fSize(0) {} ;
+ AliHLTTestDataBuffer& operator=(const AliHLTTestDataBuffer&) {
+ fInstance=NULL;
+ fBuffer=NULL;
+ fSize=0;
+ return *this;
+ }
+
+ bool IsAllocated() {return fBuffer!=NULL;}
+ bool Alloc(AliHLTUInt32_t size, int verbosity) {
+ if (!fInstance) {
+ fInstance=new AliHLTDataBufferWrapper;
+ if (!fInstance) return false;
+ fInstance->SetGlobalLoggingLevel(verbosity>1?kHLTLogAll:(AliHLTComponentLogSeverity)0x7c);
+ }
+
+ fBuffer=fInstance->GetTargetBuffer(size);
+ if (!fBuffer) return false;
+ fSize=size;
+ if (verbosity>1) {
+ cout << "allocated data buffer" << this << endl;
+ }
+ return true;
+ }
+
+ AliHLTUInt8_t* GetPointer() {return fBuffer;}
+
+ bool SetSize(AliHLTUInt32_t size, int verbosity) {
+ if (!fInstance) return false;
+ if (verbosity>1) {
+ cout << "setting size for data buffer " << this << ": allocated size " << fSize << " -> " << size << endl;
+ }
+ fSize=size;
+ return true;
+ }
+ bool Free(int verbosity) {
+ if (!fInstance) return false;
+ fSize=0;
+ if (fInstance->ResetDataBuffer()==0) {
+ if (verbosity>1) cout << "released data buffer " << this << endl;
+ fBuffer=NULL;
+ return true;
+ } else {
+ cerr << "failed to release raw buffer " << fBuffer << endl;
+ }
+
+ return false;
+ }
+
+ AliHLTUInt32_t GetTotalSize() {return fSize;}
+
+ bool GlobalClean(int verbosity) {
+ if (!fInstance) return false;
+ if (verbosity>=1) {
+ fInstance->PrintStatistics();
+ }
+ delete fInstance;
+ return true;
+ }
+
+ void Print(const char* options) {
+ if (strcmp(options, "pages")==0 ||
+ strcmp(options, "global")==0) {
+ fInstance->PrintStatistics();
+ }
+ }
+private:
+ AliHLTDataBufferWrapper* fInstance;
+ AliHLTUInt8_t* fBuffer;
+ AliHLTUInt32_t fSize;
+};
+
+struct testDataBufferDescription {
+ testDataBufferDescription() : fLevel(0), fBufferSize(0), fDataId(-1), fTest() {};
+ int fLevel;
+ AliHLTUInt32_t fBufferSize;
+ int fDataId;
+ AliHLTTestDataBuffer fTest;
+};
+
+template<class T,class S>
+int CopyDescription(vector<T>& target, vector<S>& source)
+{
+ target.clear();
+ for (unsigned i=0; i<source.size(); i++) {
+ T clone;
+ clone.fLevel=source[i].fLevel;
+ clone.fBufferSize=source[i].fBufferSize;
+ clone.fDataId=source[i].fDataId;
+ target.push_back(clone);
+ }
return 0;
}
-int fillTestSample(int levels, int processes, vector<testProcessDescription>& descriptions)
+template<class T>
+int fillTestSample(int levels, int processes, vector<T>& descriptions)
{
int availableProcesses=processes;
for (int level=0; level<levels; level++) {
int levelprocesses=GetRandom(availableProcesses/2, availableProcesses-(levels-level));
for (int process=0; process<levelprocesses; process++) {
- testProcessDescription desc;
+ T desc;
desc.fLevel=level;
- desc.fBufferSize=GetRandom(1024, 3*gPagesize/processes);
- desc.fRawBuffer=NULL;
+ desc.fBufferSize=GetRandom(1024, gPagesize/levelprocesses);
+ //desc.fTest.Clear();
descriptions.push_back(desc);
}
availableProcesses-=levelprocesses;
return 0;
}
-int allocateBuffers(vector<testProcessDescription>& descriptions,
- vector<AliHLTDataBuffer::AliHLTRawPage*>& pages,
+template<class T>
+int allocateBuffers(vector<T>& descriptions,
int level=-1)
{
for (unsigned i=0; i<descriptions.size(); i++) {
if (level>=0 && descriptions[i].fLevel<level) continue;
if (level>=0 && descriptions[i].fLevel>level) break;
- if (descriptions[i].fRawBuffer) {
+ if (descriptions[i].fTest.IsAllocated()) {
cerr << "warning: buffer already allocated" << endl;
continue;
}
- vector<AliHLTDataBuffer::AliHLTRawPage*>::iterator page=pages.begin();
- for (page=pages.begin();page!=pages.end(); page++) {
- if ((descriptions[i].fRawBuffer=(*page)->Alloc(GetRandom(descriptions[i].fBufferSize/2, descriptions[i].fBufferSize)))!=NULL) {
- if (gVerbosity>1) {
- printf("allocated raw buffer %p from page %p\n", descriptions[i].fRawBuffer, *page);
- descriptions[i].fRawBuffer->Print("min");
- }
- break;
- }
- }
- if (!descriptions[i].fRawBuffer) {
- AliHLTDataBuffer::AliHLTRawPage* rawpage=new AliHLTDataBuffer::AliHLTRawPage(gPagesize);
- if (!rawpage) {
- cerr << "can not create raw page" << endl;
- return -ENOMEM;
- }
- pages.push_back(rawpage);
- if ((descriptions[i].fRawBuffer=rawpage->Alloc(GetRandom(descriptions[i].fBufferSize/2, descriptions[i].fBufferSize)))!=NULL) {
- if (gVerbosity>1) {
- printf("allocated raw buffer %p from page %p\n", descriptions[i].fRawBuffer, rawpage);
- descriptions[i].fRawBuffer->Print("min");
- }
- }
+ if (descriptions[i].fTest.Alloc(GetRandom(descriptions[i].fBufferSize/2, descriptions[i].fBufferSize), gVerbosity)) {
}
- if (!descriptions[i].fRawBuffer) {
+
+ if (!descriptions[i].fTest.IsAllocated()) {
cerr << "failed to allocate buffer for process " << i << endl;
return -EFAULT;
}
return 0;
}
-int setBufferSizes(vector<testProcessDescription>& descriptions,
- vector<AliHLTDataBuffer::AliHLTRawPage*>& pages,
+template<class T>
+int setBufferSizes(vector<T>& descriptions,
int level=-1)
{
// set buffer size for all processes of the specified level
// buffer size is chosen randomly between 0 and the allocated size
vector<unsigned> positions;
for (unsigned i=0; i<descriptions.size(); i++) {
- if (!descriptions[i].fRawBuffer) continue;
+ if (!descriptions[i].fTest.IsAllocated()) continue;
if (level>=0 && descriptions[i].fLevel<level) continue;
if (level>=0 && descriptions[i].fLevel>level) break;
positions.push_back(i);
random_shuffle(positions.begin(), positions.end());
for (vector<unsigned>::iterator position=positions.begin();
position!=positions.end(); position++) {
- vector<AliHLTDataBuffer::AliHLTRawPage*>::iterator page=pages.begin();
- for (; page!=pages.end(); page++) {
- if ((*page)->SetSize(descriptions[*position].fRawBuffer, GetRandom(0, descriptions[*position].fBufferSize))==0) {
- if (gVerbosity>1) {
- cout << "setting size for raw buffer " << descriptions[*position].fRawBuffer << " of page " << *page << ", process level " << descriptions[*position].fLevel << endl;
- descriptions[*position].fRawBuffer->Print("min");
- }
- break;
- }
+ AliHLTUInt32_t datasize=GetRandom(0, descriptions[*position].fTest.GetTotalSize());
+ descriptions[*position].fDataId=gData.CreateInstance(datasize);
+ const AliHLTUInt8_t* pSrc=gData.GetBuffer(descriptions[*position].fDataId);
+ if (!pSrc) {
+ cout << "error creating data instance" << endl;
+ return -1;
}
- if (page==pages.end()) {
- cerr << "failed to set size for raw buffer " << descriptions[*position].fRawBuffer << "of process " << *position << endl;
- return -EFAULT;
+ AliHLTUInt8_t* pTgt=descriptions[*position].fTest.GetPointer();
+ if (!pTgt) {
+ cout << "error target buffer" << endl;
+ return -1;
+ }
+ memcpy(pTgt, pSrc, datasize);
+ if (descriptions[*position].fTest.SetSize(datasize, gVerbosity)) {
+ if (gVerbosity>1) cout << " process " << *position << " level " << descriptions[*position].fLevel << endl;
+ } else {
+ cout << "failed allocation for process " << *position << " level " << descriptions[*position].fLevel << endl;
+ return -1;
}
}
return 0;
}
-
-int releaseBuffers(vector<testProcessDescription>& descriptions,
- vector<AliHLTDataBuffer::AliHLTRawPage*>& pages,
+
+template<class T>
+int releaseBuffers(vector<T>& descriptions,
int level=-1)
{
// find the processes to be releases according to the specified level
// shuffle the processes and then release in this random order
vector<unsigned> positions;
for (unsigned i=0; i<descriptions.size(); i++) {
- if (!descriptions[i].fRawBuffer) continue;
+ if (!descriptions[i].fTest.IsAllocated()) continue;
if (level>=0 && descriptions[i].fLevel<level) continue;
if (level>=0 && descriptions[i].fLevel>level) break;
positions.push_back(i);
random_shuffle(positions.begin(), positions.end());
for (vector<unsigned>::iterator position=positions.begin();
position!=positions.end(); position++) {
- vector<AliHLTDataBuffer::AliHLTRawPage*>::iterator page=pages.begin();
- for (; page!=pages.end(); page++) {
- if ((*page)->Free(descriptions[*position].fRawBuffer)==0) {
- if (gVerbosity>1) cout << "released raw buffer " << descriptions[*position].fRawBuffer << " from page " << *page << ", process level " << descriptions[*position].fLevel << endl;
- descriptions[*position].fRawBuffer=NULL;
- break;
- }
+ if (!gData.CompareBuffer(descriptions[*position].fTest.GetPointer(), descriptions[*position].fTest.GetTotalSize(), descriptions[*position].fDataId)) {
+ cout << "data comparison failed for process " << *position << " level " << descriptions[*position].fLevel << endl;
+ return -1;
}
- if (page==pages.end()) {
- cerr << "failed to release raw buffer " << descriptions[*position].fRawBuffer << "of process " << *position << endl;
- return -EFAULT;
+ gData.ReleaseInstance(descriptions[*position].fDataId);
+ if (descriptions[*position].fTest.Free(gVerbosity)) {
+ if (gVerbosity>1) cout << " process " << *position << " level " << descriptions[*position].fLevel << endl;
+ } else {
+ cout << "failed deallocation for process " << *position << " level " << descriptions[*position].fLevel << endl;
+ return -1;
}
}
return 0;
}
-
-int testAliHLTRawPage()
+
+template<class T>
+int testLoop(vector<T>& descriptions, int nofLevels, int nofCycles)
{
+ // test the buffer allocation and deallocation from the AliHLTRawPage
+ //
+ // build a hierarchy of processes like in a real chain, in the first
+ // implementation it is only a linear tree
+ // the number of levels and processes are randomly chosen from a certain range
+ // processes are distributed among the levels such that there is a decreasing
+ // number of processes but at least one process
+ //
+ // The test cycle consists of the following steps
+ // - loop over all levels
+ // - allocation of a maximum buffer for every process of the current level
+ // - set the buffer size to be used for every process of the current level
+ // - release buffers of the previous level
+ // - check all created raw pages and throw error, if
+ // - not all buffers released
+ // - not all memory released
+ // - still fragmented
+ //
int iResult=0;
- int nofLevels=GetRandom(3,5);
- int nofProcesses=GetRandom(nofLevels,10);
- int nofCycles=GetRandom(5,10);
-
- vector<testProcessDescription> descriptions;
- if ((iResult=fillTestSample(nofLevels, nofProcesses, descriptions))<0) {
- cerr << "failed to fill test sample" << endl;
- return iResult;
- }
-
- if (gVerbosity>1) {
- for (unsigned i=0; i<descriptions.size(); i++) {
- cout << "process " << i << ": level " << descriptions[i].fLevel << " max size " << descriptions[i].fBufferSize << endl;
- }
- }
- vector<AliHLTDataBuffer::AliHLTRawPage*> pages;
- vector<AliHLTDataBuffer::AliHLTRawPage*>::iterator page=pages.end();
- for (int cycle=0; cycle<nofCycles; cycle++) {
- unsigned process=0;
+ AliHLTDataBuffer::AliHLTRawPage* rawpage=NULL;
+ for (int cycle=0; cycle<nofCycles && iResult>=0; cycle++) {
for (int level=0; level<=nofLevels; level++) {
// allocate buffers
if (level<nofLevels) {
- if ((iResult=allocateBuffers(descriptions, pages, level))<0) {
+ if ((iResult=allocateBuffers(descriptions, level))<0) {
cerr << "failed to allocate buffers" << endl;
return iResult;
}
}
- if (gVerbosity>0) cout << "level " << level << " - status of pages: " << pages.size() << endl;
- for (page=pages.begin();page!=pages.end(); page++) {
- if (gVerbosity>0) (*page)->Print("");
+ if (gVerbosity>1) {
+ cout << "finished allocation - level " << level << " ";
+ descriptions[0].fTest.Print("pages");
}
if (level<nofLevels) {
- if ((iResult=setBufferSizes(descriptions, pages, level))<0) {
+ if ((iResult=setBufferSizes(descriptions, level))<0) {
cerr << "failed setting buffers" << endl;
return iResult;
}
}
- if (gVerbosity>0) cout << "level " << level << " - status of pages: " << pages.size() << endl;
- for (page=pages.begin();page!=pages.end(); page++) {
- if (gVerbosity>0) (*page)->Print("");
+ if (gVerbosity>1) {
+ cout << "finished size modification - level " << level << " ";
+ descriptions[0].fTest.Print("pages");
}
if (level>0) {
- if ((iResult=releaseBuffers(descriptions, pages, level-1))<0) {
+ if ((iResult=releaseBuffers(descriptions, level-1))<0) {
cerr << "failed to release buffers" << endl;
return iResult;
}
}
}
- if (gVerbosity>0)
- cout << "status of released pages:" << endl;
- for (page=pages.begin();page!=pages.end(); page++) {
- if (gVerbosity>0) (*page)->Print("");
- if ((*page)->IsUsed()) {
- cerr << "page " << *page << " has used buffers" << endl;
- return -EFAULT;
- }
- if ((*page)->Size()!=(*page)->Capacity()) {
- cerr << "page " << *page << " not completely released" << endl;
- return -EFAULT;
- }
- if ((*page)->IsFragmented()) {
- cerr << "page " << *page << " is still fragmented" << endl;
- return -EFAULT;
+ if (gVerbosity>1) {
+ cout << "status of released pages: " << endl << " ";
+ descriptions[0].fTest.Print("pages");
+ }
+ for (rawpage=AliHLTDataBuffer::AliHLTRawPage::NextPage(NULL);
+ rawpage!=NULL;
+ rawpage=AliHLTDataBuffer::AliHLTRawPage::NextPage(rawpage)) {
+ if (rawpage->IsUsed()) {
+ cerr << "page " << rawpage << " has used buffers" << endl;
+ iResult=-EFAULT;
+ } else if (rawpage->Size()!=rawpage->Capacity()) {
+ cerr << "page " << rawpage << " not completely released" << endl;
+ iResult=-EFAULT;
+ } else if (rawpage->IsFragmented()) {
+ cerr << "page " << rawpage << " is still fragmented" << endl;
+ iResult=-EFAULT;
}
}
+ AliHLTDataBuffer::SetGlobalEventCount(cycle);
}
- return 0;
+ if (iResult<0) {
+ descriptions[0].fTest.Print("pages");
+ }
+
+ descriptions[0].fTest.GlobalClean(gVerbosity);
+
+ return iResult;
}
/**
return min+rand.Integer(max-min);
}
+int testAliHLTDataBuffer()
+{
+ int iResult=0;
+ const int iMaxLevels=10;
+ int nofLevels=GetRandom(3,iMaxLevels);
+ int nofProcesses=GetRandom(nofLevels,2*iMaxLevels);
+ int nofCycles=100;//GetRandom(1000,10000);
+
+ vector<testDataBufferDescription> descriptions;
+
+ if ((iResult=fillTestSample(nofLevels, nofProcesses, descriptions))<0) {
+ cerr << "failed to fill test sample" << endl;
+ return iResult;
+ }
+
+ // adjust to the level of the last entry
+ nofLevels=descriptions[descriptions.size()-1].fLevel;
+
+ if (gVerbosity>0) {
+ cout << " Test setup: " << nofLevels << " levels/ " << nofProcesses << " processes/ " << nofCycles << " cycle(s)" << endl;
+ cout << " Pagesize: " << gPagesize << endl;
+ for (unsigned i=0; i<descriptions.size(); i++) {
+ cout << " process " << i << ": level " << descriptions[i].fLevel << " max size " << descriptions[i].fBufferSize << endl;
+ }
+ }
+
+ if ((iResult=testLoop(descriptions, nofLevels, nofCycles))<0) return iResult;
+
+ if (gVerbosity>0) {
+ cout << "checking memory allocation using AliHLTRawPage directly ..." << endl;
+ }
+ vector<testRawPageDescription> rawpagedesc;
+ CopyDescription(rawpagedesc, descriptions);
+ if ((iResult=testLoop(rawpagedesc, nofLevels, nofCycles))<0) return iResult;
+
+ return 0;
+}
+
/////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////
//
int main(int /*argc*/, const char** /*argv*/)
{
int iResult=0;
- iResult=testAliHLTDataBuffer();
+ const int nofCycles=10;
+ for (int cycle=0; cycle<nofCycles && iResult>=0; cycle++) {
+ iResult=testAliHLTDataBuffer();
+ if (((cycle+1)*100/nofCycles)%10==0) {
+ cout << (cycle+1)*100/nofCycles << "% complete" << endl;
+ }
+ }
+
return iResult;
}
git://git.uio.no / u / mrichter / AliRoot.git / commitdiff