Mantid
Loading...
Searching...
No Matches
DiskBuffer.cpp
Go to the documentation of this file.
1// Mantid Repository : https://github.com/mantidproject/mantid
2//
3// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
4// NScD Oak Ridge National Laboratory, European Spallation Source,
5// Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
6// SPDX - License - Identifier: GPL - 3.0 +
9#include <sstream>
10#include <utility>
11
12using namespace Mantid::Kernel;
13
14namespace Mantid::Kernel {
15
16//----------------------------------------------------------------------------------------------
20 : m_writeBufferSize(50), m_writeBufferUsed(0), m_nObjectsToWrite(0), m_free(), m_free_bySize(m_free.get<1>()),
21 m_fileLength(0) {
22 m_free.clear();
23}
24
25//----------------------------------------------------------------------------------------------
32DiskBuffer::DiskBuffer(uint64_t m_writeBufferSize)
33 : m_writeBufferSize(m_writeBufferSize), m_writeBufferUsed(0), m_nObjectsToWrite(0), m_free(),
34 m_free_bySize(m_free.get<1>()), m_fileLength(0) {
35 m_free.clear();
36}
37
38//---------------------------------------------------------------------------------------------
48 if (item == nullptr)
49 return;
50 // if (!m_useWriteBuffer) return;
51
52 if (item->getBufPostion()) // already in the buffer and probably have changed
53 // its size in memory
54 {
55 // forget old memory size
56 std::unique_lock<std::mutex> uniqueLock(m_mutex);
58 // add new size
59 size_t newMemorySize = item->getDataMemorySize();
60 m_writeBufferUsed += newMemorySize;
61 uniqueLock.unlock();
62 item->setBufferSize(newMemorySize);
63 } else {
64 std::lock_guard<std::mutex> lock(m_mutex);
65 m_toWriteBuffer.push_front(item);
68 }
69
70 // Should we now write out the old data?
73}
74
75//---------------------------------------------------------------------------------------------
84 if (item == nullptr)
85 return;
86 // have it ever been in the buffer?
87 std::unique_lock<std::mutex> uniqueLock(m_mutex);
88 auto opt2it = item->getBufPostion();
89 if (opt2it) {
91 m_toWriteBuffer.erase(*opt2it);
92 } else {
93 return;
94 }
95
96 // indicate to the object that it is not stored in memory any more
97 item->clearBufferState();
98 uniqueLock.unlock();
99
100 // Mark the amount of space used on disk as free
101 if (item->wasSaved())
102 this->freeBlock(item->getFilePosition(), item->getFileSize());
103}
104
105//---------------------------------------------------------------------------------------------
110
111 std::lock_guard<std::mutex> _lock(m_mutex);
112 // Holder for any objects that you were NOT able to write.
113 std::list<ISaveable *> couldNotWrite;
114 size_t objectsNotWritten(0);
115 size_t memoryNotWritten(0);
116
117 // Iterate through the list
118 auto it = m_toWriteBuffer.begin();
119 auto it_end = m_toWriteBuffer.end();
120
121 ISaveable *obj = nullptr;
122
123 for (; it != it_end; ++it) {
124 obj = *it;
125 if (!obj->isBusy()) {
126 uint64_t NumObjEvents = obj->getTotalDataSize();
127 uint64_t fileIndexStart;
128 if (!obj->wasSaved()) {
129 fileIndexStart = this->allocate(NumObjEvents);
130 // Write to the disk; this will call the object specific save function;
131 // Prevent simultaneous file access (e.g. write while loading)
132 obj->saveAt(fileIndexStart, NumObjEvents);
133 } else {
134 uint64_t NumFileEvents = obj->getFileSize();
135 if (NumObjEvents != NumFileEvents) {
136 // Event list changed size. The MRU can tell us where it best fits
137 // now.
138 fileIndexStart = this->relocate(obj->getFilePosition(), NumFileEvents, NumObjEvents);
139 // Write to the disk; this will call the object specific save
140 // function;
141 obj->saveAt(fileIndexStart, NumObjEvents);
142 } else // despite object size have not been changed, it can be modified
143 // other way. In this case, the method which changed the data
144 // should set dataChanged ID
145 {
146 if (obj->isDataChanged()) {
147 fileIndexStart = obj->getFilePosition();
148 // Write to the disk; this will call the object specific save
149 // function;
150 obj->saveAt(fileIndexStart, NumObjEvents);
151 // this is questionable operation, which adjust file size in case
152 // when the file postions were allocated externaly
153 if (fileIndexStart + NumObjEvents > m_fileLength)
154 m_fileLength = fileIndexStart + NumObjEvents;
155 } else // just clean the object up -- it just occupies memory
156 obj->clearDataFromMemory();
157 }
158 }
159 // tell the object that it has been removed from the buffer
160 obj->clearBufferState();
161 } else // object busy
162 {
163 // The object is busy, can't write. Save it for later
164 couldNotWrite.emplace_back(obj);
165 // When a prefix or postfix operator is applied to a function argument,
166 // the value of the argument is
167 // NOT GUARANTEED to be incremented or decremented before it is passed to
168 // the function.
169 memoryNotWritten += obj->setBufferPosition(--couldNotWrite.end());
170 objectsNotWritten++;
171 }
172 }
173
174 // use last object to clear NeXus buffer and actually write data to HDD
175 if (obj) {
176 // NXS needs to flush the writes to file by closing and re-opening the data
177 // block.
178 // For speed, it is best to do this only once per write dump, using last
179 // object saved
180 obj->flushData();
181 }
182
183 // Exchange with the new map you built out of the not-written blocks.
184 m_toWriteBuffer.swap(couldNotWrite);
185 m_writeBufferUsed = memoryNotWritten;
186 m_nObjectsToWrite = objectsNotWritten;
187}
188
189//---------------------------------------------------------------------------------------------
193 // Now write everything out.
195}
196
197//---------------------------------------------------------------------------------------------
205void DiskBuffer::freeBlock(uint64_t const pos, uint64_t const size) {
206 if (size == 0 || size == std::numeric_limits<uint64_t>::max())
207 return;
208 std::lock_guard<std::mutex> lock(m_freeMutex);
209
210 // Make the block
211 FreeBlock newBlock(pos, size);
212 // Insert it
213 std::pair<freeSpace_t::iterator, bool> p = m_free.insert(newBlock);
214
215 // Failed insert? Should not happen since the map is NOT unique
216 // Or, if the map has only 1 item then it cannot do any merging. This solves a
217 // hanging bug in MacOS. Refs #3652
218 if (!p.second || m_free.size() <= 1) {
219 return;
220 }
221
222 // This is where we inserted
223 freeSpace_t::iterator it = p.first;
224 if (it != m_free.begin()) {
225 freeSpace_t::iterator it_before = it;
226 --it_before;
227 // There is a block before
228 FreeBlock block_before = *it_before;
229 if (FreeBlock::merge(block_before, newBlock)) {
230 // Change the map by replacing the old "before" block with the new merged
231 // one
232 m_free.replace(it_before, block_before);
233 // Remove the block we just inserted
234 m_free.erase(it);
235 // For cases where the new block was between two blocks.
236 newBlock = block_before;
237 it = it_before;
238 }
239 }
240 // Get an iterator to the block AFTER this one
241 freeSpace_t::iterator it_after = it;
242 ++it_after;
243 // There is a block after
244 if (it_after != m_free.end()) {
245 FreeBlock block_after = *it_after;
246 if (FreeBlock::merge(newBlock, block_after)) {
247 // Change the map by replacing the old "new" block with the new merged one
248 m_free.replace(it, newBlock);
249 // Remove the block that was after this one
250 m_free.erase(it_after);
251 }
252 }
253}
254
255//---------------------------------------------------------------------------------------------
261 std::lock_guard<std::mutex> lock(m_freeMutex);
262
263 freeSpace_t::iterator it = m_free.begin();
264 FreeBlock thisBlock;
265 thisBlock = *it;
266
267 while (it != m_free.end()) {
268 // Get iterator to the block after "it".
269 freeSpace_t::iterator it_after = it;
270 ++it_after;
271
272 if (FreeBlock::merge(thisBlock, *it_after)) {
273 // Change the map by replacing the old "before" block with the new merged
274 // one
275 m_free.replace(it, thisBlock);
276 // Remove the block that was merged out
277 m_free.erase(it_after);
278 // And stay at this iterator to
279 } else {
280 // Move on to the next block
281 ++it;
282 thisBlock = *it;
283 }
284 }
285}
286
287//---------------------------------------------------------------------------------------------
294uint64_t DiskBuffer::allocate(uint64_t const newSize) {
295 std::unique_lock<std::mutex> uniqueLock(m_freeMutex);
296
297 // Now, find the first available block of sufficient size.
298 freeSpace_bySize_t::iterator it;
299 bool putAtFileEnd = true;
300 if (m_free.size() > 0) {
301 // Unless there is nothing in the free space map
302 it = m_free_bySize.lower_bound(newSize);
303 putAtFileEnd = (it == m_free_bySize.end());
304 }
305
306 if (putAtFileEnd) {
307 // No block found
308 // Go to the end of the file.
309 uint64_t retVal = m_fileLength;
310 // And we assume the file will grow by this much.
311 m_fileLength += newSize;
312 // Will place the new block at the end of the file
313 return retVal;
314 } else {
315 // std::cout << "Block found for allocate " << newSize << '\n';
316 uint64_t foundPos = it->getFilePosition();
317 uint64_t foundSize = it->getSize();
318 // Remove the free block you found - it is no longer free
319 m_free_bySize.erase(it);
320 uniqueLock.unlock();
321 // Block was too large - free the bit of space after it.
322 if (foundSize > newSize) {
323 this->freeBlock(foundPos + newSize, foundSize - newSize);
324 }
325 return foundPos;
326 }
327}
328
329//---------------------------------------------------------------------------------------------
341uint64_t DiskBuffer::relocate(uint64_t const oldPos, uint64_t const oldSize, const uint64_t newSize) {
342 // std::cout << "Relocating " << oldPos << ", " << oldSize << ", " << newSize
343 // << '\n';
344 // First, release the space in the old block.
345 this->freeBlock(oldPos, oldSize);
346 return this->allocate(newSize);
347}
348
349//---------------------------------------------------------------------------------------------
352void DiskBuffer::getFreeSpaceVector(std::vector<uint64_t> &free) const {
353 free.reserve(m_free.size() * 2);
354 freeSpace_bySize_t::const_iterator it = m_free_bySize.begin();
355 freeSpace_bySize_t::const_iterator it_end = m_free_bySize.end();
356 for (; it != it_end; ++it) {
357 free.emplace_back(it->getFilePosition());
358 free.emplace_back(it->getSize());
359 }
360}
361
364void DiskBuffer::setFreeSpaceVector(std::vector<uint64_t> &free) {
365 m_free.clear();
366
367 if (free.size() % 2 != 0)
368 throw std::length_error("Free vector size is not a factor of 2.");
369
370 for (auto it = free.begin(); it != free.end(); it += 2) {
371 auto it_next = std::next(it);
372
373 if (*it == 0 && *it_next == 0) {
374 continue; // Not really a free space block!
375 }
376
377 FreeBlock newBlock(*it, *it_next);
378 m_free.insert(newBlock);
379 }
380}
381
383std::string DiskBuffer::getMemoryStr() const {
384 std::ostringstream mess;
385 mess << "Buffer: " << m_writeBufferUsed << " in " << m_nObjectsToWrite << " objects. ";
386 return mess.str();
387}
388
389} // namespace Mantid::Kernel
double obj
the value of the quadratic function
std::mutex m_freeMutex
Mutex for modifying the free space list.
Definition: DiskBuffer.h:145
void freeBlock(uint64_t const pos, uint64_t const size)
This method is called by this->relocate when object that has shrunk and so has left a bit of free spa...
Definition: DiskBuffer.cpp:205
uint64_t relocate(uint64_t const oldPos, uint64_t const oldSize, const uint64_t newSize)
This method is called by an ISaveable object that has outgrown its space allocated on file and needs ...
Definition: DiskBuffer.cpp:341
std::list< ISaveable * > m_toWriteBuffer
A forward list for the buffer of "toWrite" objects.
Definition: DiskBuffer.h:131
uint64_t allocate(uint64_t const newSize)
Allocate a block of the given size in a free spot in the file, or at the end of the file if there is ...
Definition: DiskBuffer.cpp:294
void toWrite(ISaveable *item)
Call this method when an object is ready to be written out to disk.
Definition: DiskBuffer.cpp:47
uint64_t m_fileLength
Length of the file. This is where new blocks that don't fit get placed.
Definition: DiskBuffer.h:149
size_t m_writeBufferUsed
Total amount of memory in the "toWrite" buffer.
Definition: DiskBuffer.h:127
void defragFreeBlocks()
Method that defrags free blocks by combining adjacent ones together NOTE: This is not necessary to ru...
Definition: DiskBuffer.cpp:260
void flushCache()
Flush out all the data in the memory; and writes out everything in the to-write cache.
Definition: DiskBuffer.cpp:192
size_t m_nObjectsToWrite
number of objects stored in to write buffer list
Definition: DiskBuffer.h:129
void getFreeSpaceVector(std::vector< uint64_t > &free) const
Returns a vector with two entries per free block: position and size.
Definition: DiskBuffer.cpp:352
void objectDeleted(ISaveable *item)
Call this method when an object that might be in the cache is getting deleted.
Definition: DiskBuffer.cpp:83
void setFreeSpaceVector(std::vector< uint64_t > &free)
Sets the free space map.
Definition: DiskBuffer.cpp:364
freeSpace_bySize_t & m_free_bySize
Index into m_free, but indexed by block size.
Definition: DiskBuffer.h:142
size_t m_writeBufferSize
Do we use the write buffer? Always now.
Definition: DiskBuffer.h:124
void writeOldObjects()
Method to write out the old objects that have been stored in the "toWrite" buffer.
Definition: DiskBuffer.cpp:109
std::string getMemoryStr() const
Definition: DiskBuffer.cpp:383
std::mutex m_mutex
Mutex for modifying the the toWrite buffer.
Definition: DiskBuffer.h:134
freeSpace_t m_free
Map of the free blocks in the file.
Definition: DiskBuffer.h:139
FreeBlock: a simple class that holds the position and size of block of free space in a file.
Definition: FreeBlock.h:22
static bool merge(FreeBlock &first, const FreeBlock &second)
Attempt to merge an adjacent block into this one.
Definition: FreeBlock.h:49
An interface for objects that can be cached or saved to disk.
Definition: ISaveable.h:28
void clearBufferState()
clears the state of the object, and indicate that it is not stored in buffer any more
Definition: ISaveable.cpp:79
void setBufferSize(size_t newSize)
Definition: ISaveable.h:166
virtual uint64_t getFilePosition() const
Definition: ISaveable.h:36
bool wasSaved() const
Definition: ISaveable.h:47
size_t setBufferPosition(std::list< ISaveable * >::iterator bufPosition)
sets the iterator pointing to the location of this object in the memory buffer to write later
Definition: ISaveable.cpp:69
boost::optional< std::list< ISaveable * >::iterator > & getBufPostion()
returns the iterator pointing to the position of this object within the memory to-write buffer
Definition: ISaveable.h:162
uint64_t getFileSize() const
Return the number of units this block occipies on file.
Definition: ISaveable.h:38
size_t getBufferSize() const
return the amount of memory, this object had when it was stored in buffer last time;
Definition: ISaveable.h:165
virtual size_t getDataMemorySize() const =0
the data size kept in memory