This section of the archives stores flipcode's complete Developer Toolbox collection, featuring a variety of mini-articles and source code contributions from our readers.

| Multi-Threaded Logging Library Submitted by

This is a high performance, multi-threaded logging library which I've made, which some people may find useful. The main features are:

> - Multiple threads and *multiple processes* can access the same log file at the same time. That means if you have a number of related programs, they can all use the same log file without problems.
> - The logger automatically prepends useful information to each line before it outputs it, things like a timestamp, process name and thread ID.
> - The logger doesn't let the log file get too big. Once it hits 256KB, it makes a backup and starts with a fresh file. This is useful because if you try and open a huge file in notepad, it can take days :) Also, it makes it easy to find entries that happened weeks or months ago (since the backups include the current date/time in the filename)
> - High performance. Actual writing to the file is done in a separate thread, so you don't have to wait for the I/O to complete before logging another line (it'll get added to a queue if you try and log a line and it's already writing a line) or continuing on with your program.

If you like this library, the don't forget to head on over to my home page:
http://www.codeka.com - it's a little empty now, but I'll be adding to it as I go. I'll post any updates to this library there.
Dean Harding.

Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger/internal.h] - (2,074 bytes)

    //-----------------------------------------------------------------------------
    // INTERNAL.H
    // By Dean Harding
    //
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    #pragma once

    //-----------------------------------------------------------------------------
    #include <windows.h>
    #include <tchar.h>
    #include <stdio.h>
    #include <stdlib.h>
    #include <assert.h>

    #ifdef _DEBUG
    #define ASSERT(x)		assert(x)
    #else
    #define ASSERT(x)		(x)
    #endif

    //-----------------------------------------------------------------------------
    #include "logger.h"
    #include "item.h"
    #include "queue.h"

    //-----------------------------------------------------------------------------
    // worker thread that does all the actual logging work.  See worker.cpp for
    // it's implementation.
    DWORD WINAPI LoggerWorkerThread( void *pData );

    //-----------------------------------------------------------------------------
    // This is the main queue that all the loggers add to and the worker thread
    // reads from.  I know, I know, it's a global variable - so kill me!
    extern CQueue<CLoggerItem>g_LoggerQueue;

    //-----------------------------------------------------------------------------
    // this is the class that actually implements the functionality of CLogger.
    // see loggerimpl.cpp for it's implementation.
    class CLoggerImpl : public CLogger
    {
    protected:
    	// current format, as set by SetFormat()
    	DWORD m_dwFormat;

    	// current process name, as set by SetName()
    	TCHAR *m_szProcName;

    	// current log file name, as set by SetLogFile()
    	TCHAR *m_szLogFile;
    public:
    	CLoggerImpl( TCHAR *szLogFile );
    	~CLoggerImpl();

    	void Release();

    	void SetFormat( DWORD dwFormat );
    	DWORD GetFormat();

    	void SetName( TCHAR *szName );
    	TCHAR *GetName();

    	void Print( TCHAR *szFormat, ... );
    };

    //-----------------------------------------------------------------------------
    // End of INTERNAL.H
    //-----------------------------------------------------------------------------

Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger/item.h] - (1,210 bytes)

    //-----------------------------------------------------------------------------
    // ITEM.H
    // By Dean Harding
    //
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    // This represents one item to be written to the log file.
    //-----------------------------------------------------------------------------
    #pragma once

    //-----------------------------------------------------------------------------
    // Note, anything not specified by dwFormat is ignored.
    class CLoggerItem
    {
    public:
    	TCHAR *szLine;
    	TCHAR *szLogFile;
    	TCHAR *szProcName;
    	DWORD dwThreadId;
    	FILETIME ftTimestamp;

    	DWORD dwFormat;

    	CLoggerItem( DWORD dwFormat, TCHAR *szLine, TCHAR *szLogFile,
    		TCHAR *szProcName, DWORD dwThreadId, FILETIME ftTimestamp )
    		: dwFormat( dwFormat ), szLine( szLine ), szLogFile( szLogFile )
    		, szProcName( szProcName ), dwThreadId( dwThreadId )
    		, ftTimestamp( ftTimestamp )
    	{}

    	~CLoggerItem()
    	{
    		delete[] szLine;
    	}
    };

    //-----------------------------------------------------------------------------
    // End of ITEM.H
    //-----------------------------------------------------------------------------

Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger/logger.cpp] - (1,458 bytes)

    //-----------------------------------------------------------------------------
    // LOGGER.CPP
    // By Dean Harding
    //
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    #include "internal.h"

    HANDLE g_hThread = NULL;

    //-----------------------------------------------------------------------------
    void CLogger::Initialize()
    {
    	if( g_hThread != NULL )
    	{
    		// initialize has already been called.
    		return;
    	}

    	DWORD dwThreadId;

    	// create the thread that will actually do the logging
    	g_hThread = CreateThread( NULL, 0, LoggerWorkerThread, NULL, 0, &dwThreadId );

    }

    //-----------------------------------------------------------------------------
    void CLogger::Shutdown()
    {
    	if( g_hThread == NULL )
    	{
    		// Initialize hasn't been called, or Shutdown has already been called.
    		return;
    	}

    	// add a NULL item to the queue.  That the thread's queue to quit.
    	g_LoggerQueue.AddItem( NULL );

    	WaitForSingleObject( g_hThread, INFINITE );
    	CloseHandle( g_hThread );

    	g_hThread = NULL;
    }

    //-----------------------------------------------------------------------------
    CLogger *CLogger::Attach( TCHAR *szLogFile )
    {
    	return new CLoggerImpl( szLogFile );
    }

    //-----------------------------------------------------------------------------
    // End of LOGGER.CPP
    //-----------------------------------------------------------------------------

Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger/logger.h] - (2,481 bytes)

    //-----------------------------------------------------------------------------
    // LOGGER.H
    // By Dean Harding
    //
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    #pragma once

    #include <tchar.h>

    //-----------------------------------------------------------------------------
    #ifdef LOGGER_EXPORTS
    #define LOGGER_API	__declspec(dllexport)
    #else
    #define LOGGER_API	__declspec(dllimport)
    #endif

    //-----------------------------------------------------------------------------
    // The logger can print a number of things along with the actual line it's
    // logging.  These are what it can also print

    #define LOG_TIMESTAMP			1	// the time the line was logged
    #define LOG_PROCNAME			2	// the name of the process that logged the
    									// line.  This can be set with
    									// CLogger::SetName()
    #define LOG_THREADID			4	// the thread ID that logged this line

    //-----------------------------------------------------------------------------
    // Maximum length of a line
    #define LOG_MAX_LINE			256

    //-----------------------------------------------------------------------------
    class LOGGER_API CLogger
    {
    public:
    	// Initialize the logger.  This must be called before any calls to Attach()
    	// (and it must have completed before any calls to Attach as well.
    	static void Initialize();

    	// Shutdown the logger.  Any loggers that have been created will become
    	// invalid!
    	static void Shutdown();

    	// Attach to the logger.  This just creates a new CLogger class for you to
    	// use.
    	static CLogger *Attach( TCHAR *szLogFile = _T("errors.log") );

    	// When you're finished logging, call this to destroy the CLogger class.
    	virtual void Release() = 0;

    	// Sets/Gets the current things that are printed along with actual line.
    	virtual void SetFormat( DWORD dwFormat ) = 0;
    	virtual DWORD GetFormat() = 0;

    	// set/get the name displayed by LOG_PROCNAME.  If NULL is given, then we
    	// use the name of the executable this process is running from (eg
    	// myapp.exe)
    	virtual void SetName( TCHAR *szName ) = 0;
    	virtual TCHAR *GetName() = 0;

    	// this is the big one!  This actually writes a line to the log file.
    	virtual void Print( TCHAR *szFormat, ... ) = 0;
    };

    //-----------------------------------------------------------------------------
    // End of LOGGER.H
    //-----------------------------------------------------------------------------

Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger/loggerimpl.cpp] - (2,944 bytes)

    //-----------------------------------------------------------------------------
    // LOGGERIMPL.CPP
    // By Dean Harding
    //
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    #include "internal.h"

    //-----------------------------------------------------------------------------
    CLoggerImpl::CLoggerImpl( TCHAR *szLogFile )
    : m_szLogFile( NULL ), m_szProcName( NULL ), m_dwFormat( 0 )
    {
    	m_szLogFile = new TCHAR[_tcslen( szLogFile ) + 1];
    	_tcscpy( m_szLogFile, szLogFile );

    	SetName( NULL );
    }
    //-----------------------------------------------------------------------------
    CLoggerImpl::~CLoggerImpl()
    {
    	delete[] m_szProcName;
    	delete[] m_szLogFile;
    }

    //-----------------------------------------------------------------------------
    void CLoggerImpl::Release()
    {
    	// wait for the queue to empty.  That's the only way we know that none of
    	// our data members are still being used.
    	g_LoggerQueue.WaitForEmpty();

    	delete this;
    }

    //-----------------------------------------------------------------------------
    void CLoggerImpl::SetFormat( DWORD dwFormat )
    {
    	m_dwFormat = dwFormat;
    }
    //-----------------------------------------------------------------------------
    DWORD CLoggerImpl::GetFormat()
    {
    	return m_dwFormat;
    }

    //-----------------------------------------------------------------------------
    void CLoggerImpl::SetName( TCHAR *szName )
    {
    	if( m_szProcName != NULL )
    	{
    		delete[] m_szProcName;
    	}

    	if( szName != NULL )
    	{
    		m_szProcName = new TCHAR[_tcslen(szName) + 1];
    		_tcscpy( m_szProcName, szName );
    	}
    	else
    	{
    		TCHAR szBuffer[MAX_PATH];
    		GetModuleFileName( NULL, szBuffer, MAX_PATH );

    		TCHAR *p = (TCHAR *)(szBuffer + _tcslen(szBuffer));
    		while( *(p - 1) != '\\' && p != szBuffer ) p--;

    		m_szProcName = new char[_tcslen(p) + 1];
    		_tcscpy( m_szProcName, p );
    	}
    }
    //-----------------------------------------------------------------------------
    TCHAR *CLoggerImpl::GetName()
    {
    	return m_szProcName;
    }

    //-----------------------------------------------------------------------------
    void CLoggerImpl::Print( TCHAR *szFormat, ... )
    {
    	FILETIME ftTimestamp = { 0 };
    	DWORD dwThreadId = 0;
    	TCHAR *szBuffer = new TCHAR[LOG_MAX_LINE];

    	va_list args;
    	va_start( args, szFormat );
    	_vstprintf( szBuffer, szFormat, args );
    	va_end( args );

    	if( (m_dwFormat & LOG_TIMESTAMP) != 0 )
    	{
    		GetSystemTimeAsFileTime( &ftTimestamp );
    	}

    	if( (m_dwFormat & LOG_THREADID) != 0 )
    	{
    		dwThreadId = GetCurrentThreadId();
    	}

    	CLoggerItem *pItem = new CLoggerItem( m_dwFormat, szBuffer, m_szLogFile, m_szProcName, dwThreadId, ftTimestamp );
    	g_LoggerQueue.AddItem( pItem );
    }

    //-----------------------------------------------------------------------------
    // End of LOGGERIMPL.CPP
    //-----------------------------------------------------------------------------

Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger/queue.h] - (2,613 bytes)

    //-----------------------------------------------------------------------------
    // QUEUE.H
    // By Dean Harding
    //
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    // I needed a fast queue system, where it's very fast *adding* to the *end* of
    // the queue, and *removing* from the beginning (i.e. FIFO).  I don't know of
    // any STL container that have those properties, so I made my own.
    //
    // It's just a doubly-linked list :)
    //
    // Also, this class is thread safe (a thread can add to the queue while another
    // is getting stuff from the top)  And finally, you can block on a GetNext()
    // call until data is added.  Note that this only works when there is one
    // *and only one* thread using GetNext().  That's OK - that's exactly the
    // functionality we want!
    //-----------------------------------------------------------------------------
    #pragma once

    //-----------------------------------------------------------------------------
    template< class T >
    class CQueue
    {
    protected:
    	struct SQueueData
    	{
    		T *pData;

    		SQueueData *pNext;
    		SQueueData *pPrev;
    	};

    	SQueueData *m_pTop;
    	SQueueData *m_pBottom;

    	// We just lock the whole queue when we want to add/remove to/from it,
    	// because that's nice and easy, and besides, the things we do on the queue
    	//should be simple enough that it won't really matter.
    	HANDLE m_hQueueLock;

    	// When we add an item queue, this even it fired to tell anything waiting
    	// in GetNext().
    	HANDLE m_hAddEvent;

    	// This is used by WaitForEmpty().  When GetNext() removes the last item
    	// from the queue, this is signalled.
    	HANDLE m_hEmptyEvent;
    public:
    	inline CQueue();
    	inline ~CQueue();

    	// Get the first item off the top of the queue.  If the queue is empty
    	// and bBlock is false, NULL is returned.  If the queue is empty and bBlock
    	// is true, then the function blocks until the queue is added to.
    	inline T* GetNext( bool bBlock = false );

    	// Add an item to the queue.  If there's a blocking call to GetNext() then
    	// that call will wake up and return this item.
    	inline void AddItem( T* pItem );

    	// returns true if the queue is empty.
    	inline bool IsEmpty();

    	// blocks until the queue is empty
    	inline void WaitForEmpty();
    };

    //-----------------------------------------------------------------------------
    #include "queue.inl"

    //-----------------------------------------------------------------------------
    // End of QUEUE.H
    //-----------------------------------------------------------------------------

Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger/worker.cpp] - (5,959 bytes)

    //-----------------------------------------------------------------------------
    // WORKER.CPP
    // By Dean Harding
    //
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    // This is the thread that actually does all the work.  It waits on the
    // queue for more items, whenever a new item is added, it writes that item
    // to a file!
    //-----------------------------------------------------------------------------
    #include "internal.h"

    //-----------------------------------------------------------------------------
    CQueue<CLoggerItem>g_LoggerQueue;

    //-----------------------------------------------------------------------------
    // While this lock is set, no threads from other processes will try and open
    // the file.  This is for things like when renaming a large file or such
    HANDLE g_hFileLock = NULL;
    const TCHAR g_szFileLock[] = _T("dhLoggerFileLock");

    //-----------------------------------------------------------------------------
    // renames an open file, and returns a handle to a new file of the same name.
    HANDLE RenameFile( HANDLE hFile, TCHAR *szFileName );

    //-----------------------------------------------------------------------------
    // opens a file.  If access is denied, keep trying
    HANDLE OpenFile( TCHAR *szFileName );

    //-----------------------------------------------------------------------------
    DWORD WINAPI LoggerWorkerThread( void *pData )
    {
    	TCHAR szBuffer[1024];
    	DWORD dwNumWritten;

    	g_hFileLock = CreateMutex( NULL, FALSE, g_szFileLock );

    	while( true )
    	{
    		CLoggerItem *pItem = g_LoggerQueue.GetNext( true );

    		// if it's a "NULL" item, then that's our queue to quit!
    		if( pItem == NULL )
    			break;

    		HANDLE hFile = OpenFile( pItem->szLogFile );

    		DWORD dwFileSize = GetFileSize( hFile, NULL );
    		if( dwFileSize > (256 * 1024) )
    		{
    			// if the file is bigger than 256KB, then rename it and make a new
    			// one
    			hFile = RenameFile( hFile, pItem->szLogFile );
    		}

    		// seek to the end of the file
    		SetFilePointer( hFile, 0, 0, FILE_END );

    		// if they want a timestamp, format that for them
    		if( (pItem->dwFormat & LOG_TIMESTAMP) != 0 )
    		{
    			FILETIME ftLocalTime;
    			SYSTEMTIME stLocalTime;

    			FileTimeToLocalFileTime( &pItem->ftTimestamp, &ftLocalTime );
    			FileTimeToSystemTime( &ftLocalTime, &stLocalTime );

    			_stprintf( szBuffer, _T("%2d/%02d/%04d %2d:%02d:%02d.%03d "),
    				stLocalTime.wDay, stLocalTime.wMonth, stLocalTime.wYear,
    				stLocalTime.wHour, stLocalTime.wMinute, stLocalTime.wSecond,
    				stLocalTime.wMilliseconds );

    			WriteFile( hFile, szBuffer, _tcslen( szBuffer ), &dwNumWritten, NULL );
    		}

    		// if they want a process name & thread id
    		if( (pItem->dwFormat & LOG_PROCNAME) != 0 && (pItem->dwFormat & LOG_THREADID) != 0 )
    		{
    			_stprintf( szBuffer, _T("(%s:%d) "), pItem->szProcName, pItem->dwThreadId );

    			WriteFile( hFile, szBuffer, _tcslen( szBuffer ), &dwNumWritten, NULL );
    		}
    		// if it's just the process name
    		else if( (pItem->dwFormat & LOG_PROCNAME) != 0 )
    		{
    			_stprintf( szBuffer, _T("(%s)"), pItem->szProcName );

    			WriteFile( hFile, szBuffer, _tcslen( szBuffer ), &dwNumWritten, NULL );
    		}
    		// if it's just the thread id
    		else if( (pItem->dwFormat & LOG_THREADID) != 0 )
    		{
    			_stprintf( szBuffer, _T("(%d)"), pItem->dwThreadId );

    			WriteFile( hFile, szBuffer, _tcslen( szBuffer ), &dwNumWritten, NULL );
    		}

    		// now write the actual line
    		WriteFile( hFile, pItem->szLine, _tcslen( pItem->szLine ), &dwNumWritten, NULL );

    		// and an end-of-line
    		WriteFile( hFile, _T("\r\n"), _tcslen( _T("\r\n") ), &dwNumWritten, NULL );

    		// and close the file!
    		CloseHandle( hFile );

    		delete pItem;
    	}

    	CloseHandle( g_hFileLock );

    	return 0;
    }

    //-----------------------------------------------------------------------------
    HANDLE RenameFile( HANDLE hFile, TCHAR *szFileName )
    {
    	ASSERT( WaitForSingleObject( g_hFileLock, INFINITE ) == WAIT_OBJECT_0 );

    	// now that we own the file lock, we can close the file, rename and make a
    	// new one!!

    	CloseHandle( hFile );

    	TCHAR szBuffer[MAX_PATH];
    	TCHAR szNewFile[MAX_PATH];
    	FILETIME ftCurrent, ftLocal;
    	SYSTEMTIME stLocal;

    	TCHAR *p = (TCHAR *)(szFileName + _tcslen(szFileName) );
    	while( *(p - 1) != '\\' && p != szFileName ) p--;

    	GetSystemTimeAsFileTime( &ftCurrent );
    	FileTimeToLocalFileTime( &ftCurrent, &ftLocal );
    	FileTimeToSystemTime( &ftLocal, &stLocal );

    	_stprintf( szBuffer, _T("%04d-%02d-%2d %02d-%02d-%02d %s"),
    		stLocal.wYear, stLocal.wMonth, stLocal.wDay,
    		stLocal.wHour, stLocal.wMinute, stLocal.wSecond,
    		p );

    	_tcsncpy( szNewFile, szFileName, (p - szFileName) );
    	p = szNewFile + (p - szFileName);
    	*p = '\0';

    	_tcscat( szNewFile, szBuffer );

    	MoveFile( szFileName, szNewFile );

    	ReleaseMutex( g_hFileLock );
    	return OpenFile( szFileName );
    }

    //-----------------------------------------------------------------------------
    HANDLE OpenFile( TCHAR *szFileName )
    {
    	// open the file, keep trying until it's not INVALID_HANDLE_VALUE
    	// There has to be a better way than this.  I mean, I want to be able
    	// to block the thread until the file is available again.  I don't know
    	// if there's any API for that...
    	HANDLE hFile = INVALID_HANDLE_VALUE;
    	while( hFile == INVALID_HANDLE_VALUE )
    	{
    		// we only try to open the file if we own the hFileLock mutex.
    		ASSERT( WaitForSingleObject( g_hFileLock, INFINITE ) == WAIT_OBJECT_0 );

    		hFile = CreateFile(
    			szFileName,
    			GENERIC_WRITE,
    			FILE_SHARE_READ,
    			NULL,
    			OPEN_ALWAYS,
    			FILE_ATTRIBUTE_NORMAL,
    			NULL );
    		Sleep( 0 );

    		ReleaseMutex( g_hFileLock );
    	}

    	return hFile;
    }

    //-----------------------------------------------------------------------------
    // End of WORKER.CPP
    //-----------------------------------------------------------------------------

Currently browsing [mlogger.zip] (76,329 bytes) - [logger/logger_test/logger_test.cpp] - (1,762 bytes)

    //-----------------------------------------------------------------------------
    // LOGGER_TEST.CPP
    // By Dean Harding
    //
    // Copyright (c) 2000/2001 Dean Harding
    //-----------------------------------------------------------------------------
    // This is just a little console application which creates a number of threads
    // and writes stuff to the log file.  It's just to test logger.dll
    //-----------------------------------------------------------------------------
    #include <windows.h>
    #include <stdio.h>
    #include <conio.h>
    #include <iostream.h>
    #include "..\logger\logger.h"

    //-----------------------------------------------------------------------------
    int main( int argc, char **argv )
    {
    	cout << "Hello World!!" << endl;

    	CLogger::Initialize();

    	CLogger *logger = CLogger::Attach( _T("log\\errors.log") );
    	if( logger == NULL )
    	{
    		cout << "Could not attack to logger." << endl;
    		return 1;
    	}

    	logger->SetFormat( LOG_PROCNAME | LOG_TIMESTAMP | LOG_THREADID );
    	DWORD dwProcessId = GetCurrentProcessId();
    	DWORD dwRun = 1;
    	while( !kbhit() )
    	{
    		cout << "Process: " << dwProcessId << ", Run: " << dwRun << endl;
    		logger->Print( "Process: %d, Run: %d", dwProcessId, dwRun );
    		dwRun ++;

    		// make it so the log file doesn't get too big, too fast...
    		// if you remove this, then expect a large lot of log files :)
    		//Sleep( 100 );
    	}

    	cout << "Flushing log file..." << endl;
    	logger->Release();
    	logger = NULL;

    	cout << "Finishing logging..." << endl;
    	CLogger::Shutdown();

    	getch();

    	return 0;
    }

    //-----------------------------------------------------------------------------
    // End of LOGGER_TEST.CPP
    //-----------------------------------------------------------------------------

The zip file viewer built into the Developer Toolbox made use of the zlib library, as well as the zlibdll source additions.

Copyright 1999-2008 (C) FLIPCODE.COM and/or the original content author(s). All rights reserved.
Please read our Terms, Conditions, and Privacy information.