Re: Memory Cleanup for Hashtables

Tech-Archive recommends: Fix windows errors by optimizing your registry

Could you post a short but complete program which demonstrates the
problem?

Hi Jon,

I am just writing the functions I am calling to read files & storing the
Data in to HashTables / Arraylist.

Method 1:

#region setupReaderThreads
/// <summary>
/// Setup the Multi Threaded environment to read the Log files
/// </summary>
private void setupReaderThreads()
{

int fileNameCount = fileNames.Count;

_fileName = new string[_arrayLength];
_logAnalyserBL = new LogAnalyserBL[_arrayLength];
_logAnalyserBLThread = new Thread[_arrayLength];

for(int i=0; i< _arrayLength; i++)
{
//fileNames is an arraylist

_fileName[i] = fileNames[i + _fileCounter].ToString();

if(! File.Exists(_fileName[i]))
{
return;
}
else
{
_logAnalyserBL[i] = new LogAnalyserBL(false);
_logAnalyserBL[i].FileName = _fileName[i];
}

_logAnalyserBLThread[i] = new Thread(new
ThreadStart(_logAnalyserBL[i].DoWork));
_logAnalyserBLThread[i].Name = "Log Reader " + i;

_logAnalyserBLThread[i].Start();

}

}

************************************************

Method 2:

#region SearchTimer_Tick
/// <summary>
/// SearchTimer_Tick
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void SearchTimer_Tick(object sender, System.EventArgs e)
{
try
{
SearchTimer.Enabled = false;
//
// Setting up the Multiple Threades to read the log files
//
if (_state == 0)
{
setupReaderThreads();
_state = 1;
}
//
// Checking if the file Reading is finished or it is still going on
//
else if (_state == 1)
{
Cursor = Cursors.WaitCursor;
bool flag = false;

StatusLabel.Text = _logAnalyserBL[0].statusBar;

for(int i=0; i< _arrayLength; i++)
{
if(!_logAnalyserBL[i].isRunning)
flag = true;
else
{
flag = false;
break;
}
}

if(flag)
_state = 2;

}
//
// Adding all the Logfiles Hash Table objects into the arraylist to
prepare the Result Nodes
//
else if (_state == 2)
{
StatusLabel.Text = string.Format("Preparing Results...", _logs.Count);

// Adding all the log file hashtable objects into the
Arraylist "_logs"
for(int i=0; i< _arrayLength; i++)
{
foreach (LogEntryNode n in _logAnalyserBL[i].results)
{
_logs.Add(n);
}

_logAnalyserBL[i] = null;
}

//Logic for Increment the counter to read 12 files at a time
if(_fileCounter < fileNames.Count - fileNames.Count%12)
{
_fileCounter += 12;

_state = 0;

if( _fileCounter == fileNames.Count - fileNames.Count%12)
{
_arrayLength = fileNames.Count%12;

if(_fileCounter == fileNames.Count)
{
_state = 3;
}
}
else if(_fileCounter == fileNames.Count)
{
_state = 3;
}
}
else
_state = 3;
}
//
// Reading the Log arraylist & Creating the corresponding Hashtables
//
else if (_state == 3)
{
_logMatcher = new LogMatcher();
_logMatcher.logs = _logs;

ThreadStart threadStart = new ThreadStart(_logMatcher.DoWork);
Thread logAnalysisThread = new Thread(threadStart);
logAnalysisThread.Name = "Log Analysis";
logAnalysisThread.Start();

_state = 4;
}
//
// Checking the thread is running or not
//
else if (_state == 4)
{
if (!_logMatcher.isRunning)
_state = 5;
else
StatusLabel.Text = _logMatcher.status;
}
//
// Updating the Status
//
else if (_state == 5)
{
this.StatusLabel.Text = "Converting to TreeView...";
_state = 6;
}
//
// Converting the resultant nodes into TreeView form
//
else if (_state == 6)
{
this.StatusLabel.Text = "Converting to TreeView...";
MetaDataTreeView.BeginUpdate();

MetaDataTreeView.Nodes.Add(new TreeNode("QUERY Nodes"));
MetaDataTreeView.Nodes.Add(new TreeNode("SAVE Nodes"));

int userCount = 1;

foreach (string key in _logMatcher.orderedResults)
{

ArrayList logEntryArrayList = (ArrayList) _logMatcher.results[key];

TreeNode tNode = null;
string startTime = string.Empty;
string endTime = string.Empty;
string timeTaken = string.Empty;

if (_logMatcher.methods.ContainsKey(key))
{
LogEntryNode logEntryNode = (LogEntryNode) _logMatcher.methods[key];

startTime = logEntryNode.GetVal("Time");

tNode = new TreeNode(string.Format("{0} {1}", logEntryNode.themethod,
logEntryNode.GetVal("Time")));

// get the user
string user = logEntryNode.GetVal("Original User");
if (user.Length == 0)
{
string refString = logEntryNode.GetVal("Ref");
user = refString.Substring(0, refString.IndexOf("-"));
}

//user += " - Ping";

if (!_users.ContainsKey(user))
_users.Add(user, new LogUserDetails(user, userCount++));

LogUserDetails logUserDetails = (LogUserDetails) _users[user];
tNode.BackColor = logUserDetails.color;
logUserDetails.count++;
}
else
{
tNode = new TreeNode( "Unknown" );
tNode.BackColor = Color.LightCoral;
}

TreeNode keyNode = new
TreeNode(((LogEntryNode)logEntryArrayList[0]).key);
tNode.Nodes.Add(keyNode);

foreach (LogEntryNode n in logEntryArrayList)
{
tNode.Nodes.Add(n);

if( n.Text.IndexOf(": Completed") > 0)
{
endTime = n.GetVal("Time");
if( startTime != string.Empty && endTime != string.Empty )
{
timeTaken = Convert.ToString( Convert.ToDateTime(endTime) -
Convert.ToDateTime(startTime));
}
}
}

TreeNode timeTakenNode = new TreeNode("Time Taken: " + timeTaken );
timeTakenNode.BackColor = Color.Red;
timeTakenNode.ForeColor = Color.White;

tNode.Nodes.Add(timeTakenNode);

// Distinguishing the Log entries on the Basis of the query & Save
if(tNode.Text.IndexOf("Query") == 0)
{
MetaDataTreeView.Nodes[0].Nodes.Add(tNode);
}
else
{
MetaDataTreeView.Nodes[1].Nodes.Add(tNode);
}
}

MetaDataTreeView.EndUpdate();

_logMatcherHash.Clear();

_logMatcherHash.Add("SEARCH_LOG", _logMatcher);
_logMatcher = null;
_state = 7;
}
//
// Analysing the Treeview Nodes
//
else if (_state == 7)
{
this.StatusLabel.Text = "Analysing ...";

this.LogSummaryTreeView.BeginUpdate();
LogSummaryTreeView.Nodes.Clear();

TreeNode usersNode = new TreeNode("Users");
LogSummaryTreeView.Nodes.Add(usersNode);

int count = 0;
foreach (LogUserDetails ld in this._users.Values)
{
count += ld.count;
ld.Update();
usersNode.Nodes.Add(ld);
}

usersNode.Text = string.Format("Users - Total = {0}", count);
usersNode.ExpandAll();
_users.Clear();

_state = 8;
}
else if (_state == 8)
{
StatusLabel.Text = "Log Entries are available now in Tree View
Controls.";
LogSummaryTreeView.EndUpdate();
_state = -1;

Cursor = Cursors.Default;
}
else
{
_state = -1;
SearchTimer.Enabled = false;
}

if (_state != -1)
SearchTimer.Enabled = true;
}
catch(Exception ex)
{
MessageBox.Show(ex.Message);
}
}
#endregion

*************************************************

Method 3: LogAnalyserBL.DoWork()

#region DoWork
/// <summary>
/// Reading the Selected Log files and writing the file contents into
ArrayList
/// </summary>
public void DoWork()
{
try
{
statusBar = string.Format( "Reading {0} ...", _fileName);
Stream strm = File.Open(_fileName, FileMode.Open, FileAccess.Read,
FileShare.Read);

logFileReader = new StreamReader(strm);
string line = null;

long currentCount = 0;
LogEntryNode activeNode = null;

//Reading the selected Log file Line by Line
while ((line = logFileReader.ReadLine()) != null)
{
if (line == "[LOG_ENTRY]")
{
if (activeNode != null && isCompatability)
results.Add(activeNode);

activeNode = new LogEntryNode(isCompatability);
}
else if (line == "[END_LOG_ENTRY]")
{
results.Add(activeNode);
activeNode = null;
}
else if (activeNode != null)
{
//adding the Logfile Lines into the LogEntryNode value
activeNode.AddVal(line);
}
currentCount++;

}
logFileReader.Close();
logFileReader = null;
strm.Close();
statusBar = "Reading Complete";

long count = 0;
long repCounter = 0;
statusBar = string.Format("Updating {0} Nodes...", results.Count);
foreach (LogEntryNode n in results)
{
// update the value of the node...
n.UpdateNow() ;

} }
catch(System.Exception ex)
{
Debug.WriteLine(ex.Message);
statusBar = "Error";
MessageBox.Show(ex.Message);
return;
}
finally
{
isRunning = false;
if (logFileReader != null)
logFileReader.Close();
}
}
#endregion

************************************************

Method 4: LogMatcher.DoWork

#region DoWork

public Hashtable results = new Hashtable();
public Hashtable methods = new Hashtable();
public ArrayList orderedResults = new ArrayList();
public ArrayList logs = null;

/// <summary>
/// Adding the LogEntries in the HashTables
/// </summary>
public void DoWork()
{
ArrayList kiddies = null;
foreach (LogEntryNode leNode in logs)
{
if (!results.ContainsKey(leNode.key))
{

results.Add(leNode.key, new ArrayList());

orderedResults.Add(leNode.key);
}

if (leNode.startCall)
{
if (!methods.ContainsKey(leNode.key))
methods.Add(leNode.key, leNode);
}

kiddies = (ArrayList) results[leNode.key];
kiddies.Add(leNode);
}
isRunning = false;
}
#endregion

**************************************************

Jon, If u see the above 4 methods then u can get the idea How I am reading
the files & how I am storing the Data into Hashtables & ArrayLists
Actually as u know the size of each file is 2.5 MB and there r atmost 60
files in each directory.

Now please suggest me the correct solution to reduce the memory usage in my
application.

Thanks
Arun








"Jon Skeet [C# MVP]" wrote:

Venkatesh <Venkatesh@xxxxxxxxxxxxxxxxxxxxxxxxx> wrote:
I am working on a tool which compares log files from different servers.
The application allows comparision of data from 2 servers at a time. A max
of 120 logs are compared in one go. To make things simpler, I am loading the
data of the servers in 2 huge hash tables and then comparing them. Each file
has a max size of 2.5MB. I am facing an issue with the memory usage of the
application. If I check in task manager, the application uses close to 1 GB
of memory. Even if the data of the 60 files are cached in the RAM, it should
not cross more than 2.5 * 60 = 150 MB.

Well, that depends on how you're storing the data. For a start, if
you're loading the data as text and it's in ASCII to start with, you'll
double the size of the data due to .NET strings being in Unicode.
Beyond that, however, we'll need to know more details.

I am spawning 10 threads to load the files so that the initial load is
faster.

Do you have any evidence that loading the files in parallel is actually
speeding things up? Unless loading the files really eats processor
time, you're likely to be slowing things down by reading several files
at a time. Even if it *does* speed things up, I'd expect the number of
useful threads to be significantly less than 10.

Once the load completes, I am deallocating the threads too.

How exactly are you trying to "deallocate" the threads?

I am not
quite sure why the memory usage is extremely high. Can you please provide
some pointers or alternative approaches for the implementation.

Could you post a short but complete program which demonstrates the
problem?

See http://www.pobox.com/~skeet/csharp/complete.html for details of
what I mean by that.

--
Jon Skeet - <skeet@xxxxxxxxx>
http://www.pobox.com/~skeet Blog: http://www.msmvps.com/jon.skeet
If replying to the group, please do not mail me too

.

Relevant Pages

  • Indizierte ArrayLists?
    ... Gibts in c# sowas wie "indzierte Arraylists"? ... Von den 1,5 Billionen kommen wir leider aus Kompatibilitätsgründen, nicht so ... In manchen Bereichen konnten derartige Konstrukte durch Hashtables ersetzen ...
    (microsoft.public.de.german.entwickler.dotnet.csharp)
  • Re: dynamically name dynamic arraylists
    ... I mean put the ArrayLists in the Hashtable. ... // Now you have all the arrays and each hashtable element is your varaible. ... > tried using hashtables but the data got all jumbled around.. ... > using sortedlists as well but it sorted my data... ...
    (microsoft.public.dotnet.framework.aspnet)
  • Re: Arraylist of Hashtables of arraylist
    ... new strings to existing_strings after comparing the hashtable keys. ... arraylist it also add the new value to all other hashtables in the ... ArrayList viewfields = getListOfViewAttributes; ... the view arraylists inside the loop. ...
    (microsoft.public.dotnet.languages.csharp)
  • Re: Multidimensional arrays? any other options?
    ... You can put hashtables into hastables to get ... The same, of course, applies to ArrayLists. ... If you really need a the data structure that can increase its size ... collectors don't like linked lists) ...
    (microsoft.public.dotnet.languages.csharp)