Copyright ©2012 Steven Whitney. Last modified Sun 07/29/2012 10:44:20 -0700.
|
25 Years of Programming
An open source source for C, C++, OWL, BASIC, MDB, XLS, DOT, and more... |
Home Projects Up Sitemap Search Blog Forum+Chat About Us Privacy Terms of Use Feedback FAQ Images Services Payments Humor Music |
C++ "visible" backpropagation artificial neural network program, Borland C++, MSDOS, BGI graphicsDNEURAL.CPP is the Borland C++ 4.0 MSDOS version of my neural net program.A more complete description of my neural net programs with links to later and more capable versions is on the index page for these neural network projects. I call this a "visible" neural network because the graphic display allows you to watch the node and connection states change. This code uses the Borland BIDS container classes, the BGI graphics routines, and some other Borland DOS functions. It also requires several support files, which are linked in the code below. |
/* dneural.cpp DOS 4-28-97
Copyright (C)1997 Steven Whitney.
Published under GNU GPL (General Public License) Version 3, with ABSOLUTELY NO WARRANTY.
Initially published by http://25yearsofprogramming.com.
Experiment with neural nets. Because I've never seen any other neural net program,
the goal here is a generic program flow that any neural net program would be
likely to need and data structures that are easy to understand and easily modified
as I learn more. However, this version now works quite well, and has solved
all training sets easily.
This program uses the Borland DOS BGI graphics routines.
^C is the only way to abort
TO DO:
Any further development should be done in the Windows version. It contains
everything from here, though some things, like work() aren't implemented there.
If you delete this version, save this DOS drawnodes() first, just in case,
and maybe run() and anything else that was greatly changed for Windows.
--------------------
LATER:
NOTES:
--This program was inspired by the book Complexity, by M. Mitchell Waldrop,
but most of the practical info that helped make it work came from
Chaos Under Control, by Peak and Frame.
--all other notes are in complex.doc
FILES:
-- *.TRS Training set data, problems and solutions.
NETWORK.DAT Node connection data for rebuilding network from disk.
*/
#include <stdio.h>
#include <graphics.h>
#include <math.h>
#include <fstream.h>
#include <classlib\arrays.h>
#include "c:\bcs\my.h"
// if the inclusion of mylib.cpp makes the source file too large to compile,
// put it into its own project node for separate compilation.
#include "c:\bcs\mylib.cpp"
#include "c:\bcs\library\stopwatc.cpp"
// #include "c:\bcs\library\serialco.cpp" (was planned for future, but is not required)
//////////////////////////////////////////////////////////////////////////////
// global variables
enum { SENSORY, PROCESSOR, OUTPUT }; // neuron types
constream scr; // console display in case color text ever used
class Node; // forward reference
//////////////////////////////////////////////////////////////////////////////
// a Synapse holds a pointer to the source node and the associated weight
// of the connection.
class Synapse
{
public:
Synapse() { from = 0; strength = 0.; } // default constructor (don't use)
Synapse(Node* fromnode); // normal constructor
Synapse(Node*, double); // for specifying strength
BOOL operator == (const Synapse& other) const;
Node* from; // pointer to the donor node
double strength; // weighting: input from donor = from->output * strength
};
//----------------------------------------------------------------------------
// constructor
Synapse::Synapse(Node* fromnode)
{
from = fromnode;
// random starting weight (CUC:343), from -1 to +1 inclusive, 4 digit precision
strength = ((double)random(10001)) / 10000.; // 0 to 1
if(random(2)) // allow negative weights
strength = -strength;
}
//----------------------------------------------------------------------------
// constructor where strength is specified. (for building net from disk data)
Synapse::Synapse(Node* fromnode, double startstrength)
{
from = fromnode;
strength = startstrength;
}
//----------------------------------------------------------------------------
BOOL Synapse::operator == (const Synapse& other) const
{
return(&other == this);
}
//----------------------------------------------------------------------------
// end class Synapse
//////////////////////////////////////////////////////////////////////////////
// a single neuron
class Node
{
public: // at 12-18-96, everything must be public
Node(); // default constructor required for container (don't use!)
Node(int nodetype); // constructor
~Node(); // destructor
BOOL operator == (const Node& other) const;
BOOL operator < (const Node& other) const;
friend ostream& operator << (ostream&, const Node&);
TArrayAsVector<Synapse> inlist; // it collects impulses from these nodes
TIArrayAsVector<Node> outlist; // these nodes collect inputs from this one
BOOL outputsto(Node* tonode); // whether this node already outputs to tonode
BOOL compute(); // sum its inputs and determine its current state
BOOL adjuststrength(); // determine errors at donor nodes
int type; // type of node
double input; // sum of all inputs
double output; // output sent when fired
double error; // backpropagated error at this node
int x; // screen coordinates for display
int y;
// number of times summed input has changed.
// calculated in compute(), and reset in createnode()
long inputchanges;
};
//----------------------------------------------------------------------------
// default constructor required for the container class (don't use this ctor!)
Node::Node() : inlist(1,0,0), outlist(1,0,0)
{
type = SENSORY;
input = 0.;
output = 0.;
error = 0.;
x = y = 0;
inputchanges = 0L;
}
//----------------------------------------------------------------------------
// constructor for normal use
Node::Node(int nodetype) : inlist(1,0,1), outlist(1,0,1)
{
type = nodetype;
input = 0.;
output = 0.;
error = 0.;
inputchanges = 0L;
switch(nodetype) // position across screen by node type, assumes VGAHI mode
{
case SENSORY: x = 10; break;
case PROCESSOR: x = random(610) + 10; break;
case OUTPUT: x = 620; break;
}
y = random(480);
}
//----------------------------------------------------------------------------
// destructor
Node::~Node()
{
inlist.Flush();
outlist.Flush(TShouldDelete::NoDelete); // don't delete other nodes!
}
//----------------------------------------------------------------------------
BOOL Node::operator == (const Node& other) const
{
return(&other == this);
}
//----------------------------------------------------------------------------
// orders by node type: SENSORY < PROCESSOR < OUTPUT
// might be useful in some searches, and if nodes are ever kept in a
// sorted container (which, however, is a bad idea since maintaining the
// order of creation is the only way to determine "forward connections").
BOOL Node::operator < (const Node& other) const
{
return(type < other.type);
}
//----------------------------------------------------------------------------
// write node data to a stream. This is only for formatted display to screen,
// or maybe to a file for later review. A node has no permanent data that needs
// to be saved to disk except its connection lists, which can only be written
// by the NeuralNet object.
ostream& operator << (ostream& os, const Node& n)
{
char ch;
switch(n.type)
{
case SENSORY: ch = 'S'; break;
case PROCESSOR: ch = 'P'; break;
case OUTPUT: ch = 'O'; break;
}
os << ch << ": In: " << n.input << " Out: " << n.output << " Err: " << n.error;
os << " WtIn:";
int incount = n.inlist.GetItemsInContainer(); // connection strengths
for(int j = 0 ; j < incount ; j++)
os << " " << setprecision(3) << n.inlist[j].strength;
return(os);
}
//----------------------------------------------------------------------------
// gather its inputs and determine its state. making each node responsible
// for collecting its inputs at the time it needs them may make feedback
// loops more achievable. Remember that if the o-node computation is different
// from other nodes, the calculated errors are meaningless, so you must not,
// for example, copy input to output to allow a wider range of o-node output
// values. They must remain 0 to 1, same as the others. So more complex
// answers must be built from multiple o-nodes (e.g. 8 of them to build a byte!),
// or you could have different answers falling within very close tolerances of
// each other (e.g. you can fit 100 different answers into 0-100 if they're
// spaced .01 apart with, say, .001 error tolerance. Whatever the method,
// you can always have a routine completely separate from the workings of the
// network to translate the output into more complex or useful activity.
// returns TRUE if computation was done, FALSE if input node list is empty,
// but return value not used.
BOOL Node::compute()
{
// sensory nodes already have their inputs, but must calculate their output
if(type == SENSORY)
{
// Using a passthrough gives maximum output separation with binary input:
// input 0 = output 0, input 1 = output 1
// However, I don't know that better separation is any advantage.
output = input; // passthrough
// This is the equivalent of the computation used by other nodes, poorer separation:
// input 0 = output .5, input 1 = output 1
// output = .5 + (.5 * input);
// See note below.
output = range(0.,1.,output); // enforce limits just to be sure
return(TRUE);
}
int count = inlist.GetItemsInContainer();
if(!count) // no computations to perform
return(FALSE);
// gather its inputs: get output value from each source node,
// adjust it by the strength it's allowed to have, and sum them all.
double oldinput = input; // save old value to compare with new
input = 0.; // initialize, then sum
for(int i = 0 ; i < count ; i++)
input += inlist[i].from->output * inlist[i].strength;
if(input != oldinput) // totalled input has changed
inputchanges++;
// a neuron has a basic "cruising" firing rate (output level) of 0.5;
// negative inputs push it towards 0, positive inputs push it towards 1.
// The relationship here is linear, unlike the diagram on CUC:349.
// Although the limits 0 and 1 are artificially enforced, you nevertheless want
// the computed values to fall naturally within 0 to 1. Otherwise, they'll
// always be either 0 or 1, which isn't the graded response we want.
// If the maximum node output is 1, then the maximum possible input to a node is
// 1 times the number of nodes it receives input from. If all input nodes are
// at maximum output and maximum strength, you want the total change in output
// to be +.5. If all input nodes are at maximum output with minimum strength
// (-1), you want the total change in output to be -.5.
// The above paragraph is no longer correct. There's no strength limit.
// Note that output can never be negative, so negative strength MUST be allowed.
// determine output state
output = .5; // basic output level
output += .5 * input / (double)count; // adjust according to inputs
// Exceeding this range wasn't originally supposed to be possible, but now it
// happens regularly. try removing the limitation and see what happens.
// It seems to run through the training sets much faster.
output = range(0.,1.,output); // enforce limits just to be sure
return(TRUE);
} // end compute
//----------------------------------------------------------------------------
// propagate the error at this node back to its donor nodes,
// then adjust the connection strengths accordingly.
// returns TRUE if any connection strength was changed, FALSE if not
// return not used.
BOOL Node::adjuststrength()
{
if(type == SENSORY) // backprop stops at sensory nodes anyway (no inputs)
return(FALSE);
if(error == 0.) // if no error at our node, there's nothing to propagate
return(FALSE);
int count = inlist.GetItemsInContainer();
if(!count) // no nodes to propagate the error to
return(FALSE);
// apportion error equally among inputs, instead of entire error to all (as in book)
// it worked, but first try book version
error /= (double)count;
for(int i = 0 ; i < count ; i++)
{
// calculate error at donor node. Once it's calculated, we are through
// with it. It is just the value IT will use to backpropagate its error.
// We adjust the connection strength using the error HERE (see below).
// inlist[i].from->error was set to zero at start of backprop() run.
if(inlist[i].from->error == 0.)
{
// if it does not already have a value (set by other nodes),
// give it our full calculated error. This matches the book version,
// but its example node has only 1 output, so the error calculation
// is the simple =. But if the node has multiple outputs, it will receive
// an error correction from each. How do you manage them? Sum them?
// Average them? Use the largest, smallest? See below...
inlist[i].from->error = error * inlist[i].strength;
}
else
{
// Choose only one of these possibilities at a time!
// sum them: probably overcorrects
// inlist[i].from->error += error * inlist[i].strength;
// largest: probably the best
// inlist[i].from->error = max(inlist[i].from->error,error * inlist[i].strength);
// smallest: probably undercorrects (?)
// inlist[i].from->error = min(inlist[i].from->error,error * inlist[i].strength);
// average:
// averaging slowed error value growth, but slowed learning.
// Also, all 3 output types now appear: <.5, .5, >.5
// (Don't remember if I was sure this appearance was definitely due to the averaging.)
inlist[i].from->error = (inlist[i].from->error + (error * inlist[i].strength)) / 2.;
}
// adjust the strength of the connection from our donor node to us,
// using our donor node's output and our error HERE. (CUC:346)
inlist[i].strength += error * inlist[i].from->output;
// keep strength in bounds. Actually, if this is ever invoked, it probably
// means something is wrong with the calculations above, and may be a clue
// to choosing a calculation. E.g. summing the errors can create a summed
// error greater than a node's maximum output, which could activate these adjustments.
// these should really be 1. not 1000,
// but it only works when they're 1000 (!?) even better when they're absent!
// if(inlist[i].strength > 1000.) inlist[i].strength = 1000.;
// if(inlist[i].strength < -1000.) inlist[i].strength = -1000.;
}
return(TRUE);
} // end adjuststrength
//----------------------------------------------------------------------------
// tests whether two nodes are already connected, whether this node outputs
// to the given node.
BOOL Node::outputsto(Node* tonode)
{
int count = outlist.GetItemsInContainer(); // search our outlist for tonode
for(int i = 0 ; i < count ; i++)
if(outlist[i] == tonode)
return(TRUE);
return(FALSE);
} // end outputsto
//----------------------------------------------------------------------------
// end class Node
//////////////////////////////////////////////////////////////////////////////
// a TrainingSet consists of an array of input values that define one problem
// and an array of the outputs that define its correct answer.
//
// Note that logical AND is actually 4 training sets, not 1. That is, there
// are 4 separate problems (each one a set) that must be mastered to know "AND".
//
class TrainingSet
{
public:
TrainingSet();
~TrainingSet();
BOOL operator == (const TrainingSet& other) const { return(&other == this); }
friend ostream& operator << (ostream& os, const TrainingSet&);
friend istream& operator >> (istream& is, TrainingSet&);
BOOL hassameinputs(const TrainingSet& other) const;
TArrayAsVector<double> iset; // list of values destined for S-nodes
TArrayAsVector<double> aset; // list of o-node values that constitute correct answer
};
//----------------------------------------------------------------------------
// constructor
TrainingSet::TrainingSet() : iset(1,0,1), aset(1,0,1)
{
}
//----------------------------------------------------------------------------
// destructor
TrainingSet::~TrainingSet()
{
iset.Flush();
aset.Flush();
}
//----------------------------------------------------------------------------
// write to a stream
ostream& operator << (ostream& os, const TrainingSet& t)
{
int count = t.iset.GetItemsInContainer();
for(int i = 0 ; i < count ; i++)
os << t.iset[i] << " ";
os << endl;
count = t.aset.GetItemsInContainer();
for(i = 0 ; i < count ; i++)
os << t.aset[i] << " ";
os << endl;
return(os);
}
//----------------------------------------------------------------------------
// read from a stream. File format is simply 2 lines:
// input1 input2 input3... // list of input values
// output1 output2... // list of output values that are the correct answer
// #error check whether unread/unchanged t members need default values set, for multiple reads
istream& operator >> (istream& is, TrainingSet& t)
{
string s; // to receive unknown-length file line (can be huge)
double d;
char *buf;
t.iset.Flush(); // empty any existing data
t.aset.Flush();
is >> ws; // eat whitespace in case there are blank lines
// READ THE INPUT VALUES
getline(is,s,'\n'); // read the file line
buf = new char[s.length() + 1]; // make a matching size char array
strcpy(buf,s.c_str()); // copy the string to it
s.remove(0); // free some memory
istrstream inbuf(buf); // a stream we can read values from
while(inbuf >> d) // while we can read doubles,
t.iset.Add(d); // add them to the iset array
delete[] buf; // deallocate buf
buf = 0; // make it null
// READ OUTPUT VALUES
// If the line is blank, none are read, and the set has no correct answer,
// which makes it a "real" problem.
getline(is,s,'\n');
buf = new char[s.length() + 1];
strcpy(buf,s.c_str());
s.remove(0);
istrstream inbuf2(buf);
while(inbuf2 >> d)
t.aset.Add(d);
delete[] buf;
return(is);
}
//----------------------------------------------------------------------------
// a less rigorous alternative to ==
// returns TRUE if both sets have the same iset[] elements in the same order.
// In an array of TrainingSets, all the inputs must be unique. That is, a
// particular set of inputs can't appear twice with different
// correct answers or the training winds up ambiguous.
BOOL TrainingSet::hassameinputs(const TrainingSet& other) const
{
int thiscount = iset.GetItemsInContainer();
int othercount = other.iset.GetItemsInContainer();
if(thiscount != othercount) // different # of elements
return(FALSE);
for(int i = 0 ; i < thiscount ; i++)
if(iset[i] != other.iset[i]) // if 1 doesn't match, we're ok
return(FALSE);
return(TRUE);
} // end hassameinputs
//----------------------------------------------------------------------------
// end class TrainingSet
//////////////////////////////////////////////////////////////////////////////
// the entire neural network
// only drawnodes() (called from run()) requires graphics mode
class NeuralNet
{
public:
NeuralNet(int s, int p, int o);
~NeuralNet();
BOOL operator == (const NeuralNet& other) const { return(&other == this); }
friend ostream& operator << (ostream& os, const NeuralNet&);
friend istream& operator >> (istream& is, NeuralNet&);
void savetodisk(const string& filename); // save connection data to disk
void drawnodes(BOOL showconnect); // display the entire network graphically (graphics)
string nodereport(); // report node data for every node in network (text)
BOOL think(); // 1 forward pass to produce outputs
BOOL backprop(); // 1 backward pass to adjust strengths
long run(); // process training sets until all correct outputs
BOOL work(); // after trained, solve user-provided problems
int count(int type) const; // count how many nodes of this type exist
BOOL randomnewconnection(); // connect two random nodes
BOOL newconnection(); // connect the 2 nodes that will correct the largest error
// iset is now the standardized, publicly available array into which
// inputs for ONE computation run are put, for transfer to the S-nodes.
TArrayAsVector<double> iset; // inputs for a problem
TArrayAsVector<double> aset; // correct answer, if any, for the problem
TArrayAsVector<double> oset; // translated from o-nodes, using TOLERANCE (not yet used)
BOOL graphicon; // whether to display graphics while thinking
protected:
BOOL createregularnetwork(int,int,int); // create a regularly-connected network
BOOL createrandomnetwork(int,int,int); // create a randomly-connected network
Node* createnode(int nodetype); // initialize a new node and add to master list
BOOL ripnode(Node*); // tear a node out and delete its connections
BOOL disconnect(Node* from, Node* to); // take out one connection
Node* selectatrandom(int offset); // randomly choose a node (possibly from a range)
BOOL wirein(Node* n); // establish a new node's minimum initial connections
BOOL connect(Node* from, Node* to); // connect the two nodes
BOOL connect(Node* from, Node* to, double strength); // connect the two nodes
BOOL randomconnect(Node*); // connect the node to a random node
Node* bestfromnode(Node*); // best donor candidate
Node* besttonode(Node*); // best recipient candidate
BOOL densityadjustment(); // add new node if too dense
BOOL sense(); // initialize s-nodes with a problem
BOOL testforsolution(); // determine if outputs hold correct answer
string displayoutput(); // display the contents of the output nodes
string displayoset(); // display contents of oset[]
void copyoutputs(); // copy translated o-node values to oset array
TIArrayAsVector<Node> nw; // master database of neurons
double TOLERANCE; // maximum allowable error for a correct solution
int DENSITYLIMIT; // maximum allowed average connections/node
long connectioncount; // convenient to have a counter
};
//----------------------------------------------------------------------------
// constructor
NeuralNet::NeuralNet(int s = 8, int p = 8, int o = 1)
: nw(10,0,10), iset(3,0,1), aset(1,0,1), oset(1,0,1)
{
connectioncount = 0L;
// maximum difference between two doubles to be considered equal, .1, .01 work well.
// 0 probably risks overflow error, but anything greater seems ok. .0001 worked.
// lower tolerance takes longer, but gives you longer to watch.
TOLERANCE = 0.01; // parameters could become inheritable later
DENSITYLIMIT = 8; // what effect of under or overconnection?
createrandomnetwork(s,p,o); // build and connect the entire network
// If you create a regular network, you must prevent the addition of
// new nodes and connections, or add and connect any new ones using the same
// pattern and rules as the initial set.
// createregularnetwork();
} // end constructor
//----------------------------------------------------------------------------
// destructor
NeuralNet::~NeuralNet()
{
nw.Flush(TShouldDelete::Delete); // assumes this net's nodes belong to no other net
}
//----------------------------------------------------------------------------
// write to a stream all data required for recreating the network.
// Format:
// snodecount pnodecount onodecount
// fromnode# tonode# strength
// fromnode# tonode# strength...
// could write tolerance and densitylimit if required
ostream& operator << (ostream& os, const NeuralNet& n)
{
// write how many nodes of each type
os << n.count(SENSORY) << " " << n.count(PROCESSOR) << " " << n.count(OUTPUT) << endl;
int nwcount = n.nw.GetItemsInContainer();
for(int i = 0 ; i < nwcount ; i++) // write all connection data
{
// locate each of this node's source nodes and the strength
int incount = n.nw[i]->inlist.GetItemsInContainer();
for(int j = 0 ; j < incount ; j++)
{
os << n.nw.Find(n.nw[i]->inlist[j].from) << " " << i << " ";
os << n.nw[i]->inlist[j].strength << endl;
}
}
return(os);
} // end <<
//----------------------------------------------------------------------------
// read from a stream and reconstruct the identical network.
// could be the basis for a NeuralNet::NeuralNet(const string& filename) constructor
istream& operator >> (istream& is, NeuralNet& n)
{
int i, scount, pcount, ocount;
// read specified # of each node type
is >> scount >> pcount >> ocount;
if(is.fail()) // avoid destroying existing network if read already failed
return(is);
// destroy any existing network, which assumes this net's nodes belong to no other net
n.nw.Flush(TShouldDelete::Delete);
// create network to match
for(i = 0 ; i < scount ; i++) n.createnode(SENSORY);
for(i = 0 ; i < pcount ; i++) n.createnode(PROCESSOR);
for(i = 0 ; i < ocount ; i++) n.createnode(OUTPUT);
// connect the node #s using strength
int fromnode, tonode;
double strength;
while(is >> fromnode >> tonode >> strength)
n.connect(n.nw[fromnode],n.nw[tonode],strength);
return(is);
} // end >>
//----------------------------------------------------------------------------
// save node and connection data to disk. The information saved is sufficient
// to rebuild an identical net.
void NeuralNet::savetodisk(const string& filename)
{
ofstream outfile(filename.c_str());
outfile << *this;
}
//----------------------------------------------------------------------------
// count how many nodes of this type are in the net
int NeuralNet::count(int type) const
{
int tally = 0;
int ncount = nw.GetItemsInContainer();
for(int i = 0 ; i < ncount ; i++)
if(nw[i]->type == type)
tally++;
return(tally);
} // end count
//----------------------------------------------------------------------------
// initialize a new neuron and add to master list. During network creation,
// it appends to end of master list, but after network creation, nodes of any
// type can be inserted into their respective sections.
// S nodes are inserted at the end of the s-node section.
// P nodes are inserted at a random location in the p-node section.
// O nodes are appended to end of o-node section (end of master list).
// returns a pointer to the new node, so it can be wired into the net, or 0.
//
// Don't incorporate wirein() here. During initial network creation, you should do no
// wiring till all the nodes have been created, so you get even connection density
// throughout the network.
//
Node* NeuralNet::createnode(int nodetype) // type of node to create
{
int nwcount = nw.GetItemsInContainer();
// reset the measure of activity so all nodes restart together
for(int i = 0 ; i < nwcount ; i++)
nw[i]->inputchanges = 0L;
// First node created must be sensory
if(!nwcount && (nodetype != SENSORY))
return(0);
Node* newnode = new Node(nodetype); // create and initialize the node
// if the last node is an o-node, this is a post-createnetwork() insertion.
// (Note that during createnetwork(), o-nodes after the first one are technically
// inserted, though the net result is the same.)
if(nwcount && (nw[nwcount - 1]->type == OUTPUT))
{
int offset;
switch(nodetype)
{
case PROCESSOR: // insert at a random point in the P-node section
offset = random(count(PROCESSOR) + 1) + count(SENSORY);
nw.AddAt(newnode,offset); // place it at nw[offset]
break;
case SENSORY: // insert at end of s-node section
offset = 0;
while(nw[offset]->type == SENSORY) // last s-node + 1
offset++;
nw.AddAt(newnode,offset); // place it at nw[offset]
break;
case OUTPUT:
nw.Add(newnode); // end of o-node section is also end of the array
break;
}
}
else // append the node to the master list, used during createnetwork()
nw.Add(newnode);
putchar(7); // inform viewer of new node creation
return(newnode);
} // end createnode
//----------------------------------------------------------------------------
// tear a node out of the net and delete its connections
// The goal is to call this when a node is proving of no use to the net.
// returns TRUE if successful, FALSE if not
BOOL NeuralNet::ripnode(Node* n) // the node to remove
{
if(!n)
return(FALSE);
// search inlist, and delete references to n from each of the other nodes's outlist
while(n->inlist.GetItemsInContainer()) // use while because each element
disconnect(n->inlist[0].from,n); // is deleted by disconnect()
// search outlist, and delete references to n from each of the other nodes's inlist
while(n->outlist.GetItemsInContainer())
disconnect(n,n->outlist[0]);
nw.Destroy(n); // now it's ok to delete the node
return(TRUE);
} // end ripnode
//----------------------------------------------------------------------------
// take out one connection, between two specified nodes
// The goal is to call this when a connection is proving of no use to the net.
// returns TRUE if successful, FALSE if not
BOOL NeuralNet::disconnect(Node* from, Node* to)
{
if(!from || !to)
return(FALSE);
BOOL found = FALSE; // whether they actually were connected
int count = to->inlist.GetItemsInContainer(); // remove from to's inlist
for(int i = 0 ; i < count ; i++)
if(to->inlist[i].from == from)
{
to->inlist.Destroy(i--); // undo upcoming loop increment
found = TRUE;
}
count = from->outlist.GetItemsInContainer(); // remove from from's outlist
for(i = 0 ; i < count ; i++)
if(from->outlist[i] == to)
{
from->outlist.Detach(i--,TShouldDelete::NoDelete);
found = TRUE;
}
if(found)
connectioncount--;
return(found);
} // end disconnect
//----------------------------------------------------------------------------
// select a node at random from either the entire array or from that portion
// of the array from (offset) to the array end.
// (useful for allowing only forward node connections)
// It CAN return a pointer to the node AT offset.
// returns a pointer to the chosen node, or 0 if no nodes in pool to choose from.
Node* NeuralNet::selectatrandom(int offset = 0) // pool = from this point to array end
{
int available = nw.GetItemsInContainer() - offset; // number of elements to choose from
if(available <= 0) // none!
return(0);
return(nw[offset + random(available)]);
} // end selectatrandom
//----------------------------------------------------------------------------
// connect two specified existing neurons
// doesn't check whether connection between the two already exists,
// so check before calling.
// returns TRUE
BOOL NeuralNet::connect(Node* fromnode, Node* tonode)
{
putchar(7); // inform viewer of new connection
fromnode->outlist.Add(tonode); // add to fromnode's output list
tonode->inlist.Add(Synapse(fromnode)); // add to tonode's donor list
connectioncount++; // increment global count
return(TRUE);
} // end connect
//----------------------------------------------------------------------------
// connect two specified existing neurons with a specified strength.
// doesn't check whether connection between the two already exists,
// returns TRUE
BOOL NeuralNet::connect(Node* fromnode, Node* tonode, double strength)
{
putchar(7); // inform viewer of new connection
fromnode->outlist.Add(tonode); // add to fromnode's output list
tonode->inlist.Add(Synapse(fromnode,strength)); // add to tonode's donor list
connectioncount++; // increment global count
return(TRUE);
} // end connect
//----------------------------------------------------------------------------
// establish a new connection from a specified node to a random node that it
// doesn't already output to. only allows forward connections.
// automatically ensures only legal connections between node types.
// returns TRUE if successful, FALSE if not
BOOL NeuralNet::randomconnect(Node* fromnode)
{
if(fromnode->type == OUTPUT) // an output node can't connect to anything
return(FALSE); // test is unnecessary IF it's always checked before calling
int offset = nw.Find(fromnode); // find this node's index in nw
if(offset == INT_MAX) // An existing node not found in node array
return(FALSE); // (merely returns, though it's a serious error)
offset++; // next index is first acceptable one to choose a node at
Node* tonode;
int passcount = 0, passlimit = 10 * nw.GetItemsInContainer();
BOOL found = FALSE;
while(!found)
{
// if node too hard to find a connection for, return to get a different fromnode
if(++passcount >= passlimit)
return(FALSE);
tonode = selectatrandom(offset); // choose any node later in the array (forward only)
if(!tonode) // there weren't any to choose from
return(FALSE);
if(tonode->type == SENSORY) // a type this node can't connect to?
continue;
if(tonode == fromnode) // prevent fromnode connecting to itself
continue;
if(fromnode->outputsto(tonode)) // are they already connected?
continue;
found = TRUE; // finally, it passed all tests
}
return(connect(fromnode,tonode));
} // end randomconnect
//----------------------------------------------------------------------------
// if net's density exceeds limit, wire in a new node
// returns TRUE if a node was added, FALSE if not
BOOL NeuralNet::densityadjustment()
{
// note that integer math will actually allow greater density than DENSITYLIMIT
if((connectioncount / nw.GetItemsInContainer()) > DENSITYLIMIT)
{
wirein(createnode(PROCESSOR)); // wire up a new node
return(TRUE);
}
return(FALSE);
} // end densityadjustment
//----------------------------------------------------------------------------
// establish new connection between 2 randomly selected nodes.
// returns TRUE if successfully connected, FALSE if not.
BOOL NeuralNet::randomnewconnection()
{
int nwcount = nw.GetItemsInContainer();
if(nwcount < 2) // minimum 2 nodes to connect
return(FALSE);
if(densityadjustment()) // net becoming too densely connected?
return(TRUE); // its connections are a sufficient enhancement
Node* fromnode;
BOOL success = FALSE;
while(!success) // keep trying until successful
{
do
{
fromnode = selectatrandom();
}
while(fromnode->type == OUTPUT); // output nodes can't be donors.
success = randomconnect(fromnode); // fails if connection couldn't be made
if(!success) // create a new node
{
wirein(createnode(PROCESSOR)); // wire it in.
return(TRUE); // its connections are a sufficient enhancement
}
}
return(TRUE);
} // end randomnewconnection
//----------------------------------------------------------------------------
// Determines which node can make the best use of a new incoming connection.
// Experiment with different criteria.
// returns a pointer to the node, or 0
// select randomly in case of a tie, OR hold a Classifier-style lottery to choose?
// if a fromnode is specified, tonode must be after it in the array
// #error these both (best...()) need more work. Make sure they rank the nodes
// properly, and choose only those not already connected, and prevent choosing
// nodes that can't connect anyway.
// Send H89 status report when it fails to see how often it happens.
Node* NeuralNet::besttonode(Node* fromnode = 0)
{
int nwcount = nw.GetItemsInContainer();
int fromindex = 0;
if(fromnode) // screen out illegal fromnodes
{
fromindex = nw.Find(fromnode);
// wasn't found (shouldn't happen) or is last item (can't output), or an o-node
if((fromindex == INT_MAX) || (fromindex >= nwcount - 1) || (fromnode->type == OUTPUT))
return(0);
}
// our search is from the NEXT item after fromindex to the end of the array
Node* tonode = nw[++fromindex]; // start with first eligible element selected
for(int i = fromindex ; i < nwcount ; i++)
{
if(nw[i]->type == SENSORY) // s-node can't receive
continue;
if(fromnode)
{
// If the current tonode (either from the initialization or from the
// previous loop pass) already gets input from the fromnode (if any),
// then the current nw[i] is a better choice, even before applying the
// selection criteria, and even if it will itself be disqualified on
// the next pass..
if(fromnode->outputsto(tonode))
tonode = nw[i];
// if they're already connected, it's disqualified
if(fromnode->outputsto(nw[i]))
continue;
}
// selection criteria go here
// Ideas:
// node with fewest inputs
if(nw[i]->inlist.GetItemsInContainer() < tonode->inlist.GetItemsInContainer())
tonode = nw[i];
else
if(nw[i]->inlist.GetItemsInContainer() == tonode->inlist.GetItemsInContainer())
if(nw[i]->error > tonode->error) // node with greatest error breaks tie
tonode = nw[i];
}
// final checks on the one we chose
if(tonode->type == SENSORY)
tonode = 0;
if(fromnode && fromnode->outputsto(tonode))
tonode = 0;
return(tonode);
} // end besttonode
//----------------------------------------------------------------------------
// Determines which node will contribute most by growing a new output connection.
// Experiment with different criteria.
// returns a pointer to the node, or 0
// avoid connecting *from* a node that isn't useful in the calculations anyway
// (such as an input node that is always the same for all training sets).
// select randomly in case of a tie,
// fromnode must be before tonode in the array, unless tonode is 0.
Node* NeuralNet::bestfromnode(Node* tonode = 0)
{
int toindex = 0;
if(tonode) // screen out illegal tonodes
{
toindex = nw.Find(tonode);
// wasn't found (shouldn't happen) or is the first item (can't receive), or an s-node
if((toindex == INT_MAX) || (toindex <= 0) || (tonode->type == SENSORY))
return(0);
}
// our search is from the array start to the item before toindex, inclusive
Node* fromnode = nw[0]; // start with first element selected
for(int i = 0 ; i < toindex ; i++)
{
if(nw[i]->type == OUTPUT) // o-node can't send
continue;
if(tonode)
{
// If the current fromnode (either from the initialization or from the
// previous loop pass) already outputs to the tonode (if any),
// then it's disqualified, and the current nw[i] is a better choice,
// even before applying the selection criteria, and even if it will
// itself be disqualified on the next pass.
if(fromnode->outputsto(tonode))
fromnode = nw[i];
// if they're already connected, it's disqualified
if(nw[i]->outputsto(tonode))
continue;
}
// selection criteria go here
// Ideas:
// node whose inputs (and thus output) change often.
if(nw[i]->inputchanges > fromnode->inputchanges) // > favors earlier in array if a tie
fromnode = nw[i];
else // if a tie, use node with fewest outputs
if(nw[i]->inputchanges == fromnode->inputchanges)
if(nw[i]->outlist.GetItemsInContainer() < fromnode->outlist.GetItemsInContainer())
fromnode = nw[i];
}
// final checks on the one we chose
if(fromnode->type == OUTPUT)
fromnode = 0;
if(tonode && fromnode->outputsto(tonode))
fromnode = 0;
return(fromnode);
} // end bestfromnode
//----------------------------------------------------------------------------
// Unlike randomnewconnection, this connects the 2 nodes whose connection is
// most likely to help the network.
// returns TRUE if a new connection (and possibly node) was added, FALSE if not.
BOOL NeuralNet::newconnection()
{
// return(FALSE); // enable this to force random connections instead
if(densityadjustment()) // net becoming too densely connected?
return(TRUE); // its connections are a sufficient enhancement
Node* fromnode = 0;
Node* tonode = 0;
if(random(2)) // choose which to do first
{
fromnode = bestfromnode();
tonode = besttonode(fromnode);
}
else
{
tonode = besttonode();
fromnode = bestfromnode(tonode);
}
if(fromnode && tonode && !fromnode->outputsto(tonode))
return(connect(fromnode,tonode));
return(FALSE);
} // end newconnection
//----------------------------------------------------------------------------
// wire a newly created node into the existing network, giving it the minimum
// starting number of input or output connections required for its type.
// Note that this is only for random networks. Regular nets need a different method.
// returns TRUE if connections successfully established, FALSE if not.
BOOL NeuralNet::wirein(Node* n)
{
if(!n) // guard against null pointer
return(FALSE);
int i;
Node* donornode;
switch(n->type) // # and type of connections depends on node type
{
case SENSORY:
for(i = 0 ; i < 2 ; i++) // number of s-node output connections
if(!randomconnect(n)) // Unable to connect
return(FALSE);
break;
case PROCESSOR: // this doesn't guarantee any input connections
for(i = 0 ; i < 1 ; i++) // number of p-node output connections
if(!randomconnect(n)) // Unable to connect
return(FALSE);
break;
case OUTPUT: // backwards: o-node must be recipient
while(n->inlist.GetItemsInContainer() < 2) // (minimum acceptable donor count)
{
// randomly select a s- or p-node donor that it's not already connected to
do
{
donornode = selectatrandom();
}
while((donornode->type == OUTPUT) || (donornode->outputsto(n)));
if(!connect(donornode,n))
return(FALSE);
}
break;
}
return(TRUE);
} // end wirein
//----------------------------------------------------------------------------
// create all the neurons and establish their connections to each other
// returns TRUE if successful, FALSE if not
BOOL NeuralNet::createrandomnetwork(int scount, int pcount, int ocount)
{
int i;
// create the neurons. The creation order determines calculation order, too,
// and several routines depend on them having been created in this order:
for(i = 0 ; i < scount ; i++) if(!createnode(SENSORY)) return(FALSE);
for(i = 0 ; i < pcount ; i++) if(!createnode(PROCESSOR)) return(FALSE);
for(i = 0 ; i < ocount ; i++) if(!createnode(OUTPUT)) return(FALSE);
// now connect each node to designated number of other nodes. the way this
// is done, s- and p-nodes will have at least n outputs, but might have
// more if any were added to correct an under-connected o-node.
// s- and p-nodes can have any number of input nodes. o-nodes will have at
// least n inputs. n varies with the node type. (See wirein())
int nodecount = nw.GetItemsInContainer();
for(i = 0 ; i < nodecount ; i++)
if(!wirein(nw[i]))
return(FALSE);
return(TRUE);
} // end createrandomnetwork
//----------------------------------------------------------------------------
// create all the neurons and establish their connections to each other
// This version creates a regularly-connected net with only 1 hidden layer.
// Later, expand it to allow multiple hidden layers. Each sensory node will
// connect to each pnode in the first layer. Each pnode in each pnode layer
// will connect to each pnode in the next layer. Each pnode in the last
// hidden layer connects to each output node.
// This was originally created when the random network learned poorly.
// I think there is now not much difference between the two.
// returns TRUE if successful, FALSE if not
BOOL NeuralNet::createregularnetwork(int scount, int pcount, int ocount)
{
int i;
// create the neurons. The creation order determines calculation order, too,
// and several routines depend on them having been created in this order:
for(i = 0 ; i < scount ; i++) if(!createnode(SENSORY)) return(FALSE);
for(i = 0 ; i < pcount ; i++) if(!createnode(PROCESSOR)) return(FALSE);
for(i = 0 ; i < ocount ; i++) if(!createnode(OUTPUT)) return(FALSE);
// now connect each node to designated number of other nodes.
int nodecount = nw.GetItemsInContainer();
for(int k = 0 ; k < nodecount ; k++)
{
switch(nw[k]->type)
{
case SENSORY: // each sensory node outputs to each processor node
for(i = 0 ; i < nodecount ; i++)
if(nw[i]->type == PROCESSOR)
if(!connect(nw[k],nw[i]))
return(FALSE);
break;
case PROCESSOR: // each processor node connects to each output node
for(i = 0 ; i < nodecount ; i++)
if(nw[i]->type == OUTPUT)
if(!connect(nw[k],nw[i]))
return(FALSE);
break;
case OUTPUT: // output connects to nothing
break;
}
} // end for(connect all nodes)
return(TRUE);
} // end createregularnetwork
//----------------------------------------------------------------------------
// display the entire network graphically
void NeuralNet::drawnodes(BOOL showconnect) // whether to show connections
{ // FALSE if just updating node fills
int color;
int nwcount = nw.GetItemsInContainer();
for(int i = 0 ; i < nwcount ; i++)
{
switch(nw[i]->type) // set color for this node
{
case SENSORY: color = LIGHTRED; break;
case PROCESSOR: color = LIGHTGREEN; break;
case OUTPUT: color = WHITE; break;
}
// currently, if a node is ever filled in, this method won't un-fill it
// if it subsequently turns off. Set color to black? What about bounding color?
// probably better to use getimage, putimage.
setcolor(color);
// display as numbers
outtextxy(nw[i]->x, nw[i]->y, tostring(i).c_str());
// or display as circles
// circle(nw[i]->x, nw[i]->y, 5);
// if(nw[i]->input > 0.) // inputs positive, so fill it in
// {
// setfillstyle(SOLID_FILL,color);
// floodfill(nw[i]->x,nw[i]->y,color);
// }
if(!showconnect)
continue;
// Now draw the connections to or from this node.
// If both versions prove useful, can use a user-settable variable to choose.
// This version colors the connections according to the connection strength.
// To do so, it must draw the connections using inlist, so it
// draws the connections coming IN to this node.
// int count = nw[i]->inlist.GetItemsInContainer();
// for(int j = 0 ; j < count ; j++)
// {
// switch((int)signum(nw[i]->inlist[j].strength))
// {
// case 1: setcolor(LIGHTGRAY); break; // strength is positive (stimulating)
// case -1: setcolor(RED); break; // strength is negative (inhibiting)
// case 0: setcolor(BLUE); break; // strength is ZERO
// }
// line(nw[i]->inlist[j].from->x, nw[i]->inlist[j].from->y, nw[i]->x, nw[i]->y);
// }
// This version colors the connections by whether this node's output is <,>,= .5
// same color for all the drawn connections. Uses outlist to find connections,
// so it draws the connections going OUT from this node.
switch((int)signum(nw[i]->output - .5))
{
case 1: setcolor(LIGHTGRAY); break; // output > .5
case -1: setcolor(RED); break; // output < .5
case 0: setcolor(BLUE); break; // output = .5 ("base rate")
}
int count = nw[i]->outlist.GetItemsInContainer();
for(int j = 0 ; j < count ; j++)
line(nw[i]->x, nw[i]->y, nw[i]->outlist[j]->x, nw[i]->outlist[j]->y);
} // end for(all nodes in network)
} // end drawnodes
//----------------------------------------------------------------------------
// initialize sensory nodes with a problem: Transfers data from the sensory
// array to the s-nodes. It first determines if there are enough sensory nodes
// to hold all the input. If there aren't, it adds s-nodes. It also makes sure
// there are enough o-nodes that a correct answer can be generated. If there
// aren't, it adds o-nodes.
// returns TRUE usually, or FALSE if any nodes were added.
// --> If nodes had to be added, the net probably needs retraining.
BOOL NeuralNet::sense()
{
BOOL addednode = FALSE;
int isetcount = iset.GetItemsInContainer();
int asetcount = aset.GetItemsInContainer();
while(isetcount > count(SENSORY)) // meet minimum required s-nodes
{
wirein(createnode(SENSORY));
addednode = TRUE;
}
while(asetcount > count(OUTPUT)) // meet minimum required o-nodes
{
wirein(createnode(OUTPUT));
addednode = TRUE;
}
int index = 0; // index into iset[]
int count = nw.GetItemsInContainer();
for(int i = 0 ; i < count ; i++) // run through all the nodes
if(nw[i]->type == SENSORY) // can only fill sensory nodes
{
// if there is input data for this s-node, use it
// else if we ran out of input data, set the node to zero
if(index < isetcount)
{
nw[i]->input = iset[index++];
nw[i]->inputchanges++; // filling a node changes it, marks it as active,
} // to distinguish it from excess nodes
else
nw[i]->input = 0.; // excess nodes are considered "unchanged"
}
return(addednode); // whether any new nodes had to be created
} // end sense
//----------------------------------------------------------------------------
// display the raw o-node outputs.
// returns a string containing the text of the report.
// You can then print it to screen (DOS) or in a window (WIN).
string NeuralNet::displayoutput()
{
ostrstream report;
report << "Raw O-Node Result: ";
int count = nw.GetItemsInContainer();
for(int i = 0 ; i < count ; i++)
if(nw[i]->type == OUTPUT)
report << nw[i]->output << " ";
report << endl << ends;
string s(report.str());
delete[] report.str();
return(s);
} // end displayoutput
//----------------------------------------------------------------------------
// display the contents of the output array (as translated).
// returns a string containing the text of the report.
// You can then print it to screen (DOS) or in a window (WIN).
string NeuralNet::displayoset()
{
ostrstream report;
report << "Output Array Result: ";
int count = oset.GetItemsInContainer();
for(int i = 0 ; i < count ; i++)
report << oset[i] << " ";
report << endl << ends;
string s(report.str());
delete[] report.str();
return(s);
} // end displayoset
//----------------------------------------------------------------------------
// report various node data for every node in network
// returns a string containing the text of the report.
// You can then print it to screen (DOS) or in a window (WIN).
string NeuralNet::nodereport()
{
ostrstream report;
int nwcount = nw.GetItemsInContainer();
for(int i = 0 ; i < nwcount ; i++)
report << i << "-" << *(nw[i]) << endl;
report << "There are " << nwcount << " nodes, " << connectioncount << " connections.";
report << endl << ends;
string s(report.str());
delete[] report.str();
return(s);
} // end nodereport
//----------------------------------------------------------------------------
// determine whether output nodes contain the correct answer, if known,
// and sets the error value of each o-node as the start point for backprop().
// error = desired output - actual output
// returns TRUE if solution for this set was found, FALSE if not
BOOL NeuralNet::testforsolution()
{
BOOL solutionfound = TRUE; // any node with an error turns it FALSE
int asindex = 0; // index into aset
int asetcount = aset.GetItemsInContainer();
int count = nw.GetItemsInContainer();
for(int i = 0 ; i < count ; i++)
if(nw[i]->type == OUTPUT) // only test output nodes
{
// calculate the error. nw[i]->output was already calculated in compute()
// If we have extra o-nodes not needed, error is zero.
nw[i]->error = (asindex < asetcount) ? (aset[asindex] - nw[i]->output) : 0.;
asindex++;
// do not break from loop as soon as you know it's false,
// because you're calculating the errors also, for all the nodes.
if(fabs(nw[i]->error) > TOLERANCE)
solutionfound = FALSE;
}
return(solutionfound);
} // end testforsolution
//----------------------------------------------------------------------------
// make ONE computational pass forward through the network,
// Traverses nodes in sequential order, the order in which they were born,
// returns TRUE
BOOL NeuralNet::think()
{
int count = nw.GetItemsInContainer();
for(int i = 0 ; i < count ; i++)
nw[i]->compute(); // calculate
return(TRUE);
} // end think
//----------------------------------------------------------------------------
// make ONE computational pass backwards through the network,
// calculating errors and adjusting connection strengths.
// returns TRUE
BOOL NeuralNet::backprop()
{
// first reset all errors to 0 because error calculations are usually cumulative
int count = nw.GetItemsInContainer();
for(int i = 0 ; i < count ; i++)
if(nw[i]->type != OUTPUT) // must keep o-node errors from testforsolution()
nw[i]->error = 0.; // because they are our starting point.
for(i = count - 1 ; i >= 0 ; i--) // start at end and traverse backwards
nw[i]->adjuststrength(); // propagate node's error to its inputs
// and adjust connection strengths.
return(TRUE);
} // end backprop
//----------------------------------------------------------------------------
// Transfer the current o-node values to the oset output array. Values in
// the array are translated using TOLERANCE to exactly 0 or exactly 1.
// They can then be used 1) to build a compound output (byte or string),
// (which could then be used to trigger a complex set of actions) or
// 2) as passthrough input to another network.
// It assumes that output does not have a mid-range value (i.e. it really can
// be translated to either 0 or 1). A midrange value is translated to 0.
void NeuralNet::copyoutputs()
{
oset.Flush(); // empty previous values
int count = nw.GetItemsInContainer();
for(int i = 0 ; i < count ; i++)
if(nw[i]->type == OUTPUT)
oset.Add((nw[i]->output < (1. - TOLERANCE)) ? 0. : 1.);
} // end copyoutputs
//----------------------------------------------------------------------------
// accept inputs and generate an answer for one problem. If a correct answer
// has been provided, it cycles with backprop() until that answer is matched.
// returns the number of times backprop() had to be invoked. Thus, if it
// returns 0, it achieved the correct solution immediately.
// In any event, it doesn't return until problem is solved.
long NeuralNet::run()
{
// passlimit and opasslimit should be inheritable later, to find optimum values
long passcount = 0L, passlimit = 1000L; // passes through inner loop, was 20,000
sense(); // fill sensory nodes with the current problem
think(); // forward-process
// testforsolution will return TRUE if it's a real (not a training) run,
// that is, if there is no data determining what a correct solution is.
while(testforsolution() == FALSE) // also calculates error at each o-node
{
if(kbhit()) // test for user input
switch(tolower(getch()))
{
case 27: // ESC quits
return(1L);
case 'g': // toggle graphic node display
graphicon = !graphicon;
if(graphicon)
scr.clrscr();
break;
}
if(graphicon) // graphic display of nodes
drawnodes(TRUE);
backprop();
if((++passcount % passlimit) == 0L) // if too many failures, increase net size
if(!newconnection()) // add new connection (and maybe new p-node)
randomnewconnection(); // if selection method didn't work, do it randomly
think();
}
copyoutputs();
return(passcount);
} // end run
//----------------------------------------------------------------------------
// get sensory input from user; calculate and display output
// permits random-sequence testing as to whether net is really trained.
// and experimenting with inputs *similar* to those in the training sets.
// input goes into the iset[0] array
// returns FALSE if user wants to quit, TRUE otherwise
// not really a function of the net: should be in Trainer or a TSet constructor?
BOOL NeuralNet::work()
{
iset.Flush(); // empty so we can specify them
aset.Flush(); // must be empty for this "real" run
double d;
char buf[80];
// get exactly enough inputs (s-node count) from user
string prompt("\nNeed inputs for ");
prompt += tostring(count(SENSORY)) + " sensory nodes, one at a time...\n";
scr << prompt;
for(int snode = 0 ; snode < count(SENSORY) ; snode++) // which s-node is being filled
{
prompt = string("Enter input[") + tostring(snode) + "], <CR> to quit: ";
scr << prompt;
gets(buf);
istrstream is(buf);
is >> d;
if(is.fail())
return(FALSE);
iset.Add(d);
}
run(); // aset is empty, so it just fills s-nodes and calculates result
scr << displayoutput(); // display result
scr << displayoset();
return(TRUE);
} // end work
//----------------------------------------------------------------------------
// end class NeuralNet
//////////////////////////////////////////////////////////////////////////////
// the Trainer is responsible for loading or creating training sets, presenting them
// to the network, and judging whether the outputs are correct and whether
// the network is trained on all sets.
class Trainer
{
public:
Trainer();
~Trainer();
BOOL operator == (const Trainer& other) const { return(&other == this); }
void flushsets(); // empty sets array
BOOL loadsets(const string& filename); // load training sets from disk
BOOL poseproblem(int, NeuralNet&);
long trainnetwork(NeuralNet&);
TArrayAsVector<TrainingSet> sets;
};
//----------------------------------------------------------------------------
// constructor
Trainer::Trainer() : sets(3,0,1)
{
}
//----------------------------------------------------------------------------
// destructor
Trainer::~Trainer()
{
sets.Flush();
}
//----------------------------------------------------------------------------
void Trainer::flushsets()
{
sets.Flush();
}
//----------------------------------------------------------------------------
// read all training set info from disk. A file containing multiple training
// sets simply has multiple 2-line pairs, as described above for one set.
// If you want to start empty, flushsets() first.
// You can load sets from multiple files without flushing to create a large compound set.
// But you must be sure that every set of input node values is unique.
// returns TRUE if successful, FALSE if not (bad file or an input not unique).
//----------------------------------------------------------------------------
BOOL Trainer::loadsets(const string& filename)
{
ifstream infile(filename.c_str());
if(!infile)
return(FALSE);
BOOL succeeded = TRUE; // all loaded AND all were unique
TrainingSet t;
while(infile >> t)
{
BOOL unique = TRUE;
int count = sets.GetItemsInContainer();
for(int i = 0 ; i < count ; i++)
if(sets[i].hassameinputs(t))
{
succeeded = unique = FALSE;
break;
}
if(unique)
sets.Add(t);
// scr << "Loading set:" << endl << t;
}
// presskey(scr);
return(succeeded); // if FALSE, you probably don't want to use the set
} // end loadsets
//----------------------------------------------------------------------------
// pose a problem to the network: fill its sensory array, and optionally its
// answer array.
BOOL Trainer::poseproblem(int tsindex, NeuralNet& n)
{
n.iset.Flush(); // empty old input values
n.aset.Flush(); // empty any old answer values
// Fill the input array with sensory data.
int count = sets[tsindex].iset.GetItemsInContainer();
for(int i = 0 ; i < count ; i++)
n.iset.Add(sets[tsindex].iset[i]);
// If there are known correct answers, put them in the answers array.
count = sets[tsindex].aset.GetItemsInContainer();
for(i = 0 ; i < count ; i++)
n.aset.Add(sets[tsindex].aset[i]);
return(TRUE);
} // end poseproblem
//----------------------------------------------------------------------------
// drill net until it correctly solves all training sets. The network is
// trained only when it solves ALL the sets consecutively without ANY set
// requiring backprop. It only returns when the net is fully trained, unless user aborts.
// returns the number of passes through the complete training set that were required.
// 1 pass always required, so if it returns 0, user aborted, and net isn't trained.
long Trainer::trainnetwork(NeuralNet& n)
{
long passcount = 0L, passlimit = 100L;
BOOL trained = FALSE;
while(!trained)
{
if(kbhit()) // test for user input
switch(tolower(getch()))
{
case 27: // ESC quits
return(0L);
}
if((++passcount % passlimit) == 0L)
if(!n.newconnection())
n.randomnewconnection();
trained = TRUE; // any failure sets it false
int setcount = sets.GetItemsInContainer();
for(int tsindex = 0 ; tsindex < setcount ; tsindex++) // each training set
{
poseproblem(tsindex,n);
if(n.run() > 0L) // if it used backprop,
trained = FALSE; // it's not fully trained.
}
}
n.drawnodes(TRUE);
return(passcount);
} // end trainnetwork
//----------------------------------------------------------------------------
// end class Trainer
//////////////////////////////////////////////////////////////////////////////
// global functions
//----------------------------------------------------------------------------
// show help screen
void drawhelp(const string& filename)
{
ostrstream os;
os << "Usage: NEURAL trainingset(.TRS)" << endl;
os << " Default trainingset file is AND.TRS" << endl;
os << "While running:" << endl;
os << "<ESC> quit." << endl;
os << "G Toggle Graphic display (program runs faster when off)" << endl;
os << endl;
os << "In graphic display, sensory nodes are RED, on left side of screen," << endl;
os << "processor nodes are GREEN in center, and output nodes are WHITE at right." << endl;
os << "Flow is from sensory nodes through processor nodes, to output nodes." << endl;
os << endl;
os << "Connection coloring indicates a neuron's output state:" << endl;
os << "WHITE = high " << "RED = low " << "BLUE = neutral (base rate)" << endl;
os << "A click sounds when a new node or connection is created" << endl;
os << endl;
os << "When trained, a tone plays and the final network diagram is shown." << endl;
os << "Press a key to return to text mode." << endl;
os << endl;
os << "Current input file is: " << filename << endl << endl << ends;
scr << os.str();
delete[] os.str();
presskey(scr);
} // end drawhelp
//////////////////////////////////////////////////////////////////////////////
// main()
int main(int argc, char **argv)
{
randomize();
string::set_paranoid_check(TRUE);
string::set_case_sensitive(FALSE);
scr.window(1,1,80,25);
scr.clrscr();
string filename = "AND.TRS"; // set up a default training set
// filename = "logic.trs"; // takes forever
// filename = "andor.trs"; // it can solve this in a reasonable time
if(argc > 1) // user-specified training set?
{
filename = argv[1];
if(!filename.contains(".")) // append extension if omitted
filename += ".trs";
}
filename.to_upper();
drawhelp(filename);
Trainer trainer;
NeuralNet network; // constructor aborts if it fails
network.graphicon = TRUE;
if(!trainer.loadsets(filename))
graborts("File missing or bad, or training sets not unique.");
initgraf(); // initialize graphics, VGAHI
Stopwatch sw;
sw.start();
long trained = trainer.trainnetwork(network); // train needs graphics mode, for drawnodes()
sw.stop();
if(trained) // when trained, alert user
{
sound(440);
sleep(1);
nosound();
getch(); // wait for user key before erasing node diagram on screen
}
restorecrtmode();
if(!trained)
scr << "Network not trained." << endl;
else
{
scr << "Trained on all input sets. Time: " << sw.trialtime() << " seconds.";
scr << " Trials: " << trained << endl;
}
presskey();
scr << network.nodereport(); // show some node data
getch();
// note it may or may not be trained properly
while(network.work()); // get inputs from user & calculate results
network.savetodisk("network.dat");
closegraph();
return(0);
} // end main
|
|
|
|
|
|
Copyright ©2012 Steven Whitney. Last modified Sun 07/29/2012 10:44:20 -0700. |
||