KB Neural Data Mining - Python sources « Python recipes

http://www.freeopen.org/?p=85 The aim of this book is to present and describe in detail the algorithms to extract the knowledge hidden inside data using Python language, which allows us to read and easily understand the nature and the characteristics of the rules of the computing utilized, as opposed to what happens in commercial applications, which are available only in the form of running codes, which remain impossible to modify.
The algorithms of computing contained within the book are minutely described, documented and available in the Python source format, and serve to extract the hidden knowledge within the data whether they are textual or numerical kinds. There are also various examples of usage, underlining the characteristics, method of execution and providing comments on the obtained results.
      # -*- coding: utf-8 -*-
############################################################################### 
# KB_CAT KNOWLEDGE DISCOVERY IN DATA MINING (CATALOG PROGRAM)                 # 
# by ROBERTO BELLO (COPYRIGHT MARCH 2011 ALL RIGHTS RESERVED)                 # 
# Language used: PYTHON                               .                       # 
############################################################################### 
import os
import random
import copy
import datetime

def mean(x):     	# mean
  n = len(x)
  mean = sum(x) / n
  return mean

def sd(x):		# standard deviattion
  n = len(x)
  mean = sum(x) / n
  sd = (sum((x-mean)**2 for x in x) / n) ** 0.5
  return sd

def is_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False

class ndim:             # from 3D array to flat array
    def __init__(self,x,y,z,d):
        self.dimensions=[x,y,z]
        self.numdimensions=d
        self.gridsize=x*y*z

    def getcellindex(self, location):
        cindex = 0
        cdrop = self.gridsize
        for index in xrange(self.numdimensions):
            cdrop /= self.dimensions[index]
            cindex += cdrop * location[index]
        return cindex

    def getlocation(self, cellindex):
        res = []
        for size in reversed(self.dimensions):
            res.append(cellindex % size)
            cellindex /= size
        return res[::-1]

""" how to use ndim class
n=ndim(4,4,5,3)
print n.getcellindex((0,0,0))
print n.getcellindex((0,0,1))
print n.getcellindex((0,1,0))
print n.getcellindex((1,0,0))

print n.getlocation(20)
print n.getlocation(5)
print n.getlocation(1)
print n.getlocation(0)
"""

print("###############################################################################")
print("# KB_CAT KNOWLEDGE DISCOVERY IN DATA MINING (CATALOG PROGRAM)                 #")
print("# by ROBERTO BELLO (COPYRIGHT MARCH 2011 ALL RIGHTS RESERVED)                 #")
print("# Language used: PYTHON                                                       #")
print("###############################################################################") 

# input and run parameters
error = 0

while True:
  arch_input = raw_input('InputFile                              : ')
  if not os.path.isfile(arch_input):
    print("Oops! File does not exist. Try again... or CTR/C to exit")     
  else:
    break

while True:
  try:
    num_gruppi = int(raw_input('Number of Groups (3 - 20)              : '))
  except ValueError:
    print("Oops!  That was no valid number.  Try again...")
  else:
    if(num_gruppi < 3):
      print("Oops! Number of Groups too low. Try again...")
    else:
      if(num_gruppi > 20):
        print("Oops! Number of Groups too big. Try again...")
      else:
        break

while True:
  normaliz   = raw_input('Normalization(Max, Std, None)          : ')
  normaliz   = normaliz.upper()
  normaliz   = normaliz[0]
  if(normaliz <> 'M' and normaliz <> 'S' and normaliz <> 'N'):
    print("Oops! Input M, S or N. Try again...")
  else:
    break

while True:
  try:
    max_alpha   = float(raw_input('Start value of alpha (1.8 - 0.9)       : '))
  except ValueError:
    print("Oops!  That was no valid number.  Try again...")
  else:
    if(max_alpha > 1.8):
      print("Oops! Start value of alpha too big. Try again...")
    else:
      if(max_alpha < 0.9):
        print("Oops! Start value of alpha too low. Try again...")
      else:
        break

while True:
  try:
    min_alpha  = float(raw_input('End value of alpha (0.5 - 0.0001)      : '))
  except ValueError:
    print("Oops!  That was no valid number.  Try again...")
  else:
    if(min_alpha > 0.5):
      print("Oops! alpha too big. Try again...")
    else:
      if(min_alpha < 0.0001):
        print("Oops! alpha too low. Try again...")
      else:
        break

while True:
  try:
    step_alpha  = float(raw_input('Decreasing step of alpha (0.1 - 0.001) : '))
  except ValueError:
    print("Oops!  That was no valid number.  Try again...")
  else:
    if(step_alpha > 0.1):
      print("Oops! Decreasing step of alpha too big. Try again...")
    else:
      if(step_alpha < 0.001):
        print("Oops! Decreasing step of alpha too low. Try again...")
      else:
        break


file_input   = arch_input 
gruppi_num   = num_gruppi
tipo_norm    = normaliz 
alpha_min    = min_alpha 
alpha_max    = max_alpha
alpha_step   = step_alpha

# outputs files
file_input   = arch_input
tipo_norm    = normaliz
gruppi_num   = num_gruppi
nome_input   = file_input.split(".") 
arch_output  = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_out.txt" 
arch_outsrt  = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_outsrt.txt" 
arch_sort    = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_sort.txt" 
arch_catal   = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_catal.txt" 
arch_medsd   = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_medsd.txt" 
arch_cv      = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_cv.txt" 
arch_grid    = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_grid.txt" 
arch_log     = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_log.txt" 

# start time
t0 = datetime.datetime.now()

# read input file 
arr_r    = [] 
arr_orig = [] 
arr_c    = [] 
mtchx    = [] 
mtchy    = [] 
txt_col  = [] 
xnomi    = [] 

# the numbers of variables / columns in all record must be the same 
n_rows = 0 
n_cols = 0 
err_cols = 0 
index = 0 
for line in open(file_input).readlines():
  linea = line.split()
  if(index == 0):
    xnomi.append(linea)
    n_cols = len(linea)
  else:
    arr_r.append(linea)
    if(len(linea) != n_cols):
      err_cols = 1
      print("Different numbers of variables / columns in the record " + str(index)
        + " cols " + str(len(linea))) 
  index += 1
if(err_cols == 1):
  print("File " + file_input + " contains errors. Exit ")
  quit()
index = 0
while index < len(arr_r):
  linea = arr_r[index]
  index_c = 0
  while index_c < len(linea):
    if linea[index_c].isdigit():
      linea[index_c] = float(linea[index_c])
    index_c += 1
  arr_r[index] = linea 
  index += 1
arr_orig = copy.deepcopy(arr_r)        # original input file
testata_cat = copy.deepcopy(xnomi[0])  # original header row

# finding columns containing strings and columns containing numbers
testata = xnomi[0]
testata_orig = copy.deepcopy(xnomi[0])
n_cols = len(testata) - 1
n_rows = len(arr_r)
ind_c  = 1
err_type = 0
while ind_c < len(testata):
  ind_r    = 1
  tipo_num = 0 
  tipo_txt = 0 
  while ind_r < len(arr_r):

    arr_c = arr_r[ind_r]

    if is_number(arr_c[ind_c]):
      tipo_num = 1
    else:
      tipo_txt = 1

    ind_r += 1

  if tipo_num == 1 and tipo_txt == 1:
    print "The columns / variables " + testata[ind_c] + " contains both strings and numbers." 
    print arr_c
    err_type = 1 
  ind_c += 1
if err_type == 1:
  print "Oops! The columns / variables contains both strings and numbers. Exit. " 
  quit()

index_c = 1 
while index_c <= n_cols:
  txt_col = [] 
  index = 0 
  while index < len(arr_r):
    arr_c = arr_r[index] 
    if(isinstance(arr_c[index_c],str)):
      txt_col.append(arr_c[index_c]) 
    index += 1 
  set_txt_col = set(txt_col)             # remove duplicates 
  txt_col = list(set(set_txt_col))
  txt_col.sort()
  
  # from strings to numbers 
  if(len(txt_col) > 0): 
    if(len(txt_col) > 1): 
      passo1 = 1.0 / (len(txt_col) - 1) 
    else:
      passo1 = 0.0
    index = 0
    while index < len(arr_r): 
      arr_c = arr_r[index] 
      campo1 = arr_c[index_c] 
      indice1 = txt_col.index(campo1) 
      if(len(txt_col) == 1):  # same values in the column
        val_num1 = float(1) 
      else: 
        val_num1 = float(passo1 * indice1)
      arr_c[index_c] = val_num1 + 0.00000001   # to avoid zero values in means
                                               # (to prevent zero divide in CV)
      index += 1 
  index_c += 1 

# means, max & std 
xmeans = [] 
xmaxs  = [] 
xmins  = []            ### aggiunto Roberto 4/03/2012
xsds   = [] 
xcv    = [] 
index_c = 0
while index_c <= n_cols:
  xmeans.append(0.0) 
  xmaxs.append(-9999999999999999.9) 
  xmins.append(9999999999999999.9)	### aggiunto Roberto 4/03/2012
  xsds.append(0.0) 
  xcv.append(0.0)
  index_c += 1 

# means & max
index = 0 
while index < n_rows:
  arr_c = arr_r[index] 
  index_c = 1 
  while index_c <= n_cols:
    xmeans[index_c] += arr_c[index_c] 
    if(arr_c[index_c] > xmaxs[index_c]):
      xmaxs[index_c] = arr_c[index_c] 
    index_c += 1 
  index += 1 
index_c = 1 
while index_c <= n_cols:
  xmeans[index_c] = xmeans[index_c] / n_rows
  index_c += 1 

# std
index = 0 
while index < n_rows:
  arr_c = arr_r[index] 
  index_c = 1 
  while index_c <= n_cols:
    xsds[index_c] += (arr_c[index_c] - xmeans[index_c])**2 
    index_c += 1 
  index += 1 
index_c = 1 

while index_c <= n_cols:
  xsds[index_c] = (xsds[index_c] / (n_cols - 1)) ** 0.5 
  index_c += 1 

# Means, Max, Std, CV output file
medsd_file = open(arch_medsd, 'w')
  
# columns names
medsd_file.write('%s %s ' % ('Function' , "\t"))
index_c = 1 
while index_c <= n_cols:
  medsd_file.write('%s %s ' % (testata[index_c], "\t"))
  index_c += 1
medsd_file.write('%s' % ('\n'))

# means
medsd_file.write('%s %s ' % ('Mean' , "\t"))
index_c = 1 
while index_c <= n_cols:
  valore = str(xmeans[index_c]) 
  valore = valore[0:6] 
  medsd_file.write('%s %s ' % (valore, "\t")) 
  index_c += 1 
medsd_file.write('%s' % ('\n'))

# max
medsd_file.write('%s %s ' % ('Max' , "\t"))
index_c = 1 
while index_c <= n_cols:
  valore = str(xmaxs[index_c]) 
  valore = valore[0:6] 
  medsd_file.write('%s %s ' % (valore, "\t")) 
  index_c += 1 
medsd_file.write('%s' % ('\n'))

# std
medsd_file.write('%s %s ' % ('Std' , "\t"))
index_c = 1 
while index_c <= n_cols:
  valore = str(xsds[index_c]) 
  valore = valore[0:6] 
  medsd_file.write('%s %s ' % (valore, "\t")) 
  index_c += 1 
medsd_file.write('%s' % ('\n'))

# CV 
medsd_file.write('%s %s ' % ('CV' , "\t"))
index_c = 1 
med_cv_gen = 0.0 		# cv average of all columns / variables
while index_c <= n_cols:
  if xmeans[index_c] == 0: 
    media1 = 0.000001 
  else:
    media1 = xmeans[index_c] 
  xcv[index_c] = xsds[index_c] / abs(media1) 
  valore = str(xcv[index_c])
  med_cv_gen += xcv[index_c]
  valore = valore[0:6] 
  medsd_file.write('%s %s ' % (valore, "\t")) 
  index_c += 1 
med_cv_gen = med_cv_gen / n_cols
str_med_cv_gen = str(med_cv_gen)
str_med_cv_gen = str_med_cv_gen[0:6]
medsd_file.write('%s' % ('\n'))
medsd_file.close()

# input standardization

# standardization on max

if tipo_norm == 'M':
  index = 0
  while index < n_rows:
    arr_c = arr_r[index] 
    index_c = 1 
    while index_c <= n_cols:    ## aggiornare anche kb_cla.py
      if xmaxs[index_c] == 0.0:
        xmaxs[index_c] = 0.00001 
      arr_c[index_c] = arr_c[index_c] / xmaxs[index_c] 
      index_c += 1 
    index += 1 

# standardization on std

if tipo_norm == 'S':
  index = 0
  while index < n_rows: 
    arr_c = arr_r[index] 
    index_c = 1 
    while index_c <= n_cols:    
      if xsds[index_c] == 0.0:
        xsds[index_c] = 0.00001 
      arr_c[index_c] = (arr_c[index_c] - xmeans[index_c]) / xsds[index_c]
      if arr_c[index_c] < xmins[index_c]:	### aggiunto Roberto 4/03/2012
        xmins[index_c] = arr_c[index_c]       	### aggiunto Roberto 4/03/2012
      index_c += 1 
    index += 1 
  # aggiungo xmins per eliminare i valori negativi (aggiunto da Roberto 4/03/2012)
  index = 0
  while index < n_rows: 
    arr_c = arr_r[index] 
    index_c = 1 
    while index_c <= n_cols:    
      arr_c[index_c] = arr_c[index_c] - xmins[index_c]
      print arr_c[index_c]
      index_c += 1 
    index += 1 
  # fine aggiunta da Roberto 4/03/2012

# start of kohonen algorithm 

# min and max vectors

vmaxs = []
vmins = []

index_c = 0

while index_c <= n_cols:
  vmaxs.append(-10000000000000.0) 
  vmins.append( 10000000000000.0) 
  index_c += 1 

# columns min & max 
index = 0
while index < n_rows:
  arr_c = arr_r[index] 
  index_c = 1 
  while index_c <= n_cols:
    if arr_c[index_c] > vmaxs[index_c]:
      vmaxs[index_c] = arr_c[index_c] 
    if arr_c[index_c] < vmins[index_c]:
      vmins[index_c] = arr_c[index_c] 
    index_c += 1 
  index += 1 

# run parameters and temp arrays
  
n = n_rows
m = n_cols
nx = gruppi_num 
ny = gruppi_num 
ix = 950041                         # integer as random seed
nsteps = int(10000 * nx * ny)       # number of steps
nepoks = int(nsteps / n ** 0.5)     # number of epochs
unit_calc = int(n * m * nx * ny)    # running units
passo = int(5000 / n)               # step of visualization on monitor
rmax = nx - 1
rmin = 1.0 

if passo < 1:
  passo = 1 
grid = []			    # training grid
index = 0
while index < nx * ny * m:
  grid.append(0.0)
  index += 1
n=ndim(nx,ny,m,3)
random.seed(ix)	       		    # initial value of random seed to obtain the same sequences in new runs
index = 0 
while index < nx: 
  index_c = 0 
  while index_c < ny:
    index_k = 0 
    while index_k < m: 
      ig = n.getcellindex((index,index_c,index_k))
      grid[ig] = random.random()
      index_k += 1 
    index_c += 1 
  index += 1 
gridp = copy.deepcopy(grid)     # initial previous grid = current grid 
gridm = copy.deepcopy(grid)     # initial min grid = current grid 

# for each record in each epoch 
iter 	  = 0 
discrea   = 1000000000000.0 	# current error
discrep   = 0.0 		# previous error
if nepoks < 20: 
  nepoks  = 20 			# min epochs = 20
nepokx    = 0
min_epok  = 0          		# epoch with min error
min_err   = 1000000000.0    	# min error
alpha     = float(alpha_max)   	# initial value of alpha parameter
ir        = 0.0                 # initial value of ir parameter ir
ne        = 1 

print " " 
print 'Record ' + str(n_rows) + ' Columns ' + str(n_cols)

# main loop
try:
  while ne <= nepoks: 
    if (ne % passo == 0):  # print running message when modulo division = zero
      min_err_txt = "%14.5f" % min_err    # format 8 integers and 3 decimals 
      alpha_txt  = "%12.5f" % alpha       # format 6 integers and 5 decimals 
      print ('Epoch ' + str(ne) + '   min err ' + min_err_txt + '   min epoch ' + 
          str(min_epok - 1) + "   alpha " + alpha_txt)
    if min_err < 1000000000.0: 
      nepokx += 1 
    if min_err > discrea and discrep > discrea and discrea > 0.0: 
      min_epok = ne               # current epoch (min) 
      min_err = discrea 
      # copy current grid to min grid
      gridm = copy.deepcopy(grid)       
      min_err_txt = "%12.3f" % min_err    # format 8 integers and 3 decimals 
      alpha_txt  = "%12.5f" % alpha       # format 6 integer and  5 decimals 
      print ('**** Epoch ' + str(ne - 1) + '       WITH MIN ERROR ' + min_err_txt + 
         "   alpha " + alpha_txt)
      
    # cheking the current value of alpha 
    if alpha > alpha_min:
      discrea = discrep 
      discrep = 0.0 
      # copy current grid to previous grid
      gridp = copy.deepcopy(grid)       
  
      # from the starting row to the ending row
      i = 0 
      while i < n_rows:
        iter += 1 
        # find the best grid coefficient
        ihit = 0 
        jhit = 0 
        dhit = 100000.0 
        igx = 0
        igy = 0
        while igx < nx: 
          igy = 0
          while igy < ny: 
            d = 0.0 
            neff = 0 
            k = 0
            arr_c = arr_r[i] 
            while k < m:   # update the sum of squared deviation of input 
                           # value from the grid coefficient
              ig = n.getcellindex((igx,igy,k))
              d = d + (arr_c[k+1] - grid[ig]) ** 2 
              k += 1 
            d = d / float(m)
            #  d = d / m 
            if d < dhit:
              dhit = d 
              ihit = int(igx)
              jhit = int(igy)
            igy += 1 
          igx += 1 
        # update iteration error
        discrep = discrep + dhit 
        # now we have the coordinates of the best grid coefficient
        ir = max(rmax * float(1001 - iter) / 1000.0 + 0.9999999999 , 1) 
        ir = int(ir) 
        # new alpha value to increase the radius of groups proximity 
        alpha = max(alpha_max * float(1 - ne * alpha_step) , alpha_min)
        # update the grid coefficients applying alpha parameter
        inn0 = int(ihit) - int(ir) 
        inn9 = int(ihit) + int(ir)
        jnn0 = int(jhit) - int(ir)
        jnn9 = int(jhit) + int(ir)
        while inn0 <= inn9: 
          jnn0 = int(jhit) - int(ir)
          while jnn0 <= jnn9:
            if not (inn0 < 0 or inn0 >= nx):
              if not (jnn0 < 0 or jnn0 >= ny): 
                arr_c = arr_r[i] 
                k = 0 
                while k < m:
                  ig = n.getcellindex((inn0,jnn0,k))
                  grid[ig] += alpha * (arr_c[k+1] - grid[ig])
                  k += 1 
            jnn0 += 1 
          inn0 += 1 
        i += 1 
    else: 
      print
      print "Min alpha reached " 
      print
      break   
    ne += 1 
except KeyboardInterrupt:
  print
  print "KeyboardInterrupt (Ctrl/C) " 
  print
  pass

# computing results
# grid = grid min 
grid = copy.deepcopy(gridm)       

# write min grid file 
arch_grid_file = open(arch_grid, 'w')
ii = 0 
while ii < nx:
  j = 0
  while j < ny:
    k = 0
    while k < m:
      ig = n.getcellindex((ii,j,k))
      arch_grid_file.write('%6i %s %.6i %s %.6i %s %14.7f %s' % (ii,' ', j ,' ', k,' ', grid[ig], "\n"))
      k += 1
    j += 1
  ii += 1
arch_grid_file.close()

# catalog input by min grid
ii = 0
while ii < n_rows: 
  ihit = 0 
  jhit = 0 
  dhit = 100000.0 
  # from 1 to numbers of groups
  ir = 0
  while ir < nx:         # from 1 to numbers of groups 
    jc = 0
    while jc < ny:       # from 1 to numbers of groups 
      d = 0.0 
      neff = 0 
      k = 0
      while k < n_cols:  # update the sum of squared deviation of input 
                         # value from the grid coefficient
        arr_c = arr_r[ii] 
        ig = n.getcellindex((ir,jc,k))
        d = d + (arr_c[k+1] - grid[ig]) ** 2 
        k += 1 
      d = d / m 
      if d < dhit:       # save the coordinates of the best coefficient 
        dhit = d 
        ihit = ir 
        jhit = jc 
      jc += 1 
    ir += 1 
  mtchx.append(ihit)
  mtchy.append(jhit) 
  ii += 1 

# write arch_catal file 
arch_catal_file = open(arch_catal, 'w')
ii = 0 
while ii < n_rows: 
  arch_catal_file.write("%.6i %s %.6i %s %.6i %s" % (ii, ' ', mtchx[ii], ' ', mtchy[ii], "\n")) 
  ii += 1 
arch_catal_file.close()

# matrix of statistics 
arr_cv   = []   	       # CV array of the Groups and Total
arr_med  = []      	       # means array of the Groups
riga_cv  = []          	       # CV row in arr_cv 
arr_col  = []          	       # group temporary array
arr_grsg = []         	       # input data array (normalized)
arr_grsg_c = []                # copy of arr_grsg (for file out sort) 

# input matrix sort in group sequence
ii = 0
ix = 0
while ii < n_rows: 
  ix += 1
  gr1 = str(mtchx[ii])
  if mtchx[ii] < 10:
    gr1 = '0' + str(mtchx[ii])
  sg1 = str(mtchy[ii])
  if mtchy[ii] < 10: 
    sg1 = '0' + str(mtchy[ii])
  riga_norm = arr_r[ii]
  im = 0
  riga_norm1 = []
  while im <= m: 
    riga_norm1.append(str(riga_norm[im]))
    im += 1
  riga_norm2 = " ".join(riga_norm1)  
  gr_sg_txt = "G_" + gr1 + "_" + sg1 + " " + str(ix) + " " + riga_norm2
  arr_grsg.append(gr_sg_txt) 
  ii += 1 
arr_grsg.sort()
ii = 0 
while ii < n_rows:
  arr_grsg_c.append(arr_grsg[ii]) 
  ii += 1 

# setup of arr_cv matrix 
num_gr = 0 
gruppo0 = "" 
ir = 0
while ir < n_rows:
  grsg_key = arr_grsg_c[ir].split()
  if not grsg_key[0] == gruppo0:
    gruppo0 = grsg_key[0]
    num_gr +=1 
    ic = 1 
    riga1 = [] 
    riga1.append(grsg_key[0]) 
    while ic <= m + 2:          # adding new columns for row mean and  n° of records
      riga1.append(0.0) 
      ic += 1 
    arr_cv.append(riga1)        # cv row 
  ir += 1 
riga1 = []
riga1.append("*Means*") 	# adding new row for cv mean
ic = 1
while ic <= m + 2:          	# adding new column for row mean and n° of records
  riga1.append(0.0) 
  ic += 1 
arr_cv.append(riga1)

def found(x):
  ir = 0
  while ir < len(arr_cv):
    linea_cv = arr_cv[ir]
    key_cv = linea_cv[0]
    if key_cv == x:
      return ir
    ir += 1

ir  = 0
irx = len(arr_grsg_c)
ic  = 3
linea_cv = arr_cv[0]
icx = len(linea_cv)
val_col = []

while ic < icx:
  ir = 0
  gruppo  = ""
  val_col = []
  while ir < irx:
    linea = arr_grsg_c[ir].split()
    if linea[0] == gruppo or gruppo == "":
      gruppo = linea[0] 
      val_col.append(float(linea[ic]))
    else:
      i_gruppo = found(gruppo)
      linea_cv = arr_cv[i_gruppo]
      media_v = abs(mean(val_col))
      if media_v == 0.0:
         media_v = 0.0000000001
      std_v = sd(val_col)
      cv_v  = std_v / media_v
      linea_cv[ic-2] = cv_v                      # cv value
      linea_cv[len(linea_cv)-1] = len(val_col)   # number of records
      val_col = []
      val_col.append(float(linea[ic]))
      gruppo = linea[0]
    ir += 1
  i_gruppo = found(gruppo)
  linea_cv = arr_cv[i_gruppo]
  media_v = abs(mean(val_col))
  if media_v == 0.0:
    media_v = 0.0000000001
  std_v = sd(val_col)
  cv_v  = std_v / media_v
  linea_cv[ic-2] = cv_v                          # cv value
  linea_cv[len(linea_cv)-1] = len(val_col)       # number of records
  ic += 1
ir  = 0
irx = len(arr_cv)
linea_cv = arr_cv[0]
icx = len(linea_cv) - 2
ic  = 1
num_rec1 = 0

while ir < irx:                                  # rows mean
  media_riga = 0.0
  ic = 1
  num_col1 = 0 
  linea_cv = arr_cv[ir]
  while ic < icx:
    media_riga += float(linea_cv[ic])
    num_col1 += 1
    ic += 1
  linea_cv[icx] = media_riga / num_col1
  num_rec1 += linea_cv[icx + 1]
  ir += 1
ir  = 0
ic  = 1

while ic < icx:                  # weighted mean of columns
  media_col = 0.0
  ir = 0
  num_rec1 = 0 
  while ir < irx - 1:
    linea_cv = arr_cv[ir]
    media_col = media_col + linea_cv[ic] * linea_cv[icx+1]  # linea_cv[icx+1] = number of records
    num_rec1 = num_rec1 + linea_cv[icx+1]
    ir += 1
  linea_cv = arr_cv[irx - 1]
  linea_cv[ic] = media_col / num_rec1
  ic += 1

# updating mean of the row
linea_cv = arr_cv[irx - 1]
linea_means = linea_cv[1:icx]
media_riga  = mean(linea_means)
linea_cv[icx] = media_riga        # Total mean
linea_cv[icx + 1] = num_rec1      # n° of records
cv_media_gen_after = str(media_riga)
cv_media_gen_after = cv_media_gen_after[0:6]  

# write cv  file 
testata_cv = testata
testata_cv[0] = "*Groups*"
testata_cv.append("*Mean*")
testata_cv.append("N_recs")
arch_cv_file = open(arch_cv, 'w')
ic = 0
while ic <= icx + 1:
  arch_cv_file.write('%s %s ' % (testata_cv[ic], " "*(9-len(testata_cv[ic]))))
  ic += 1
arch_cv_file.write('%s' % ('\n'))
ir = 0 
while ir < irx:
  ic = 0
  linea_cv = arr_cv[ir]
  while ic <= icx + 1:
    if ic == 0:
      arch_cv_file.write('%s %s ' % (linea_cv[0], "  "))
    else:
      if ic <= icx:
        arch_cv_file.write('%7.4f %s ' % (linea_cv[ic], "  "))
      else:
        arch_cv_file.write('%6i %s ' % (linea_cv[ic], "  "))
    ic += 1
  arch_cv_file.write('%s' % ("\n"))
  ir += 1
ic = 0

media_xcv = mean(xcv[1:icx])

while ic <= icx :   # print CV input (before catalogue)  
  if ic == 0:
    arch_cv_file.write('%s %s ' % ("*CVinp*", "  "))
  else:
    if ic < icx:
      arch_cv_file.write('%7.4f %s ' % (xcv[ic], "  "))
    else:
      arch_cv_file.write('%7.4f %s ' % (media_xcv, "  "))
      arch_cv_file.write('%6i %s ' % (linea_cv[ic+1], "  "))
  ic += 1
arch_cv_file.write('%s' % ("\n"))
#=========istruzioni aggiunte Roberto Bello 29/02/2012======================
#know_index = str(1.0 - float(cv_media_gen_after) / float(str_med_cv_gen))	
#know_index = know_index[0:6]
#arch_cv_file.write('%s %s %s' % ('*KIndex*   ', know_index, '\n'))
#=========fine istruzioni aggiunte da Roberto Bello 29/02/2012==============
arch_cv_file.close()

# writing out catalog file 
testata_cat1 = []
testata_cat1.append("*Group*")
arch_output_file = open(arch_output, 'w')
ic= 0
while ic < icx:
  testata_cat1.append(testata_cat[ic])
  ic += 1
ic= 0
while ic < len(testata_cat1):
  arch_output_file.write('%s %s ' % (testata_cat1[ic], " "*(15-len(testata_cat1[ic]))))
  ic += 1
arch_output_file.write('%s' % ("\n"))
index = 0 
while index < len(arr_orig):
  riga_orig = arr_orig[index]
  ic = 0
  while ic < len(riga_orig):
    if not(isinstance(riga_orig[ic],str)):
      riga_orig[ic] = str(riga_orig[ic])
    ic += 1   
  # place before 0 if gr / sg < 10 
  gr1 = str(mtchx[index])
  if mtchx[index] < 10:
    gr1 = '0' + str(mtchx[index])
  sg1 = str(mtchy[index])
  if mtchy[index] < 10: 
    sg1 = '0' + str(mtchy[index])
  arr_rig0 = "G_" + gr1 + "_" + sg1 + " "*8
  arch_output_file.write('%s ' % (arr_rig0))
  ic= 0
  while ic < len(riga_orig):
    arch_output_file.write('%s %s ' % (riga_orig[ic], " "*(15-len(riga_orig[ic]))))
    ic += 1
  arch_output_file.write('%s' % ("\n"))
  index += 1 
testata_cat1 = []
testata_cat1.append("*Group*")
testata_cat1.append("*RecNum*")
arch_sort_file = open(arch_sort, 'w')
ic= 0
while ic < icx:
  testata_cat1.append(testata_cat[ic])
  ic += 1
ic= 0
while ic < len(testata_cat1):
  arch_sort_file.write('%s %s ' % (testata_cat1[ic], " "*(15-len(testata_cat1[ic]))))
  ic += 1
arch_sort_file.write('%s' % ("\n"))
index = 0 
while index < len(arr_grsg_c):
  riga_grsg = arr_grsg_c[index].split()
  ic = 0
  while ic < len(riga_grsg):
    val_txt = riga_grsg[ic]
    val_txt = val_txt[0:13]
    arch_sort_file.write('%s %s ' % (val_txt, " "*(15-len(val_txt))))
    ic += 1
  if index < len(arr_grsg_c) - 1:
    arch_sort_file.write('%s' % ("\n"))
  index += 1 
arch_sort_file.close()

# writing out catalog and sorted file
arr_outsrt = []
index = 0
while index < len(arr_orig):
  riga_sort = []
  # place before 0 if gr / sg < 10 
  gr1 = str(mtchx[index])
  if mtchx[index] < 10:
    gr1 = '0' + str(mtchx[index])
  sg1 = str(mtchy[index])
  if mtchy[index] < 10: 
    sg1 = '0' + str(mtchy[index])
  riga_sort.append("G_" + gr1 + "_" + sg1)
  ic = 0
  riga_orig = arr_orig[index]
  while ic < len(riga_orig):
    val_riga = riga_orig[ic]
    riga_sort.append(val_riga)
    ic += 1
  arr_outsrt.append(riga_sort)
  index += 1

for line in arr_outsrt:
  line = "".join(line)  

arr_outsrt.sort()

testata_srt = []
testata_srt.append("*Group*")
arch_outsrt_file = open(arch_outsrt, 'w')
ic= 0
while ic < icx:
  testata_srt.append(testata_orig[ic])
  ic += 1
ic= 0
while ic < len(testata_srt):
  arch_outsrt_file.write('%s %s' % (testata_srt[ic], " "*(15-len(testata_srt[ic]))))
  ic += 1
arch_outsrt_file.write('%s' % ("\n"))
index = 0 
key_gruppo = "" 
while index < len(arr_outsrt):
  riga_sort = arr_outsrt[index] 
  index_c = 0 
  while index_c < len(riga_sort):
    if index_c == 0: 
      if riga_sort[0] != key_gruppo:
        # arch_outsrt_file.write('%s ' % ("\n"))
        key_gruppo = riga_sort[0] 
    valore = riga_sort[index_c] 
    arch_outsrt_file.write('%s %s' % (valore, " "*(15-len(valore))))
    index_c += 1 
  if index < len(arr_grsg_c) - 1:
    arch_outsrt_file.write('%s' % ("\n"))
  index += 1 
arch_outsrt_file.close()

print("###############################################################################")
print("# KB_CAT KNOWLEDGE DISCOVERY IN DATA MINING (CATALOG PROGRAM)                 #")
print("# by ROBERTO BELLO (COPYRIGHT MARCH 2011 ALL RIGHTS RESERVED)                 #")
print("# Language used: PYTHON                                                       #")
print("###############################################################################") 

arch_log_file = open(arch_log, 'w')
arch_log_file.write("%s %s" % ("############################################################################", "\n"))
arch_log_file.write("%s %s" % ("# KB_CAT KNOWLEDGE DISCOVERY IN DATA MINING (CATALOG PROGRAM)              #", "\n")) 
arch_log_file.write("%s %s" % ("# by ROBERTO BELLO (COPYRIGHT MARCH 2011 ALL RIGHTS RESERVED)              #", "\n")) 
arch_log_file.write("%s %s" % ("# Language used: PYTHON                     .                              #", "\n")) 
arch_log_file.write("%s %s" % ("############################################################################", "\n")) 
arch_log_file.write("%s %s %s" % ("Input File                                        -> ", file_input, "\n"))
arch_log_file.write("%s %s %s" % ("Numer of Groups (3 - 20)                          -> ", str(gruppi_num), "\n")) 
arch_log_file.write("%s %s %s" % ("Normalization (Max, Std, None)                    -> ", tipo_norm, "\n")) 
arch_log_file.write("%s %s %s" % ("Start Value of alpha (from 1.8 to 0.9)            -> ", str(alpha_max), "\n")) 
arch_log_file.write("%s %s %s" % ("End Value of alpha (from 0.5 to 0.0001)           -> ", str(alpha_min), "\n")) 
arch_log_file.write("%s %s %s" % ("Decreasing step of alpha (from 0.1 to 0.001)      -> ", str(alpha_step), "\n")) 
arch_log_file.write("%s"       % ("=========================OUTPUT=======================================================\n")) 
arch_log_file.write("%s %s %s" % ("Output File Catalog.original     ", arch_output, "\n"))
arch_log_file.write("%s %s %s" % ("Output File Catalog.sort         ", arch_outsrt, "\n")) 
arch_log_file.write("%s %s %s" % ("Output File Summary sort         ", arch_sort, "\n")) 
arch_log_file.write("%s %s %s" % ("Output File Matrix Catal.        ", arch_catal, "\n")) 
arch_log_file.write("%s %s %s" % ("Output File Means, STD, CV.      ", arch_medsd, "\n"))
arch_log_file.write("%s %s %s" % ("Output File CV of the Groups     ", arch_cv, "\n"))
arch_log_file.write("%s %s %s" % ("Output File Training Grid        ", arch_grid, "\n")) 
arch_log_file.write("%s %s %s" % ("Output File Run Parameters       ", arch_log, "\n"))
#=========istruzioni aggiunte Roberto Bello 29/02/2012======================
know_index = str(1.0 - float(cv_media_gen_after) / float(str_med_cv_gen))	
know_index = know_index[0:6]
arch_log_file.write('%s %s %s' % ('*KIndex*   ', know_index, '\n'))
#=========fine istruzioni aggiunte da Roberto Bello 29/02/2012==============

min_err_txt = "%12.3f" % min_err      # format 8 integer and 3 decimals 
alpha_txt  = "%12.5f" % alpha         # format 6 integer and 5 decimals 
alpha_min_txt = "%12.5f" % alpha_min  # format 6 integer and 5 decimals 

print 
if min_err == 1000000000.000:
  print("Oops! No result. Try again with new alpha parameters")
print 
print ("EPOCH " + str(min_epok -1) + "   WITH MIN ERROR " + min_err_txt + 
  " starting alpha " + alpha_min_txt + "   ending alpha " + alpha_txt + 
  " Iterations " + str(iter) + " Total Epochs " + str(ne - 1))
print 
print 'Output File Catalog.original ' + arch_output 
print 'Output File Catalog.sort     ' + arch_outsrt 
print 'Output File Summary sort     ' + arch_sort 
print 'Output File Matrix Catal.    ' + arch_catal 
print 'Output File Means, STD, CV.  ' + arch_medsd 
print 'Output File CV of the Groups ' + arch_cv 
print 'Output File Training Grid    ' + arch_grid 
print 'Output File Run Parameters   ' + arch_log 
print 'CV before Catalog            ' + str_med_cv_gen
print 'CV after Catalog             ' + cv_media_gen_after
know_index = str(1.0 - float(cv_media_gen_after) / float(str_med_cv_gen))
know_index = know_index[0:6]
print 'Knowledge Index              ' + know_index
print 

# Elapsed time
t1 = datetime.datetime.now()
elapsed_time = t1 - t0
print "Elapsed time (seconds)   :   " + str(elapsed_time.seconds)
print 

      
Tags: data_mining, neural_network, python
◄	Python recipes (4591)	►
◄	Roberto Bello's recipes (3)	►
KB Neural Data Mining - Python sources (Python recipe) by Roberto Bello
ActiveState Code (http://code.activestate.com/recipes/578853/)

Tags

Required Modules

Other Information and Tasks

Accounts

Code Recipes

Feedback & Information

ActiveState

KB Neural Data Mining - Python sources (Python recipe) by Roberto Bello ActiveState Code (http://code.activestate.com/recipes/578853/)
