Python-сообщество

 begin
end extract
Traceback (most recent call last):
  File "C:\!Work\Lab8\lab8 — копия.py", line 185, in <module>
    X=evaluate_X(D)
  File "C:\!Work\Lab8\lab8 — копия.py", line 92, in evaluate_X
    X.append(ex[1:])
MemoryError
>>>

 #!/usr/bin/python
import csv
import os
import math
import sys
import random
import numpy as np
from svmutil import*
import collections
def extract_data_from_file(data_path):
        fin = open(data_path, "rb")
        #row = fin.readline()
        data_extract = []
        for l in fin:
                line = l.split()
                float_line = []
                j=1
                float_line.append(float(line[0]))
                data_extract.append(float_line)
                for e in line[1:]:
                        line1=e.decode().split(":")
                        while(e[0]>j):
                                float_line.append(float(0))
                                j=j+1
                                float_line.append(float(e[1]))
                                data_extract.append(float_line)
                
                #row = fin.readline()
        return data_extract
def transform_datas(data):
        datas_transformed = []
        for point in data:
                data = []
                data.append(1)
                data.append(point[0])
                data.append(point[1])
                data.append(point[0]**2)
                data.append(point[1]**2)
                data.append(point[0]*point[1])
                data.append(abs(point[0]-point[1]))
                data.append(abs(point[0]+point[1]))     
                datas_transformed.append((data,point[2]))
        return datas_transformed
def linear_regression_training(N):
        X,y = zip(*N)
        X_d = np.linalg.pinv(X)
        w = np.dot(X_d,y)
        return w
def linear_regression_training_weight_decay(N, k):
        X,y = zip(*N)
        reg = np.identity(len(X[0]))*(10**k)
        X = np.matrix(X)
        y = np.matrix(y)
        X_t = np.transpose(X)
        w = (np.linalg.inv( (X_t*X + reg) ) * X_t )*np.transpose(y)
        return w
def evaluate_g(ex,w):
        r = sum(ex_i*w_i for ex_i,w_i in zip(ex,w))
        if r > 0:
                return float(1)
        elif r < 0:
                return float(-1)
def evaluate_f(X,digit):
        y=[]
        for ex in X:
                if ex[0]==digit:
                        y.append(1)
                      
                else:
                        y.append(-1)
        return y
def evaluate_f_cut(X,digit1,digit2):
        y=[]
        for ex in X:
                if ex[0]==digit1:
                        y.append(1)
                      
                elif ex[0]==digit2:
                        y.append(-1)
        return y
def evaluate_X(data):
        X=[]
        for ex in data:
                X.append(ex[1:])
        
        return X
def Cut_data(data,digit1,digit2):
        X=[]
        for ex in data:
                if((ex[0]==digit1) or (ex[0]==digit2)):X.append(ex)
        
        return X
def calculate_E(points, w):
        X,y = zip(*points)
        nWrong = 0
        for i in xrange(len(points)):
                if y[i] != evaluate_g(X[i],w):
                        nWrong += 1
        pWrong = float(nWrong) / len(points)
        return pWrong
def BinaryError(f,g):
        err=0
        for i in range(len(f)):
                if (f[i]!=g[i]): err=err+1
        return err/len(f)
def calculate_weight_decay(weights, N, k):
        l = 10**k
        return ((l/N)*sum(w*w for w in weights))
def Rand_part(D,nomer):
              N=[]
              for j in range(len(D)):
                      if(j<(nomer-1)*len(D)/10 or j>=nomer*len(D)/10+1):
                            N.append(D[j])  
             
              return N
print('begin')
in_data_path='webspam_wc_normalized_unigram.txt'
out_data_path='test.txt'
datas_train = extract_data_from_file(in_data_path)
##datas_train_transformed = transform_datas(datas_train)
##datas_test = extract_data_from_file(out_data_path)
##datas_test_transformed = transform_datas(datas_test)
print('end extract')
#ex2-4
##X=evaluate_X(datas_train)
##j=0
##while (j<10):
##        digit=j
##        y=evaluate_f(datas_train,digit)
##        prob = svm_problem(y,X)
##        options='-s 0 -t 2 -d 2 -c 0.01 -g 1 -r 1'
##        param = svm_parameter(options)
##    
##        m=svm_train(prob,param)
##        p_labels, p_acc, p_vals = svm_predict(y,X, m)
##        E_in=BinaryError(y,p_labels)
##        print(j,E_in, len(m.get_SV()) )
##        
##        j=j+2
#ex2-4
#ex5-6
##digit1=1
##digit2=5
##D=Cut_data(datas_train,digit1,digit2)
##X=evaluate_X(D)
##y=evaluate_f(D,digit1)
##print(len(X),len(y))   
##datas_test = extract_data_from_file(out_data_path)
##D=Cut_data(datas_test,digit1,digit2)
##X_test=evaluate_X(D)
##y_test=evaluate_f(D,digit1)
##print(len(X_test),len(y_test))       
##C=0.0001
##print('-s 0 -t 1 -d 5 -c '+str(C)+' -g 1 -r 1')
##while (C<1.1):
##        
##        prob = svm_problem(y,X)
##        options1='-s 0 -t 1 -d 5 -c '+str(C)+' -g 1 -r 1'
##        param1 = svm_parameter(options1)
##        m1=svm_train(prob,param1)
##        p_labels, p_acc, p_vals = svm_predict(y,X, m1)
##        E_in=BinaryError(y,p_labels)
##        p_labels_t, p_acc_t, p_vals_t = svm_predict(y_test,X_test, m1)
##        E_out=BinaryError(y_test,p_labels_t)
##        print('c=',C,'E_in',E_in,'E_out=',E_out,'SV=', len(m1.get_SV()) )
##        C=C*10
        
#ex5-6
#ex7-8
digit1=1
digit2=5
D=Cut_data(datas_train,digit1,digit2)
X=evaluate_X(D)
y=evaluate_f(D,digit1)
##datas_test = extract_data_from_file(out_data_path)
##D=Cut_data(datas_test,digit1,digit2)
##X_test=evaluate_X(D)
##y_test=evaluate_f(D,digit1)
##print(len(X_test),len(y_test))
C=0.0001
Q=[]
D1=[]
print(len(D))
E_cv=0
C_list=[]
nom=[]
i=0
for i in range(100):
        j=0
        while (j<=round(len(D)/10)):
                s=random.randint(1,len(D))
                if ((s in nom)!=True):
                        nom.append(s)
                        j=j+1
        j=0
        while (j<len(D)):
              if(j in nom):
                    Q.append(D[j])
              else: D1.append(D[j])  
              j=j+1  
       
        #print(len(D1),len(Q))
        X=evaluate_X(D1)
        y=evaluate_f(D1,digit1)
        X_val=evaluate_X(Q)
        y_val=evaluate_f(Q,digit1)
        
        #print('-s 0 -t 1 -d 2 -c '+str(C)+' -g 1 -r 1')
        prob = svm_problem(y,X)
        C=0.001
        minimum=1
        while (C<0.002):
                options1='-s 0 -t 1 -d 2 -c '+str(C)+' -g 1 -r 1'
                param1 = svm_parameter(options1)
                m1=svm_train(prob,param1)
                p_labels_t, p_acc_t, p_vals_t = svm_predict(y_val,X_val, m1)
                E_cv=E_cv+BinaryError(y_val,p_labels_t)
                #print('c=',C,'E_cv=',E_cv,'SV=', len(m1.get_SV()))
##                if(E_cv<minimum):
##                        C_min=C
##                        minimum=E_cv
                C=C*10
        #C_list.append(C_min)
        Q.clear()
        D1.clear()
        nom.clear()
print('E_cv=',E_cv/100)
#cc=collections.Counter(C_list)
#print('0.0001=',cc[0.0001],'0.001=',cc[0.001],'0.01=',cc[0.01],'0.1=',cc[0.1],'1=',cc[1.0],)
               
#ex7-8
###ex9-10
##digit1=1
##digit2=5
##D=Cut_data(datas_train,digit1,digit2)
##X=evaluate_X(D)
##y=evaluate_f(D,digit1)
##print(len(X),len(y))   
##datas_test = extract_data_from_file(out_data_path)
##D=Cut_data(datas_test,digit1,digit2)
##X_test=evaluate_X(D)
##y_test=evaluate_f(D,digit1)
##print(len(X_test),len(y_test))       
##C=0.01
##print('-s 0 -t 2 -c '+str(C)+' -g 1')
##while (C<1000001):
##        
##        prob = svm_problem(y,X)
##        options1='-s 0 -t 2  -c '+str(C)+' -g 1'
##        param1 = svm_parameter(options1)
##        m1=svm_train(prob,param1)
##        p_labels, p_acc, p_vals = svm_predict(y,X, m1)
##        E_in=BinaryError(y,p_labels)
##        p_labels_t, p_acc_t, p_vals_t = svm_predict(y_test,X_test, m1)
##        E_out=BinaryError(y_test,p_labels_t)
##        print('c=',C,'E_in',E_in,'E_out=',E_out,'SV=', len(m1.get_SV()) )
##        C=C*100
##        
#ex9-10
##datas_test = extract_data_from_file(out_data_path)
##X_test=evaluate_X(datas_test)
##y_test=evaluate_f(datas_test,digit)
##p_labels, p_acc, p_vals = svm_predict(y_test,X_test, m)

 a = ....

  
>>> def f1():
...     out = []
...     for i in 'abcd':
...         out.append(i)
...     return out
... 
>>> def f2():
...     for i in 'abcd':
...         yield i
... 
>>> f1()
['a', 'b', 'c', 'd']
>>> f2()
<generator object f2 at 0xb73bf57c>
>>> list(f2())
['a', 'b', 'c', 'd']
>>>