【kaggle】【quora insincere questions classification】1 模型融合

from collections import defaultdict, Counter
from glob import glob
import sys
import re
glob_files = sys.argv[1]
loc_outfile = sys.argv[2]
weights_strategy = "uniform"
if len(sys.argv) == 4:
weights_strategy = sys.argv[3]
def kaggle_bag(glob_files, loc_outfile, method="average", weights="uniform"):
     pattern = re.compile(r"(.)*_[w|W](\d*)_[.]*")   
     if method == "average":
         scores = defaultdict(list)  
     with open(loc_outfile,"wb") as outfile:        
     #weight_list may be usefull using a different method
         weight_list = [1]*len(glob(glob_files))  
      for i, glob_file in enumerate( glob(glob_files) ):           
          print("parsing:", glob_file)        
          if weights == "weighted":
                 weight = pattern.match(glob_file)  
                if weight and weight.group(2):                
                        print("Using weight: ",int(weight.group(2)))
                        weight_list[i] = weight_list[i]*int(weight.group(2))    
                  else:                
                         print("Using weight: 1")        
       # sort glob_file by first column, ignoring the first line
       lines = open(glob_file).readlines()
        lines = [lines[0]] + sorted(lines[1:])      
       for e, line in enumerate( lines ):            
              if i == 0 and e == 0:
                    outfile.write(line)       
              if e > 0:
                   row = line.strip().split(",")   
              for l in range(1,weight_list[i]+1):
                   scores[(e,row[0])].append(row[1])  
         for j,k in sorted(scores):
              outfile.write("%s,%s\n"%(k,Counter(scores[(j,k)]).most_common(1)[0][0]))
          print("wrote to %s"%loc_outfile)

from collections import defaultdict
from glob import glob
import sys
glob_files = sys.argv[1]
loc_outfile = sys.argv[2]
def kaggle_bag(glob_files, loc_outfile, method="average", weights="uniform"):
    if method == "average":
        scores = defaultdict(float)
    with open(loc_outfile,"wb") as outfile:
        for i, glob_file in enumerate( glob(glob_files) ):
            print("parsing:", glob_file)
        # sort glob_file by first column, ignoring the first line
       lines = open(glob_file).readlines()
        lines = [lines[0]] + sorted(lines[1:])  
       for e, line in enumerate( lines ):
           if i == 0 and e == 0:
                outfile.write(line)
           if e > 0:
                row = line.strip().split(",")
                scores[(e,row[0])] += float(row[1])
   for j,k in sorted(scores):
        outfile.write("%s,%f\n"%(k,scores[(j,k)]/(i+1)))
   print("wrote to %s"%loc_outfile)

from __future__ import division
from collections import defaultdict
from glob import glob
import sysimport math
glob_files = sys.argv[1]
loc_outfile = sys.argv[2]
def kaggle_bag(glob_files, loc_outfile, method="average", weights="uniform"):
     if method == "average":
         scores = defaultdict(float)  
      with open(loc_outfile,"wb") as outfile:
           for i, glob_file in enumerate( glob(glob_files) ):
                print("parsing:", glob_file)
        # sort glob_file by first column, ignoring the first line
        lines = open(glob_file).readlines()
        lines = [lines[0]] + sorted(lines[1:])
        for e, line in enumerate( lines ):
            if i == 0 and e == 0:
                outfile.write(line)
           if e > 0:
                row = line.strip().split(",")
               if scores[(e,row[0])] == 0:
                    scores[(e,row[0])] = 1
                scores[(e,row[0])] *= float(row[1])
       for j,k in sorted(scores):
            outfile.write("%s,%f\n"%(k,math.pow(scores[(j,k)],1/(i+1))))
       print("wrote to %s"%loc_outfile)

from __future__ import division
from collections import defaultdict
from glob import glob
import sys
glob_files = sys.argv[1]
loc_outfile = sys.argv[2]
def kaggle_bag(glob_files, loc_outfile):
with open(loc_outfile,"wb") as outfile:
     all_ranks = defaultdict(list)
     for i, glob_file in enumerate(glob(glob_files) ):
        file_ranks = []
       print("parsing:", glob_file)
        # sort glob_file by first column, ignoring the first line
        lines = open(glob_file).readlines()
        lines = [lines[0]] + sorted(lines[1:])
       for e, line in enumerate(lines):
            if e == 0 and i == 0:
                outfile.write(line)
           elif e > 0:
                r = line.strip().split(",")
                file_ranks.append((float(r[1]), e, r[0]) )
       for rank, item in enumerate(sorted(file_ranks) ):
            all_ranks[(item[1],item[2])].append(rank)

    average_ranks = []
   for k in sorted(all_ranks):
        average_ranks.append((sum(all_ranks[k])/len(all_ranks[k]),k))
    ranked_ranks = []
   for rank, k in enumerate(sorted(average_ranks)):
        ranked_ranks.append((k[1][0],k[1][1],rank/(len(average_ranks)-1)))
   for k in sorted(ranked_ranks):
        outfile.write("%s,%s\n"%(k[1],k[2]))
   print("wrote to %s"%loc_outfile)

from sklearn import cross_validation
from sklearn.metrics import log_loss, accuracy_score
import numpy as np
import pandas as pd
import random
import md5
import json
def blend_proba(clf, X_train, y, X_test, nfolds=5, save_preds="",save_test_only="", seed=300373, save_params="", clf_name="XX", generalizers_params=[], minimal_loss=0,return_score=False, minimizer="log_loss"):
    print("\nBlending with classifier:\n\t%s"%(clf))
    folds = list(cross_validation.StratifiedKFold(y, nfolds,shuffle=True,random_state=seed))    print(X_train.shape)
    dataset_blend_train = np.zeros((X_train.shape[0],np.unique(y).shape[0]))    #iterate through train set and train - predict folds
    loss = 0
    for i, (train_index, test_index) in enumerate( folds ):
       print("Train Fold %s/%s"%(i+1,nfolds))
        fold_X_train = X_train[train_index]
        fold_y_train = y[train_index]
        fold_X_test = X_train[test_index]
        fold_y_test = y[test_index]
        clf.fit(fold_X_train, fold_y_train)
        fold_preds = clf.predict_proba(fold_X_test)
        print("Logistic loss: %s"%log_loss(fold_y_test,fold_preds))
        dataset_blend_train[test_index] = fold_preds
       if minimizer == "log_loss":
            loss += log_loss(fold_y_test,fold_preds)
       if minimizer == "accuracy":
            fold_preds_a = np.argmax(fold_preds, axis=1)
            loss += accuracy_score(fold_y_test,fold_preds_a)
       #fold_preds = clf.predict(fold_X_test)

        #loss += accuracy_score(fold_y_test,fold_preds)
        if minimal_loss > 0 and loss > minimal_loss and i == 0:
           return False, False
        fold_preds = np.argmax(fold_preds, axis=1)
       print("Accuracy:      %s"%accuracy_score(fold_y_test,fold_preds))    avg_loss = loss / float(i+1)
    print("\nAverage:\t%s\n"%avg_loss)
    #predict test set (better to take average on all folds, but this is quicker)
    print("Test Fold 1/1")
    clf.fit(X_train, y)
    dataset_blend_test = clf.predict_proba(X_test)
   if clf_name == "XX":
        clf_name = str(clf)[1:3]
   if len(save_preds)>0:
        id = md5.new("%s"%str(clf.get_params())).hexdigest()
        print("storing meta predictions at: %s"%save_preds)
        np.save("%s%s_%s_%s_train.npy"%(save_preds,clf_name,avg_loss,id),dataset_blend_train)
        np.save("%s%s_%s_%s_test.npy"%(save_preds,clf_name,avg_loss,id),dataset_blend_test)
   if len(save_test_only)>0:
        id = md5.new("%s"%str(clf.get_params())).hexdigest()
        print("storing meta predictions at: %s"%save_test_only)
        dataset_blend_test = clf.predict(X_test)
        np.savetxt("%s%s_%s_%s_test.txt"%(save_test_only,clf_name,avg_loss,id),dataset_blend_test)
        d = {}
        d["stacker"] = clf.get_params()
        d["generalizers"] = generalizers_params
       with open("%s%s_%s_%s_params.json"%(save_test_only,clf_name,avg_loss, id), 'wb') as f:
          json.dump(d, f)
   if len(save_params)>0:
        id = md5.new("%s"%str(clf.get_params())).hexdigest()
        d = {}
        d["name"] = clf_name
        d["params"] = { k:(v.get_params() if "\n" in str(v) or "<" in str(v) else v) for k,v in clf.get_params().items()}
        d["generalizers"] = generalizers_params
       with open("%s%s_%s_%s_params.json"%(save_params,clf_name,avg_loss, id), 'wb') as f:
          json.dump(d, f)
   if np.unique(y).shape[0] == 2:
# when binary classification only return positive class proba
        if return_score:
           return dataset_blend_train[:,1], dataset_blend_test[:,1], avg_loss
        else:
            return dataset_blend_train[:,1], dataset_blend_test[:,1]
    else:
        if return_score:
            return dataset_blend_train, dataset_blend_test, avg_loss
        else:
            return dataset_blend_train, dataset_blend_test