#!/usr/bin/env python
# -*- coding: utf-8 -*-
from grab import Grab
import re
from grab.selector import XpathSelector
from lxml.html import fromstring
import lxml.etree
from sys import stdin
f = open('/home/oleg/Рабочий стол/Parser/proba.csv')
output = open('/home/oleg/Рабочий стол/Parser/result.csv', 'w')	
filespisok = f.readlines()
f.close()
seen = []
for i, g in enumerate(filespisok):
	url = g.split("|")[1] 
	if seen.count(url) == 0:
		seen.append(url)
		taxonomy = []
		taxonomy2 = []
		for i1, g1 in enumerate(filespisok):
			if i1 >= i:
				if url == g1.split("|")[1]:
					if taxonomy.count(g1.split("|")[0]) == 0:
						taxonomy.append(g1.split("|")[0])
					if taxonomy2.count(g1.split("|")[2]) == 0:
						taxonomy2.append(g1.split("|")[2].replace("\n",""))
		stroka = g.split("|")
		taxonomy = '%s' % ','.join(taxonomy)
		taxonomy2 = '%s' % ','.join(taxonomy2)
		stroka[0] = taxonomy
		stroka[2] = taxonomy2
		
		stroka = '%s' % '|'.join(stroka)+'|\n'
		output.write(stroka)	
		
	#print ("Обработано строк: "+str(i+1))
output.close()

for i, g in enumerate(f):

    urls = defaultdict(lambda: {'sid': set(), 'pid': set()})
    with open('proba.csv', 'r') as input_file:
        for k, s in enumerate(input_file):
            sid, url, pid = s.split(' | ')
            sid = sid.strip()
            url = url.strip()
            pid = pid.strip()
            selected = urls[url]
            selected['sid'].add(sid)
            selected['pid'].add(pid)
            if k % 65536 == 0:
                print('Processing line {}'.format(k))
    with open('result.csv', 'w') as outfile:
        for u, data in urls.items():
            outfile.write(' | '.join((','.join(data['sid']),
                                      u,
                                      ','.join(data['pid']))) + '\n')

import pandas as pd
with open('bigdata.csv') as csv:
    raw_data = csv.read()
data = map(lambda x: x.split('|'), raw_data.split('\n'))
table = pd.DataFrame(data)
result = table.groupby(1).sum()
result.to_csv(path_or_buf='result.csv', sep='|' )