# -*- coding: utf-8 -*-
"""
Created on Tue Sep 24 13:14:17 2019

@author: AMaass
"""

import glob
import json
import pandas as pd
from pandas.io.json import json_normalize
 
for filename in glob.glob(r'C:\Users\amaass\Desktop\KissmetricsDestination\EchoShip_TL_FileDump\SampleTLJsonFullBlob.json'): 
    with open(filename, encoding='utf8') as f:
        data = f.read()
    
    df_ = pd.DataFrame(index = [])
    data2 = "[" + data.replace('}', '}, ', data.count('}')-1) + "]"
    json_data = json.loads(data)
    df1 = json_normalize(json_data)    
    df1Dict = df1.to_dict()
    
    for key, val in df1Dict.items():
        val = val[0]
        print(key)
        print(val)
        print('')
    
    for key4, val4 in df1Dict.items():
        if type(val4[0]) != list:
            d0 = {key4 : val4[0]}
            print(d0)
            df1Dict.update(d0)
        
    
    for num in range(1, 14):
        for key, value in df1.items():
            value = value[0]

            ##APPENDS SELLRATEDETIALS COLUMNS TO DF1DICT
            if key == 'ServiceOptions.' +str(num) + '.SellRateDetails':
                valueDict1 = {}
                next(valueDict1.update(item1) for item1 in value if True == True)
                for key2, value2 in valueDict1.items():
                    d1 = {'Sell' + key2 + str(num) : value2}
                    df1Dict.update(d1)
                    print(key)
                    df1Dict.pop(key, None)
                    
            ##APPENDS BUYRATEDETAILS COLUMNS TO DF1DICT                    
            if key == 'ServiceOptions.' +str(num) + '.BuyRateDetails':
                valueDict2 = {}
                next(valueDict2.update(item2) for item2 in value if True == True)
                for key3, value3 in valueDict2.items():
                    d2 = {'Buy' + key3 + str(num) : value3}
                    print(d2)
                    df1Dict.update(d2)
                    df1Dict.pop(key, None)
            


    print(df1Dict['CustomerKey'][0])    
    print(type(df1Dict['CustomerKey']))
    print(df1Dict['NotificationInformation.RepresentativeContacts'][0])
    print(type(df1Dict['NotificationInformation.RepresentativeContacts'][0]))    
    
    df1Update = pd.DataFrame.from_dict(df1Dict)