FlattenData.py 6.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. Created on Wed Oct 9 15:17:34 2019
  5. @author: oskar
  6. @description: Class which flattens nested Dataframes, Dictionaries and Lists into tabular form
  7. """
  8. import sys
  9. import os
  10. import time
  11. import pandas as pd
  12. import copy
  13. sys.path.append(os.getcwd())
  14. from cdplib.log import Log
  15. class FlattenData():
  16. def __init__(self):
  17. self._log = Log("Flatten data")
  18. def flatten(self, data) -> pd.DataFrame():
  19. '''
  20. :parm data: data given in either dictionary, list or dataframe format.
  21. '''
  22. assert(isinstance(data, (list, dict, pd.DataFrame, pd.Series))),\
  23. "Parameter 'data' either be of List, Dictionary or DataFrame type"
  24. in_length=0
  25. start = time.time()
  26. if type(data) is pd.DataFrame:
  27. in_length = len(data.columns)
  28. return_data = self.flatten_dataframe(data)
  29. elif type(data) is pd.Series:
  30. data = pd.DataFrame(data)
  31. in_length = len(data.columns)
  32. return_data = self.flatten_dataframe(data)
  33. elif type(data) is dict:
  34. in_length = len(data)
  35. return_data = self.flatten_dict(data)
  36. elif type(data) is list:
  37. in_length = len(data)
  38. return_data = self.flatten_list(data)
  39. else:
  40. self._log.log_and_raise_warning(("Input data type '{}' is not supported").format(type(data)))
  41. return None
  42. result_dataframe = pd.DataFrame.from_dict(return_data, orient='index')
  43. self._log.info(('Data has been flattened, created {} columns in {} seconds').format(len(result_dataframe.columns)- in_length, time.time()-start))
  44. return result_dataframe
  45. def flatten_dataframe(self, dataframe: pd.DataFrame, incoming_key: str = None):
  46. '''
  47. :param pd.Dataframe dataframe: dataframe containing the data to be flattened
  48. :param str incoming_key: string to be appended to the key
  49. '''
  50. assert(isinstance(dataframe, pd.DataFrame)),\
  51. "Parameter 'dataframe' be of DataFrame type"
  52. if incoming_key is not None:
  53. assert(isinstance(incoming_key, str)),\
  54. "Parameter 'incoming_key' be of String type"
  55. result_dict = {}
  56. for index, row in dataframe.iterrows():
  57. temp_result_dict = {}
  58. for key, value in row.iteritems():
  59. temp_result = {}
  60. if incoming_key is not None:
  61. key = incoming_key + '_' + key
  62. if type(value) == list:
  63. temp_result = self.flatten_list(value, key)
  64. elif type(value) == dict:
  65. temp_result = self.flatten_dict(value, key)
  66. else:
  67. temp_result_dict[key] = value
  68. if len(temp_result) > 0:
  69. temp_result_dict = self.append_to_dict(temp_result_dict, temp_result)
  70. result_dict[index] = copy.deepcopy(temp_result_dict)
  71. return result_dict
  72. def flatten_dict(self, dictionary: dict, incoming_key: str = None):
  73. '''
  74. :param dict dictionary: dictionary containing the data to be flattened
  75. :param str incoming_key: string to be appended to the key
  76. '''
  77. assert(isinstance(dictionary, dict)),\
  78. "Parameter 'dictionary' be of Dictionary type"
  79. if incoming_key is not None:
  80. assert(isinstance(incoming_key, str)),\
  81. "Parameter 'incoming_key' be of String type"
  82. result_dict = {}
  83. for key in dictionary:
  84. temp_dataframe = dictionary[key]
  85. temp_result = {}
  86. if incoming_key is not None:
  87. key = incoming_key + '_' + key
  88. if type(temp_dataframe) == list:
  89. temp_result = self.flatten_list(temp_dataframe, key)
  90. elif type(temp_dataframe) == dict:
  91. temp_result = self.flatten_dict(temp_dataframe, key)
  92. else:
  93. result_dict[key] = temp_dataframe
  94. if len(temp_result) > 0:
  95. result_dict = self.append_to_dict(result_dict, temp_result)
  96. return result_dict
  97. def flatten_list(self, data_list: list, incoming_key: str = None):
  98. '''
  99. :param list data_list: list containing the data to be flattened
  100. :param str incoming_key: string to be appended to the key
  101. '''
  102. assert(isinstance(data_list, list)),\
  103. "Parameter 'data_list' be of List type"
  104. if incoming_key is not None:
  105. assert(isinstance(incoming_key, str)),\
  106. "Parameter 'incoming_key' be of String type"
  107. result_dict = {}
  108. for iteration, item in enumerate(data_list):
  109. temp_dataframe = item
  110. temp_result = {}
  111. key = incoming_key
  112. if incoming_key is not None:
  113. # OEBB SPECIFIC IF STATEMENT
  114. if type(data_list[iteration]) is dict and 'stationsnummer' in data_list[iteration].keys() and 'stage' in data_list[iteration].keys() :
  115. key = incoming_key + '_' + str(data_list[iteration]['stationsnummer']) + '_' + str(data_list[iteration]['stage'])
  116. else:
  117. key = incoming_key + '_' + str(iteration)
  118. else:
  119. key = str(iteration)
  120. if type(temp_dataframe) == list:
  121. temp_result = self.flatten_list(temp_dataframe, key)
  122. elif type(temp_dataframe) == dict:
  123. temp_result = self.flatten_dict(temp_dataframe, key)
  124. else:
  125. result_dict[key] = temp_dataframe
  126. if len(temp_result) > 0:
  127. result_dict = self.append_to_dict(result_dict, temp_result)
  128. return result_dict
  129. def append_to_dict(self, dictionary: dict, to_append):
  130. '''
  131. :param dict dictionary: dictionary which holds all the resulting data.
  132. :param dict to_append: data to be added to the resulting dictionary.
  133. '''
  134. assert(isinstance(dictionary, dict)),\
  135. "Parameter 'dictionary' be of Dictionary type"
  136. assert(isinstance(to_append, dict)),\
  137. "Parameter 'to_append' be of Dictionary type"
  138. for key in to_append:
  139. dictionary[key] = to_append[key]
  140. return dictionary