|
@@ -559,13 +559,18 @@ class MigrationCleaning:
|
|
|
|
|
|
return data
|
|
return data
|
|
|
|
|
|
- def clean_json_from_None_object(self, data: pd.DataFrame) -> pd.DataFrame():
|
|
|
|
|
|
+ def clean_json_from_None_object(self, data: pd.DataFrame, clean_bool: bool = True) -> pd.DataFrame():
|
|
|
|
+
|
|
data = data.to_json(date_format="iso")
|
|
data = data.to_json(date_format="iso")
|
|
data = json.loads(data)
|
|
data = json.loads(data)
|
|
new_data = remap(data, lambda p, k, v: v is not None)
|
|
new_data = remap(data, lambda p, k, v: v is not None)
|
|
new_data = remap(new_data, lambda p, k, v: v != 'None')
|
|
new_data = remap(new_data, lambda p, k, v: v != 'None')
|
|
new_data = remap(new_data, lambda p, k, v: v != 'inf')
|
|
new_data = remap(new_data, lambda p, k, v: v != 'inf')
|
|
- new_data = remap(new_data, lambda p, k, v: (isinstance(v,bool) or (not isinstance(v,bool) and bool(v))))
|
|
|
|
|
|
+ # cleans not only bool type also int which are 0 or 1
|
|
|
|
+ # only use if it is necessary have to be change that it only considers
|
|
|
|
+ # Ture and False for bools
|
|
|
|
+ if clean_bool:
|
|
|
|
+ new_data = remap(new_data, lambda p, k, v: (isinstance(v,bool) or (not isinstance(v,bool) and bool(v))))
|
|
return new_data
|
|
return new_data
|
|
|
|
|
|
def restrict_to_collection(self, data: pd.DataFrame, collection_name: str) -> pd.DataFrame:
|
|
def restrict_to_collection(self, data: pd.DataFrame, collection_name: str) -> pd.DataFrame:
|