|
@@ -242,23 +242,8 @@ class MongodbHandler:
|
|
else:
|
|
else:
|
|
self._database[collection_name].insert_many(data, ordered=ordered)
|
|
self._database[collection_name].insert_many(data, ordered=ordered)
|
|
|
|
|
|
- except Exception as error:
|
|
|
|
- if len(data) > 1:
|
|
|
|
-
|
|
|
|
- self._log.warning(('An error occured inserting {} documents into database: {} and collection: {}.').format(len(data), self._database_name, collection_name))
|
|
|
|
- self._log.warning('This might be because one or more documents are invalid.')
|
|
|
|
- self._log.warning('We will try to insert the documents one-by-one and report which are invalid.')
|
|
|
|
- self._log.warning(('Error: {}').format(error))
|
|
|
|
-
|
|
|
|
- for row in data:
|
|
|
|
-
|
|
|
|
- try:
|
|
|
|
- self._database[collection_name].insert_one(row)
|
|
|
|
- except Exception as error:
|
|
|
|
- pprint(row)
|
|
|
|
- self._log.warning(error)
|
|
|
|
- else:
|
|
|
|
- self._log.log_and_raise_error(('An error occured when trying to insert data into {}, {}. \nError: {}').format(self._database_name, collection_name, error))
|
|
|
|
|
|
+ except pymongo.errors.BulkWriteError as error:
|
|
|
|
+ pprint(error.details)
|
|
|
|
|
|
self._log.info(('Data has been inserted into the {} collection').format(collection_name))
|
|
self._log.info(('Data has been inserted into the {} collection').format(collection_name))
|
|
|
|
|
|
@@ -355,7 +340,10 @@ class MongodbHandler:
|
|
if records:
|
|
if records:
|
|
frames.append(pd.DataFrame(records))
|
|
frames.append(pd.DataFrame(records))
|
|
|
|
|
|
- return_df = pd.concat(frames, axis=0, sort=False)
|
|
|
|
|
|
+ if len(frames) > 1:
|
|
|
|
+ return_df = pd.concat(frames, axis=0, sort=False)
|
|
|
|
+ else:
|
|
|
|
+ return_df = frames[0]
|
|
|
|
|
|
if index is not None:
|
|
if index is not None:
|
|
return_df.set_index(index, inplace=True)
|
|
return_df.set_index(index, inplace=True)
|
|
@@ -438,6 +426,9 @@ class MongodbHandler:
|
|
return self.convert_mongo_data_into_dataframe(data, index, collection_name)
|
|
return self.convert_mongo_data_into_dataframe(data, index, collection_name)
|
|
else:
|
|
else:
|
|
return data
|
|
return data
|
|
|
|
+ else:
|
|
|
|
+ self._log.warning(('No data was found for the query: {}, in collection: {}').format(query, collection_name))
|
|
|
|
+ return None
|
|
|
|
|
|
def query_oldest_or_newest_date_in_collection(self, collection_name: str, date_label: str, oldest: bool = False):
|
|
def query_oldest_or_newest_date_in_collection(self, collection_name: str, date_label: str, oldest: bool = False):
|
|
|
|
|