Browse Source

Trying to speed up convert mongo into dataframe

ogert 4 years ago
parent
commit
cd918c19bc
1 changed files with 5 additions and 0 deletions
  1. 5 0
      cdplib/db_handlers/MongodbHandler.py

+ 5 - 0
cdplib/db_handlers/MongodbHandler.py

@@ -320,9 +320,14 @@ class MongodbHandler:
             if index % chunksize == 0:
                 frames.append(pd.DataFrame(records))
                 records = []
+                
         if records:
             frames.append(pd.DataFrame(records))
         return_df = pd.concat(frames, axis=0, sort=False)
+
+        if index is not None:
+                df.set_index(index, inplace=True)
+
         self._log.info(('{} Rows were fetched from {}. DataFrame conversion is done, took {} seconds').format(len(return_df.index), collection_name if collection_name is not None else 'the database', time.time()-start_time))
         
         return return_df