change dereference fuction and move to ParseJsonSchema

cdplib/db_handlers/MongodbHandler.py

@@ -98,67 +98,7 @@ class MongodbHandler:
         '''
         self._database[collection_name].drop()
 
-    def _read_schema(self, schema_path: str) -> dict:
-        '''
-        :param str schema_path: path to the schema file.
-        '''
-
-        assert(isinstance(schema_path, str)),\
-            "Parameter 'schema_path must be a string type"
-
-        with open(schema_path) as json_file:
-            schema = json.load(json_file)
-
-        definitions_flag = self._analyze_schema(schema)
-
-        if definitions_flag:
-            schema = self._dereference_schema(schema)
-
-        return schema
-
-    def _analyze_schema(self, schema: dict, definitions_flag: bool = False) -> dict:
-
-
-        for key in schema:
-            if key == 'definitions':
-                definitions_flag = True
-                return definitions_flag
-
-            if key == 'default' or key == 'default_values':
-                return self._remove_defaults(schema)
-
-            if type(schema[key]) == dict:
-                definitions_flag = self._analyze_schema(schema[key], definitions_flag)
-
-        return definitions_flag
-
-    def _dereference_schema(self, schema: dict) -> dict:
-        '''
-        :param dict schema: dictionary containing a schema which uses references.
-        '''
-
-        assert(isinstance(schema, dict)),\
-            "Parameter 'schema' must be a dictionary type"
-
-        schema = jsonref.loads(str(schema).replace("'", "\""))
-        schema = deepcopy(schema)
-        schema.pop('definitions', None)
-        return schema
-
-    def _remove_defaults(self, schema: dict) -> dict:
-        '''
-        :param dict schema: dictionary containing a schema which uses references.
-        '''
-        if 'default' in schema:
-            del schema['default']
-        if 'default_values' in schema:
-            del schema['default_values']
-        return schema
-
-
-        assert(isinstance(schema, dict)),\
-            "Parameter 'schema' must be a dictionary type"
-
+   
     def set_collection_schema(self, collection_name: str, schema_path: str,
                               validation_level: str = 'moderate',validation_action: str = 'error'):
         '''

cdplib/db_migration/ParseJsonSchema.py

@@ -10,6 +10,11 @@ import os
 import sys
 from copy import deepcopy
 import numpy as np
+import json
+
+import jsonref
+from pathlib import Path
+from urllib.parse import urljoin
 
 sys.path.append(os.getcwd())
 
@@ -24,7 +29,6 @@ class ParseJsonSchema(ParseDbSchema):
     def __init__(self, schema_paths: [list, str], log_file: str = None):
         '''
         '''
-        import json
         from cdplib.log import Log
 
         super().__init__(schema_paths=schema_paths, log_file=log_file)
@@ -326,13 +330,77 @@ class ParseJsonSchema(ParseDbSchema):
 
         return already_parsed
 
+    def _dereference_schema(self, schema: dict) -> dict:
+        '''
+        :param dict schema: dictionary containing a schema which uses references.
+        '''
+
+        assert(isinstance(schema, dict)),\
+            "Parameter 'schema' must be a dictionary type"
+            
+        base_dir_url = Path(os.path.join(os.getcwd(), "mongo_schema")).as_uri() + '/'
+        schema = jsonref.loads(str(schema).replace("'", "\""),base_uri=base_dir_url)
+        schema = deepcopy(schema)
+        schema.pop('definitions', None)
+        return schema
+
+    def _remove_defaults(self, schema: dict) -> dict:
+        '''
+        :param dict schema: dictionary containing a schema which uses references.
+        '''
+        if 'default' in schema:
+            del schema['default']
+        if 'default_values' in schema:
+            del schema['default_values']
+        return schema
+    
+    # Need to parse schmema for importing to mongo db 
+    # Reason:
+    # We need to drop default values since MongoDB can't handle them
+    # We need to deference json before import to Mongo DB pymongo can't deal with references
+    def read_schema_and_parse_for_mongodb(self, schema_path: str) -> dict:
+        '''
+        :param str schema_path: path to the schema file.
+        '''
+
+        assert(isinstance(schema_path, str)),\
+            "Parameter 'schema_path must be a string type"
+
+        with open(schema_path) as json_file:
+            schema = json.load(json_file)
+
+        flags = self._analyze_schema(schema)
+
+        if flags['ref_flag'] == 1:
+          schema = self._dereference_schema(schema)
+        
+        if flags['default_flag'] == 1:
+          schema = self._remove_defaults(schema)
+
+        return schema
+    
+    def _analyze_schema(self, schema: dict,
+                        flags: dict = {'ref_flag':False, 'default_flag':False}) -> dict:
+        for key in schema:
+            if key == '$ref':
+                flags['ref_flag'] = True
+            
+            if key == 'default' or key == 'default_values':
+                flags['default_flag'] = True
+            
+            if type(schema[key]) == dict:
+                flags = self._analyze_schema(schema[key], flags)
+    
+        return  flags
+
+
 
 if __name__ == "__main__":
 
     # Only for testing
 
     schema_path = os.path.join(".", "mongo_schema", "schema_wheelsets.json")
-
+    
     if os.path.isfile(schema_path):
 
         parse_obj = ParseJsonSchema(schema_paths=schema_path)