Merge branch 'master' of https://intra.acdp.at/gogs/tanja/cdplib

cdplib/db_migration/MigrationCleaning.py

@@ -352,11 +352,24 @@ class MigrationCleaning:
                     self.log.log_and_raise_error(("Column {} contains missing values "
                                         "and cannot be of integer type"
                                         .format(column)))
+                    
+                elif python_type == bool:
+                    
+                    data[column] = data[column].str.lower()
+                    accepted_bool = {'ja': True, 'j': True, '1': True, 
+                                     'yes': True, 'y': True, 'true':True, 
+                                     't': True, 'nein': False, 'n': False,
+                                     'no': False, 'false': False, 'f': False,
+                                     '0': False}
+                    data[column] = data[column].map(accepted_bool)
+                    data[column] = data[column].astype(bool)
+
 
                 elif python_type == str:
                      
                     # might not be the smoothes solution but it works
                     python_type = str
+                    data = data.copy(deep=True)
                     data[column] = data[column].astype(python_type)
                     python_type = object
                     data[column] = data[column].astype(python_type)

cdplib/db_migration/ParseJsonSchema.py

@@ -50,8 +50,9 @@ class ParseJsonSchema(ParseDbSchema):
             try:
                 with open(schema_path, "r") as f:
                    schema = json.load(f) 
-                # Load schmea dereferenced and cleaned by default values
+                # Load schmea dereferenced
                 self.schemas.append(self._dereference_schema(schema))
+#                self.schemas.append(schema)
 
             except Exception as e:
                 err = ("Could not load json schema, "
@@ -339,6 +340,9 @@ class ParseJsonSchema(ParseDbSchema):
         assert(isinstance(schema, dict)),\
             "Parameter 'schema' must be a dictionary type"
             
+        # check if schema contains no null values in list of the example attribute    
+        self._analyze_schema(schema)
+            
         base_dir_url = Path(os.path.join(os.getcwd(), "mongo_schema")).as_uri() + '/'
         schema = jsonref.loads(str(schema).replace("'", "\""), base_uri=base_dir_url)
         schema = deepcopy(schema)
@@ -384,17 +388,22 @@ class ParseJsonSchema(ParseDbSchema):
     def _analyze_schema(self, schema: dict, definitions_flag: bool = False) -> dict:
 
 
-        for key in schema:
+        for key in list(schema):
+
             if key == '$ref':
                 definitions_flag = True
                 return definitions_flag
-
-            if key == 'default' or key == 'default_values':
-                return self._remove_defaults(schema)
-
+            
+            if key == 'examples' and None in schema[key]:
+                err = ("Schema can not contain list with None values. Jsonref dosen't support it.")
+                self._log.log_and_raise_error(err)
+    
             if type(schema[key]) == dict:
                 definitions_flag = self._analyze_schema(schema[key], definitions_flag)
-
+                
+            if key == 'default' or key == 'default_values':
+                self._remove_defaults(schema)
+            
         return definitions_flag
 
 
@@ -424,4 +433,12 @@ if __name__ == "__main__":
         allowed_values = parse_obj.get_allowed_values()
 
         descriptions = parse_obj.get_field_descriptions()
+    
+    
+
+    
+    
+    
+    
+