Refactor generate_datatypes (#3138)

* Move generate_datatypes

* Move generate_datatypes

* Refactor datatypes

* Fix Codacy errors in backend

* Fix Codacy errors in parser

tools/generate_datatypes.py

@@ -5,445 +5,9 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import print_function
-import sys
-import time
-import platform
-import getpass
-from collections import OrderedDict
-import re
-import xml.etree.ElementTree as etree
-import itertools
+from nodeset_compiler.type_parser import CSVBSDTypeParser
+import nodeset_compiler.backend_open62541_typedefinitions as backend
 import argparse
-import csv
-import json
-from nodeset_compiler.opaque_type_mapping import get_base_type_for_opaque as get_base_type_for_opaque_ns0
-
-types = OrderedDict() # contains types that were already parsed
-types_imported = OrderedDict() # contains types that were already parsed and marked as imported types (do not write to source code)
-typedescriptions = {} # contains type nodeids
-user_opaque_type_mapping = {} # contains user defined opaque type mapping
-
-excluded_types = ["NodeIdType", "InstanceNode", "TypeNode", "Node", "ObjectNode",
-                  "ObjectTypeNode", "VariableNode", "VariableTypeNode", "ReferenceTypeNode",
-                  "MethodNode", "ViewNode", "DataTypeNode",
-                  "NumericRange", "NumericRangeDimensions",
-                  "UA_ServerDiagnosticsSummaryDataType", "UA_SamplingIntervalDiagnosticsDataType",
-                  "UA_SessionSecurityDiagnosticsDataType", "UA_SubscriptionDiagnosticsDataType",
-                  "UA_SessionDiagnosticsDataType"]
-
-builtin_types = ["Boolean", "SByte", "Byte", "Int16", "UInt16", "Int32", "UInt32",
-                 "Int64", "UInt64", "Float", "Double", "String", "DateTime", "Guid",
-                 "ByteString", "XmlElement", "NodeId", "ExpandedNodeId", "StatusCode",
-                 "QualifiedName", "LocalizedText", "ExtensionObject", "DataValue",
-                 "Variant", "DiagnosticInfo"]
-
-# If set to False, every defined datatype must have a corresponding ID entry in the csv file
-isInternalTypes = False
-
-# Some types can be memcpy'd off the binary stream. That's especially important
-# for arrays. But we need to check if they contain padding and whether the
-# endianness is correct. This dict gives the C-statement that must be true for the
-# type to be overlayable. Parsed types are added if they apply.
-builtin_overlayable = {"Boolean": "true",
-                       "SByte": "true", "Byte": "true",
-                       "Int16": "UA_BINARY_OVERLAYABLE_INTEGER",
-                       "UInt16": "UA_BINARY_OVERLAYABLE_INTEGER",
-                       "Int32": "UA_BINARY_OVERLAYABLE_INTEGER",
-                       "UInt32": "UA_BINARY_OVERLAYABLE_INTEGER",
-                       "Int64": "UA_BINARY_OVERLAYABLE_INTEGER",
-                       "UInt64": "UA_BINARY_OVERLAYABLE_INTEGER",
-                       "Float": "UA_BINARY_OVERLAYABLE_FLOAT",
-                       "Double": "UA_BINARY_OVERLAYABLE_FLOAT",
-                       "DateTime": "UA_BINARY_OVERLAYABLE_INTEGER",
-                       "StatusCode": "UA_BINARY_OVERLAYABLE_INTEGER",
-                       "Guid": "(UA_BINARY_OVERLAYABLE_INTEGER && " +
-                       "offsetof(UA_Guid, data2) == sizeof(UA_UInt32) && " +
-                       "offsetof(UA_Guid, data3) == (sizeof(UA_UInt16) + sizeof(UA_UInt32)) && " +
-                       "offsetof(UA_Guid, data4) == (2*sizeof(UA_UInt32)))"}
-
-whitelistFuncAttrWarnUnusedResult = []  # for instances [ "String", "ByteString", "LocalizedText" ]
-
-# Type aliases
-type_aliases = { "CharArray" : "String" }
-def getTypeName(xmlTypeName):
-    typeName = xmlTypeName[xmlTypeName.find(":")+1:]
-    return type_aliases.get(typeName, typeName)
-
-# Escape C strings:
-def makeCLiteral(value):
-    return re.sub(r'(?<!\\)"', r'\\"', value.replace('\\', r'\\\\').replace('\n', r'\\n').replace('\r', r''))
-
-# Strip invalid characters to create valid C identifiers (variable names etc):
-def makeCIdentifier(value):
-    return re.sub(r'[^\w]', '', value)
-
-def get_base_type_for_opaque(name):
-    if name in user_opaque_type_mapping:
-        return user_opaque_type_mapping[name]
-    else:
-        return get_base_type_for_opaque_ns0(name)
-
-################
-# Type Classes #
-################
-
-class StructMember(object):
-    def __init__(self, name, memberType, isArray):
-        self.name = name
-        self.memberType = memberType
-        self.isArray = isArray
-
-def getNodeidTypeAndId(nodeId):
-    if '=' not in nodeId:
-        return "UA_NODEIDTYPE_NUMERIC, {{{0}}}".format(nodeId)
-    if nodeId.startswith("i="):
-        return "UA_NODEIDTYPE_NUMERIC, {{{0}}}".format(nodeId[2:])
-    if nodeId.startswith("s="):
-        strId = nodeId[2:]
-        return "UA_NODEIDTYPE_STRING, {{ .string = UA_STRING_STATIC(\"{id}\") }}".format(id=strId.replace("\"", "\\\""))
-
-
-class Type(object):
-    def __init__(self, outname, xml, namespace):
-        self.name = None
-        if xml is not None:
-            self.name = xml.get("Name")
-            self.typeIndex = "UA_" + makeCIdentifier(outname.upper() + "_" + self.name.upper())
-        else:
-            self.typeIndex = makeCIdentifier(outname.upper())
-        self.ns0 = ("true" if namespace == 0 else "false")
-        self.outname = outname
-        self.kind = None
-        self.description = ""
-        self.pointerfree = "false"
-        self.overlayable = "false"
-        self.members = []
-        if xml is not None:
-            for child in xml:
-                if child.tag == "{http://opcfoundation.org/BinarySchema/}Documentation":
-                    self.description = child.text
-                    break
-
-    def datatype_c(self):
-        # xmlEncodingId = "0"
-        binaryEncodingId = "0"
-        if self.name in typedescriptions:
-            description = typedescriptions[self.name]
-            typeid = "{%s, %s}" % (description.namespaceid, getNodeidTypeAndId(description.nodeid))
-            # xmlEncodingId = description.xmlEncodingId
-            binaryEncodingId = description.binaryEncodingId
-        else:
-            if not isInternalTypes:
-                raise RuntimeError("NodeId for " + self.name + " not found in .csv file")
-            else:
-                typeid = "{0, UA_NODEIDTYPE_NUMERIC, {0}}"
-        idName = makeCIdentifier(self.name)
-        return "{\n    UA_TYPENAME(\"%s\") /* .typeName */\n" % idName + \
-            "    " + typeid + ", /* .typeId */\n" + \
-            "    sizeof(UA_" + idName + "), /* .memSize */\n" + \
-            "    " + self.typeIndex + ", /* .typeIndex */\n" + \
-            "    " + self.kind + ", /* .typeKind */\n" + \
-            "    " + self.pointerfree + ", /* .pointerFree */\n" + \
-            "    " + self.overlayable + ", /* .overlayable */\n" + \
-            "    " + str(len(self.members)) + ", /* .membersSize */\n" + \
-            "    " + binaryEncodingId + ", /* .binaryEncodingId */\n" + \
-            "    %s_members" % idName + " /* .members */\n}"
-
-    def members_c(self):
-        idName = makeCIdentifier(self.name)
-        if len(self.members) == 0:
-            return "#define %s_members NULL" % (idName)
-        members = "static UA_DataTypeMember %s_members[%s] = {" % (idName, len(self.members))
-        before = None
-        size = len(self.members)
-        for i, member in enumerate(self.members):
-            memberName = makeCIdentifier(member.name)
-            memberNameCapital = memberName
-            if len(memberName) > 0:
-                memberNameCapital = memberName[0].upper() + memberName[1:]
-            m = "\n{\n    UA_TYPENAME(\"%s\") /* .memberName */\n" % memberNameCapital
-            m += "    UA_%s_%s, /* .memberTypeIndex */\n" % (member.memberType.outname.upper(), makeCIdentifier(member.memberType.name.upper()))
-            m += "    "
-            if not before:
-                m += "0,"
-            else:
-                if member.isArray:
-                    m += "offsetof(UA_%s, %sSize)" % (idName, memberName)
-                else:
-                    m += "offsetof(UA_%s, %s)" % (idName, memberName)
-                m += " - offsetof(UA_%s, %s)" % (idName, makeCIdentifier(before.name))
-                if before.isArray:
-                    m += " - sizeof(void*),"
-                else:
-                    m += " - sizeof(UA_%s)," % makeCIdentifier(before.memberType.name)
-            m += " /* .padding */\n"
-            m += "    %s, /* .namespaceZero */\n" % member.memberType.ns0
-            m += ("    true" if member.isArray else "    false") + " /* .isArray */\n}"
-            if i != size:
-                m += ","
-            members += m
-            before = member
-        return members + "};"
-
-    def datatype_ptr(self):
-        return "&UA_" + self.outname.upper() + "[UA_" + makeCIdentifier(self.outname.upper() + "_" + self.name.upper()) + "]"
-
-    def functions_c(self):
-        idName = makeCIdentifier(self.name)
-        funcs = "static UA_INLINE void\nUA_%s_init(UA_%s *p) {\n    memset(p, 0, sizeof(UA_%s));\n}\n\n" % (idName, idName, idName)
-        funcs += "static UA_INLINE UA_%s *\nUA_%s_new(void) {\n    return (UA_%s*)UA_new(%s);\n}\n\n" % (idName, idName, idName, self.datatype_ptr())
-        if self.pointerfree == "true":
-            funcs += "static UA_INLINE UA_StatusCode\nUA_%s_copy(const UA_%s *src, UA_%s *dst) {\n    *dst = *src;\n    return UA_STATUSCODE_GOOD;\n}\n\n" % (idName, idName, idName)
-            funcs += "static UA_INLINE void\nUA_%s_deleteMembers(UA_%s *p) {\n    memset(p, 0, sizeof(UA_%s));\n}\n\n" % (idName, idName, idName)
-            funcs += "static UA_INLINE void\nUA_%s_clear(UA_%s *p) {\n    memset(p, 0, sizeof(UA_%s));\n}\n\n" % (idName, idName, idName)
-        else:
-            for entry in whitelistFuncAttrWarnUnusedResult:
-                if idName == entry:
-                    funcs += "UA_INTERNAL_FUNC_ATTR_WARN_UNUSED_RESULT "
-                    break
-
-            funcs += "static UA_INLINE UA_StatusCode\nUA_%s_copy(const UA_%s *src, UA_%s *dst) {\n    return UA_copy(src, dst, %s);\n}\n\n" % (idName, idName, idName, self.datatype_ptr())
-            funcs += "static UA_INLINE void\nUA_%s_deleteMembers(UA_%s *p) {\n    UA_clear(p, %s);\n}\n\n" % (idName, idName, self.datatype_ptr())
-            funcs += "static UA_INLINE void\nUA_%s_clear(UA_%s *p) {\n    UA_clear(p, %s);\n}\n\n" % (idName, idName, self.datatype_ptr())
-        funcs += "static UA_INLINE void\nUA_%s_delete(UA_%s *p) {\n    UA_delete(p, %s);\n}" % (idName, idName, self.datatype_ptr())
-        return funcs
-
-    def encoding_h(self):
-        idName = makeCIdentifier(self.name)
-        enc = "static UA_INLINE size_t\nUA_%s_calcSizeBinary(const UA_%s *src) {\n    return UA_calcSizeBinary(src, %s);\n}\n"
-        enc += "static UA_INLINE UA_StatusCode\nUA_%s_encodeBinary(const UA_%s *src, UA_Byte **bufPos, const UA_Byte *bufEnd) {\n    return UA_encodeBinary(src, %s, bufPos, &bufEnd, NULL, NULL);\n}\n"
-        enc += "static UA_INLINE UA_StatusCode\nUA_%s_decodeBinary(const UA_ByteString *src, size_t *offset, UA_%s *dst) {\n    return UA_decodeBinary(src, offset, dst, %s, NULL);\n}"
-        return enc % tuple(list(itertools.chain(*itertools.repeat([idName, idName, self.datatype_ptr()], 3))))
-
-class BuiltinType(Type):
-    def __init__(self, name):
-        Type.__init__(self, name, None, 0)
-        self.name = name
-        self.ns0 = "true"
-        self.typeIndex = makeCIdentifier("UA_TYPES_" + self.name.upper())
-        self.outname = "types"
-        self.kind = "UA_DATATYPEKIND_" + self.name.upper()
-        self.description = ""
-        self.pointerfree = "false"
-        if self.name in builtin_overlayable.keys():
-            self.pointerfree = "true"
-        self.overlayable = "false"
-        if name in builtin_overlayable:
-            self.overlayable = builtin_overlayable[name]
-        self.members = []
-
-class EnumerationType(Type):
-    def __init__(self, outname, xml, namespace):
-        Type.__init__(self, outname, xml, namespace)
-        self.pointerfree = "true"
-        self.overlayable = "UA_BINARY_OVERLAYABLE_INTEGER"
-        self.members = []
-        self.kind = "UA_DATATYPEKIND_ENUM"
-        self.typeIndex = "UA_TYPES_INT32"
-        self.elements = OrderedDict()
-        for child in xml:
-            if child.tag == "{http://opcfoundation.org/BinarySchema/}EnumeratedValue":
-                self.elements[child.get("Name")] = child.get("Value")
-
-    def typedef_h(self):
-        if sys.version_info[0] < 3:
-            values = self.elements.iteritems()
-        else:
-            values = self.elements.items()
-        return "typedef enum {\n    " + ",\n    ".join(map(lambda kv : makeCIdentifier("UA_" + self.name.upper() + "_" + kv[0].upper()) + \
-                                                           " = " + kv[1], values)) + \
-               ",\n    __UA_{0}_FORCE32BIT = 0x7fffffff\n".format(makeCIdentifier(self.name.upper())) + "} " + \
-               "UA_{0};\nUA_STATIC_ASSERT(sizeof(UA_{0}) == sizeof(UA_Int32), enum_must_be_32bit);".format(makeCIdentifier(self.name))
-
-class OpaqueType(Type):
-    def __init__(self, outname, xml, namespace, baseType):
-        Type.__init__(self, outname, xml, namespace)
-        self.kind = "UA_DATATYPEKIND_" + baseType.upper()
-        self.baseType = baseType
-        self.members = []
-
-    def typedef_h(self):
-        return "typedef UA_" + self.baseType + " UA_%s;" % self.name
-
-class StructType(Type):
-    def __init__(self, outname, xml, namespace):
-        Type.__init__(self, outname, xml, namespace)
-        self.members = []
-        lengthfields = [] # lengthfields of arrays are not included as members
-        for child in xml:
-            if child.get("LengthField"):
-                lengthfields.append(child.get("LengthField"))
-        for child in xml:
-            if not child.tag == "{http://opcfoundation.org/BinarySchema/}Field":
-                continue
-            if child.get("Name") in lengthfields:
-                continue
-            memberName = child.get("Name")
-            memberName = memberName[:1].lower() + memberName[1:]
-            memberTypeName = getTypeName(child.get("TypeName"))
-            memberType = types[memberTypeName]
-            isArray = True if child.get("LengthField") else False
-            self.members.append(StructMember(memberName, memberType, isArray))
-
-        self.pointerfree = "true"
-        self.overlayable = "true"
-        self.kind = "UA_DATATYPEKIND_STRUCTURE"
-        before = None
-        for m in self.members:
-            if m.isArray or m.memberType.pointerfree != "true":
-                self.pointerfree = "false"
-                self.overlayable = "false"
-            else:
-                self.overlayable += "\n\t\t && " + m.memberType.overlayable
-                if before:
-                    self.overlayable += "\n\t\t && offsetof(UA_%s, %s) == (offsetof(UA_%s, %s) + sizeof(UA_%s))" % \
-                                        (makeCIdentifier(self.name), makeCIdentifier(m.name), makeCIdentifier(self.name), makeCIdentifier(before.name), makeCIdentifier(before.memberType.name))
-            if "false" in self.overlayable:
-                self.overlayable = "false"
-            before = m
-
-    def typedef_h(self):
-        if len(self.members) == 0:
-            return "typedef void * UA_%s;" % makeCIdentifier(self.name)
-        returnstr =  "typedef struct {\n"
-        for member in self.members:
-            if member.isArray:
-                returnstr += "    size_t %sSize;\n" % makeCIdentifier(member.name)
-                returnstr += "    UA_%s *%s;\n" % (makeCIdentifier(member.memberType.name), makeCIdentifier(member.name))
-            else:
-                returnstr += "    UA_%s %s;\n" % (makeCIdentifier(member.memberType.name), makeCIdentifier(member.name))
-        return returnstr + "} UA_%s;" % makeCIdentifier(self.name)
-
-#########################
-# Parse Typedefinitions #
-#########################
-
-def parseTypeDefinitions(outname, xmlDescription, namespace, addToTypes=None):
-    def typeReady(element):
-        "Are all member types defined?"
-        for child in element:
-            if child.tag == "{http://opcfoundation.org/BinarySchema/}Field":
-                childname = getTypeName(child.get("TypeName"))
-                if childname not in types:
-                    return False
-        return True
-
-    def unknownTypes(element):
-        "Return all unknown types"
-        unknowns = []
-        for child in element:
-            if child.tag == "{http://opcfoundation.org/BinarySchema/}Field":
-                childname = getTypeName(child.get("TypeName"))
-                if childname not in types:
-                    unknowns.append(childname)
-        return unknowns
-
-    def skipType(name):
-        if name in excluded_types:
-            return True
-        if re.search("NodeId$", name) != None:
-            return True
-        return False
-
-    snippets = {}
-    for typeXml in etree.parse(xmlDescription).getroot():
-        if not typeXml.get("Name"):
-            continue
-        name = typeXml.get("Name")
-        snippets[name] = typeXml
-
-    detectLoop = len(snippets)+1
-    while(len(snippets) > 0):
-        if detectLoop == len(snippets):
-            name, typeXml = (snippets.items())[0]
-            raise RuntimeError("Infinite loop detected trying to processing types " + name + ": unknonwn subtype " + str(unknownTypes(typeXml)))
-        detectLoop = len(snippets)
-        for name, typeXml in list(snippets.items()):
-            if name in types or skipType(name):
-                del snippets[name]
-                continue
-            if not typeReady(typeXml):
-                continue
-            if name in builtin_types:
-                newType = BuiltinType(name)
-            elif typeXml.tag == "{http://opcfoundation.org/BinarySchema/}EnumeratedType":
-                newType = EnumerationType(outname, typeXml, namespace)
-            elif typeXml.tag == "{http://opcfoundation.org/BinarySchema/}OpaqueType":
-                newType = OpaqueType(outname, typeXml, namespace, get_base_type_for_opaque(name)['name'])
-            elif typeXml.tag == "{http://opcfoundation.org/BinarySchema/}StructuredType":
-                newType = StructType(outname, typeXml, namespace)
-            else:
-                raise Exception("Type not known")
-
-            types[name] = newType
-            if addToTypes is not None:
-                addToTypes[name] = newType
-
-            del snippets[name]
-
-##########################
-# Parse TypeDescriptions #
-##########################
-
-class TypeDescription(object):
-    def __init__(self, name, nodeid, namespaceid):
-        self.name = name
-        self.nodeid = nodeid
-        self.namespaceid = namespaceid
-        self.xmlEncodingId = "0"
-        self.binaryEncodingId = "0"
-
-def parseTypeDescriptions(f, namespaceid):
-    definitions = {}
-
-    csvreader = csv.reader(f, delimiter=',')
-    delay_init = []
-
-    for row in csvreader:
-        if len(row) < 3:
-            continue
-        if row[2] == "Object":
-            # Check if node name ends with _Encoding_(DefaultXml|DefaultBinary) and store the node id in the corresponding DataType
-            m = re.match('(.*?)_Encoding_Default(Xml|Binary)$',row[0])
-            if (m):
-                baseType = m.group(1)
-                if baseType not in types:
-                    continue
-
-                delay_init.append({
-                    "baseType": baseType,
-                    "encoding": m.group(2),
-                    "id": row[1]
-                })
-            continue
-        if row[2] != "DataType":
-            continue
-        if row[0] == "BaseDataType":
-            definitions["Variant"] = TypeDescription(row[0], row[1], namespaceid)
-        elif row[0] == "Structure":
-            definitions["ExtensionObject"] = TypeDescription(row[0], row[1], namespaceid)
-        elif row[0] not in types:
-            continue
-        else:
-            definitions[row[0]] = TypeDescription(row[0], row[1], namespaceid)
-    for i in delay_init:
-        if i["baseType"] not in definitions:
-            raise Exception("Type {} not found in definitions file.".format(i["baseType"]))
-        if i["encoding"] == "Xml":
-            definitions[i["baseType"]].xmlEncodingId = i["id"]
-        else:
-            definitions[i["baseType"]].binaryEncodingId = i["id"]
-    return definitions
-
-def merge_dicts(*dict_args):
-    """
-    Given any number of dicts, shallow copy and merge into a new dict,
-    precedence goes to key value pairs in latter dicts.
-    """
-    result = {}
-    for dictionary in dict_args:
-        result.update(dictionary)
-    return result
 
 ###############################
 # Parse the Command Line Input#
@@ -512,198 +76,11 @@ parser.add_argument('outfile',
 args = parser.parse_args()
 
 outname = args.outfile.split("/")[-1]
-inname = ', '.join(list(map(lambda x:x.name.split("/")[-1], args.type_bsd)))
-
-isInternalTypes = args.internal
-
-################
-# Create Types #
-################
-
-for builtin in builtin_types:
-    types[builtin] = BuiltinType(builtin)
-
-for f in args.opaque_map:
-    user_opaque_type_mapping.update(json.load(f))
-
-for i in args.import_bsd:
-    (outname_import, file_import) = i.split("#")
-    outname_import = outname_import.lower()
-    if outname_import.startswith("ua_"):
-        outname_import = outname_import[3:]
-    parseTypeDefinitions(outname_import, file_import, args.namespace, addToTypes=types_imported)
-
-for f in args.type_bsd:
-    parseTypeDefinitions(outname, f, args.namespace)
-
-typedescriptions = {}
-for f in args.type_csv:
-    typedescriptions = merge_dicts(typedescriptions, parseTypeDescriptions(f, args.namespace))
-
-# Read the selected data types
-selected_types = []
-for f in args.selected_types:
-    selected_types += list(filter(len, [line.strip() for line in f]))
-# Use all types if none are selected
-if len(selected_types) == 0:
-    selected_types = types.keys()
-
-#############################
-# Write out the Definitions #
-#############################
-
-fh = open(args.outfile + "_generated.h", 'w')
-ff = open(args.outfile + "_generated_handling.h", 'w')
-fe = open(args.outfile + "_generated_encoding_binary.h", 'w')
-fc = open(args.outfile + "_generated.c",'w')
-def printh(string):
-    print(string, end='\n', file=fh)
-def printf(string):
-    print(string, end='\n', file=ff)
-def printe(string):
-    print(string, end='\n', file=fe)
-def printc(string):
-    print(string, end='\n', file=fc)
-
-def iter_types(v):
-    l = None
-    if sys.version_info[0] < 3:
-        l = list(v.itervalues())
-    else:
-        l = list(v.values())
-    if len(selected_types) > 0:
-        l = list(filter(lambda t: t.name in selected_types, l))
-    if args.no_builtin:
-        l = list(filter(lambda t: type(t) != BuiltinType, l))
-    l = list(filter(lambda t: t.name not in types_imported, l))
-    return l
-
-################
-# Print Header #
-################
-
-printh('''/* Generated from ''' + inname + ''' with script ''' + sys.argv[0] + '''
- * on host ''' + platform.uname()[1] + ''' by user ''' + getpass.getuser() + \
-       ''' at ''' + time.strftime("%Y-%m-%d %I:%M:%S") + ''' */
-
-#ifndef ''' + outname.upper() + '''_GENERATED_H_
-#define ''' + outname.upper() + '''_GENERATED_H_
-
-#ifdef UA_ENABLE_AMALGAMATION
-#include "open62541.h"
-#else
-#include <open62541/types.h>
-''' + ('#include <open62541/types_generated.h>\n' if outname != "types" else '') + '''
-#endif
-
-_UA_BEGIN_DECLS
-
-''')
-
-filtered_types = iter_types(types)
-
-printh('''/**
- * Every type is assigned an index in an array containing the type descriptions.
- * These descriptions are used during type handling (copying, deletion,
- * binary encoding, ...). */''')
-printh("#define UA_" + outname.upper() + "_COUNT %s" % (str(len(filtered_types))))
-printh("extern UA_EXPORT const UA_DataType UA_" + outname.upper() + "[UA_" + outname.upper() + "_COUNT];")
-
-for i, t in enumerate(filtered_types):
-    printh("\n/**\n * " +  t.name)
-    printh(" * " + "^" * len(t.name))
-    if t.description == "":
-        printh(" */")
-    else:
-        printh(" * " + t.description + " */")
-    if type(t) != BuiltinType:
-        printh(t.typedef_h() + "\n")
-    printh("#define UA_" + makeCIdentifier(outname.upper() + "_" + t.name.upper()) + " " + str(i))
-
-printh('''
-
-_UA_END_DECLS
-
-#endif /* %s_GENERATED_H_ */''' % outname.upper())
-
-##################
-# Print Handling #
-##################
-
-printf('''/* Generated from ''' + inname + ''' with script ''' + sys.argv[0] + '''
- * on host ''' + platform.uname()[1] + ''' by user ''' + getpass.getuser() + \
-       ''' at ''' + time.strftime("%Y-%m-%d %I:%M:%S") + ''' */
-
-#ifndef ''' + outname.upper() + '''_GENERATED_HANDLING_H_
-#define ''' + outname.upper() + '''_GENERATED_HANDLING_H_
-
-#include "''' + outname + '''_generated.h"
-
-_UA_BEGIN_DECLS
-
-#if defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 6
-# pragma GCC diagnostic push
-# pragma GCC diagnostic ignored "-Wmissing-field-initializers"
-# pragma GCC diagnostic ignored "-Wmissing-braces"
-#endif
-''')
-
-for t in filtered_types:
-    printf("\n/* " + t.name + " */")
-    printf(t.functions_c())
-
-printf('''
-#if defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 6
-# pragma GCC diagnostic pop
-#endif
-
-_UA_END_DECLS
-
-#endif /* %s_GENERATED_HANDLING_H_ */''' % outname.upper())
-
-###########################
-# Print Description Array #
-###########################
-
-printc('''/* Generated from ''' + inname + ''' with script ''' + sys.argv[0] + '''
- * on host ''' + platform.uname()[1] + ''' by user ''' + getpass.getuser() + \
-       ''' at ''' + time.strftime("%Y-%m-%d %I:%M:%S") + ''' */
-
-#include "''' + outname + '''_generated.h"''')
-
-for t in filtered_types:
-    printc("")
-    printc("/* " + t.name + " */")
-    printc(t.members_c())
-
-printc("const UA_DataType UA_%s[UA_%s_COUNT] = {" % (outname.upper(), outname.upper()))
-for t in filtered_types:
-    printc("/* " + t.name + " */")
-    printc(t.datatype_c() + ",")
-printc("};\n")
-
-##################
-# Print Encoding #
-##################
-
-printe('''/* Generated from ''' + inname + ''' with script ''' + sys.argv[0] + '''
- * on host ''' + platform.uname()[1] + ''' by user ''' + getpass.getuser() + \
-       ''' at ''' + time.strftime("%Y-%m-%d %I:%M:%S") + ''' */
-
-#ifdef UA_ENABLE_AMALGAMATION
-# include "open62541.h"
-#else
-# include "ua_types_encoding_binary.h"
-# include "''' + outname + '''_generated.h"
-#endif
-
-''')
+inname = ', '.join(list(map(lambda x: x.name.split("/")[-1], args.type_bsd)))
 
-for t in filtered_types:
-    printe("\n/* " + t.name + " */")
-    printe(t.encoding_h())
+parser = CSVBSDTypeParser(args.opaque_map, args.selected_types, args.no_builtin, outname, args.namespace, args.import_bsd,
+                          args.type_bsd, args.type_csv)
+parser.create_types()
 
-fh.close()
-ff.close()
-fc.close()
-fe.close()
+generator = backend.CGenerator(parser, inname, args.outfile, args.internal)
+generator.write_definitions()

tools/nodeset_compiler/backend_open62541_typedefinitions.py

@@ -0,0 +1,418 @@
+from __future__ import print_function
+import re
+import itertools
+import sys
+import time
+import getpass
+import platform
+
+if sys.version_info[0] >= 3:
+    from nodeset_compiler.type_parser import BuiltinType, EnumerationType, OpaqueType, StructType
+else:
+    from type_parser import BuiltinType, EnumerationType, OpaqueType, StructType
+
+# Some types can be memcpy'd off the binary stream. That's especially important
+# for arrays. But we need to check if they contain padding and whether the
+# endianness is correct. This dict gives the C-statement that must be true for the
+# type to be overlayable. Parsed types are added if they apply.
+builtin_overlayable = {"Boolean": "true",
+                       "SByte": "true", "Byte": "true",
+                       "Int16": "UA_BINARY_OVERLAYABLE_INTEGER",
+                       "UInt16": "UA_BINARY_OVERLAYABLE_INTEGER",
+                       "Int32": "UA_BINARY_OVERLAYABLE_INTEGER",
+                       "UInt32": "UA_BINARY_OVERLAYABLE_INTEGER",
+                       "Int64": "UA_BINARY_OVERLAYABLE_INTEGER",
+                       "UInt64": "UA_BINARY_OVERLAYABLE_INTEGER",
+                       "Float": "UA_BINARY_OVERLAYABLE_FLOAT",
+                       "Double": "UA_BINARY_OVERLAYABLE_FLOAT",
+                       "DateTime": "UA_BINARY_OVERLAYABLE_INTEGER",
+                       "StatusCode": "UA_BINARY_OVERLAYABLE_INTEGER",
+                       "Guid": "(UA_BINARY_OVERLAYABLE_INTEGER && " +
+                               "offsetof(UA_Guid, data2) == sizeof(UA_UInt32) && " +
+                               "offsetof(UA_Guid, data3) == (sizeof(UA_UInt16) + sizeof(UA_UInt32)) && " +
+                               "offsetof(UA_Guid, data4) == (2*sizeof(UA_UInt32)))"}
+
+whitelistFuncAttrWarnUnusedResult = []  # for instances [ "String", "ByteString", "LocalizedText" ]
+
+
+# Escape C strings:
+def makeCLiteral(value):
+    return re.sub(r'(?<!\\)"', r'\\"', value.replace('\\', r'\\\\').replace('\n', r'\\n').replace('\r', r''))
+
+
+# Strip invalid characters to create valid C identifiers (variable names etc):
+def makeCIdentifier(value):
+    return re.sub(r'[^\w]', '', value)
+
+
+def getNodeidTypeAndId(nodeId):
+    if '=' not in nodeId:
+        return "UA_NODEIDTYPE_NUMERIC, {{{0}}}".format(nodeId)
+    if nodeId.startswith("i="):
+        return "UA_NODEIDTYPE_NUMERIC, {{{0}}}".format(nodeId[2:])
+    if nodeId.startswith("s="):
+        strId = nodeId[2:]
+        return "UA_NODEIDTYPE_STRING, {{ .string = UA_STRING_STATIC(\"{id}\") }}".format(id=strId.replace("\"", "\\\""))
+
+
+class CGenerator(object):
+    def __init__(self, parser, inname, outfile, is_internal_types):
+        self.parser = parser
+        self.inname = inname
+        self.outfile = outfile
+        self.is_internal_types = is_internal_types
+        self.filtered_types = None
+        self.fh = None
+        self.ff = None
+        self.fc = None
+        self.fe = None
+
+    @staticmethod
+    def get_type_index(datatype):
+        if isinstance(datatype,  BuiltinType):
+            return makeCIdentifier("UA_TYPES_" + datatype.name.upper())
+        if isinstance(datatype, EnumerationType):
+            return "UA_TYPES_INT32"
+
+        if datatype.name is not None:
+            return "UA_" + makeCIdentifier(datatype.outname.upper() + "_" + datatype.name.upper())
+        return makeCIdentifier(datatype.outname.upper())
+
+    @staticmethod
+    def get_type_kind(datatype):
+        if isinstance(datatype, BuiltinType):
+            return "UA_DATATYPEKIND_" + datatype.name.upper()
+        if isinstance(datatype, EnumerationType):
+            return "UA_DATATYPEKIND_ENUM"
+        if isinstance(datatype, OpaqueType):
+            return "UA_DATATYPEKIND_" + datatype.base_type.upper()
+        if isinstance(datatype, StructType):
+            return "UA_DATATYPEKIND_STRUCTURE"
+        raise RuntimeError("Unknown type")
+
+    @staticmethod
+    def get_struct_overlayable(struct):
+        if not struct.pointerfree == "false":
+            return "false"
+        before = None
+        overlayable = ""
+        for m in struct.members:
+            if m.is_array or not m.member_type.pointerfree:
+                return "false"
+            overlayable += "\n\t\t && " + m.member_type.overlayable
+            if before:
+                overlayable += "\n\t\t && offsetof(UA_%s, %s) == (offsetof(UA_%s, %s) + sizeof(UA_%s))" % \
+                               (makeCIdentifier(struct.name), makeCIdentifier(m.name), makeCIdentifier(struct.name),
+                                makeCIdentifier(before.name), makeCIdentifier(before.member_type.name))
+            before = m
+        return overlayable
+
+    def get_type_overlayable(self, datatype):
+        if isinstance(datatype, BuiltinType) or isinstance(datatype, OpaqueType):
+            return builtin_overlayable[datatype.name] if datatype.name in builtin_overlayable else "false"
+        if isinstance(datatype, EnumerationType):
+            return "UA_BINARY_OVERLAYABLE_INTEGER"
+        if isinstance(datatype, StructType):
+            return self.get_struct_overlayable(datatype)
+        raise RuntimeError("Unknown datatype")
+
+    def print_datatype(self, datatype):
+        binaryEncodingId = "0"
+        if datatype.name in self.parser.typedescriptions:
+            description = self.parser.typedescriptions[datatype.name]
+            typeid = "{%s, %s}" % (description.namespaceid, getNodeidTypeAndId(description.nodeid))
+            # xmlEncodingId = description.xmlEncodingId
+            binaryEncodingId = description.binaryEncodingId
+        else:
+            if not self.is_internal_types:
+                raise RuntimeError("NodeId for " + datatype.name + " not found in .csv file")
+            else:
+                typeid = "{0, UA_NODEIDTYPE_NUMERIC, {0}}"
+        idName = makeCIdentifier(datatype.name)
+        pointerfree = "true" if datatype.pointerfree else "false"
+        return "{\n    UA_TYPENAME(\"%s\") /* .typeName */\n" % idName + \
+               "    " + typeid + ", /* .typeId */\n" + \
+               "    sizeof(UA_" + idName + "), /* .memSize */\n" + \
+               "    " + self.get_type_index(datatype) + ", /* .typeIndex */\n" + \
+               "    " + self.get_type_kind(datatype) + ", /* .typeKind */\n" + \
+               "    " + pointerfree + ", /* .pointerFree */\n" + \
+               "    " + self.get_type_overlayable(datatype) + ", /* .overlayable */\n" + \
+               "    " + str(len(datatype.members)) + ", /* .membersSize */\n" + \
+               "    " + binaryEncodingId + ", /* .binaryEncodingId */\n" + \
+               "    %s_members" % idName + " /* .members */\n}"
+
+    @staticmethod
+    def print_members(datatype):
+        idName = makeCIdentifier(datatype.name)
+        if len(datatype.members) == 0:
+            return "#define %s_members NULL" % (idName)
+        members = "static UA_DataTypeMember %s_members[%s] = {" % (idName, len(datatype.members))
+        before = None
+        size = len(datatype.members)
+        for i, member in enumerate(datatype.members):
+            member_name = makeCIdentifier(member.name)
+            member_name_capital = member_name
+            if len(member_name) > 0:
+                member_name_capital = member_name[0].upper() + member_name[1:]
+            m = "\n{\n    UA_TYPENAME(\"%s\") /* .memberName */\n" % member_name_capital
+            m += "    UA_%s_%s, /* .memberTypeIndex */\n" % (
+                member.member_type.outname.upper(), makeCIdentifier(member.member_type.name.upper()))
+            m += "    "
+            if not before:
+                m += "0,"
+            else:
+                if member.is_array:
+                    m += "offsetof(UA_%s, %sSize)" % (idName, member_name)
+                else:
+                    m += "offsetof(UA_%s, %s)" % (idName, member_name)
+                m += " - offsetof(UA_%s, %s)" % (idName, makeCIdentifier(before.name))
+                if before.is_array:
+                    m += " - sizeof(void*),"
+                else:
+                    m += " - sizeof(UA_%s)," % makeCIdentifier(before.member_type.name)
+            m += " /* .padding */\n"
+            m += "    %s, /* .namespaceZero */\n" % ("true" if member.member_type.ns0 else "false")
+            m += ("    true" if member.is_array else "    false") + " /* .isArray */\n}"
+            if i != size:
+                m += ","
+            members += m
+            before = member
+        return members + "};"
+
+    @staticmethod
+    def print_datatype_ptr(datatype):
+        return "&UA_" + datatype.outname.upper() + "[UA_" + makeCIdentifier(
+            datatype.outname.upper() + "_" + datatype.name.upper()) + "]"
+
+    def print_functions(self, datatype):
+        idName = makeCIdentifier(datatype.name)
+        funcs = "static UA_INLINE void\nUA_%s_init(UA_%s *p) {\n    memset(p, 0, sizeof(UA_%s));\n}\n\n" % (
+            idName, idName, idName)
+        funcs += "static UA_INLINE UA_%s *\nUA_%s_new(void) {\n    return (UA_%s*)UA_new(%s);\n}\n\n" % (
+            idName, idName, idName, CGenerator.print_datatype_ptr(datatype))
+        if datatype.pointerfree == "true":
+            funcs += "static UA_INLINE UA_StatusCode\nUA_%s_copy(const UA_%s *src, UA_%s *dst) {\n    *dst = *src;\n    return UA_STATUSCODE_GOOD;\n}\n\n" % (
+                idName, idName, idName)
+            funcs += "static UA_INLINE void\nUA_%s_deleteMembers(UA_%s *p) {\n    memset(p, 0, sizeof(UA_%s));\n}\n\n" % (
+                idName, idName, idName)
+            funcs += "static UA_INLINE void\nUA_%s_clear(UA_%s *p) {\n    memset(p, 0, sizeof(UA_%s));\n}\n\n" % (
+                idName, idName, idName)
+        else:
+            for entry in whitelistFuncAttrWarnUnusedResult:
+                if idName == entry:
+                    funcs += "UA_INTERNAL_FUNC_ATTR_WARN_UNUSED_RESULT "
+                    break
+
+            funcs += "static UA_INLINE UA_StatusCode\nUA_%s_copy(const UA_%s *src, UA_%s *dst) {\n    return UA_copy(src, dst, %s);\n}\n\n" % (
+                idName, idName, idName, self.print_datatype_ptr(datatype))
+            funcs += "static UA_INLINE void\nUA_%s_deleteMembers(UA_%s *p) {\n    UA_clear(p, %s);\n}\n\n" % (
+                idName, idName, self.print_datatype_ptr(datatype))
+            funcs += "static UA_INLINE void\nUA_%s_clear(UA_%s *p) {\n    UA_clear(p, %s);\n}\n\n" % (
+                idName, idName, self.print_datatype_ptr(datatype))
+        funcs += "static UA_INLINE void\nUA_%s_delete(UA_%s *p) {\n    UA_delete(p, %s);\n}" % (
+            idName, idName, self.print_datatype_ptr(datatype))
+        return funcs
+
+    def print_datatype_encoding(self, datatype):
+        idName = makeCIdentifier(datatype.name)
+        enc = "static UA_INLINE size_t\nUA_%s_calcSizeBinary(const UA_%s *src) {\n    return UA_calcSizeBinary(src, %s);\n}\n"
+        enc += "static UA_INLINE UA_StatusCode\nUA_%s_encodeBinary(const UA_%s *src, UA_Byte **bufPos, const UA_Byte *bufEnd) {\n    return UA_encodeBinary(src, %s, bufPos, &bufEnd, NULL, NULL);\n}\n"
+        enc += "static UA_INLINE UA_StatusCode\nUA_%s_decodeBinary(const UA_ByteString *src, size_t *offset, UA_%s *dst) {\n    return UA_decodeBinary(src, offset, dst, %s, NULL);\n}"
+        return enc % tuple(
+            list(itertools.chain(*itertools.repeat([idName, idName, self.print_datatype_ptr(datatype)], 3))))
+
+    @staticmethod
+    def print_enum_typedef(enum):
+        if sys.version_info[0] < 3:
+            values = enum.elements.iteritems()
+        else:
+            values = enum.elements.items()
+        return "typedef enum {\n    " + ",\n    ".join(
+            map(lambda kv: makeCIdentifier("UA_" + enum.name.upper() + "_" + kv[0].upper()) +
+                           " = " + kv[1], values)) + \
+               ",\n    __UA_{0}_FORCE32BIT = 0x7fffffff\n".format(makeCIdentifier(enum.name.upper())) + "} " + \
+               "UA_{0};\nUA_STATIC_ASSERT(sizeof(UA_{0}) == sizeof(UA_Int32), enum_must_be_32bit);".format(
+                   makeCIdentifier(enum.name))
+
+    @staticmethod
+    def print_struct_typedef(struct):
+        if len(struct.members) == 0:
+            return "typedef void * UA_%s;" % makeCIdentifier(struct.name)
+        returnstr = "typedef struct {\n"
+        for member in struct.members:
+            if member.is_array:
+                returnstr += "    size_t %sSize;\n" % makeCIdentifier(member.name)
+                returnstr += "    UA_%s *%s;\n" % (
+                    makeCIdentifier(member.member_type.name), makeCIdentifier(member.name))
+            else:
+                returnstr += "    UA_%s %s;\n" % (
+                    makeCIdentifier(member.member_type.name), makeCIdentifier(member.name))
+        return returnstr + "} UA_%s;" % makeCIdentifier(struct.name)
+
+    @staticmethod
+    def print_datatype_typedef(datatype):
+        if isinstance(datatype, EnumerationType):
+            return CGenerator.print_enum_typedef(datatype)
+        if isinstance(datatype, OpaqueType):
+            return "typedef UA_" + datatype.base_type + " UA_%s;" % datatype.name
+        if isinstance(datatype, StructType):
+            return CGenerator.print_struct_typedef(datatype)
+        raise RuntimeError("Type does not have an associated typedef")
+
+    def write_definitions(self):
+        self.fh = open(self.outfile + "_generated.h", 'w')
+        self.ff = open(self.outfile + "_generated_handling.h", 'w')
+        self.fe = open(self.outfile + "_generated_encoding_binary.h", 'w')
+        self.fc = open(self.outfile + "_generated.c", 'w')
+
+        self.filtered_types = self.iter_types(self.parser.types)
+
+        self.print_header()
+        self.print_handling()
+        self.print_description_array()
+        self.print_encoding()
+
+        self.fh.close()
+        self.ff.close()
+        self.fc.close()
+        self.fe.close()
+
+    def printh(self, string):
+        print(string, end='\n', file=self.fh)
+
+    def printf(self, string):
+        print(string, end='\n', file=self.ff)
+
+    def printe(self, string):
+        print(string, end='\n', file=self.fe)
+
+    def printc(self, string):
+        print(string, end='\n', file=self.fc)
+
+    def iter_types(self, v):
+        l = None
+        if sys.version_info[0] < 3:
+            l = list(v.itervalues())
+        else:
+            l = list(v.values())
+        if len(self.parser.selected_types) > 0:
+            l = list(filter(lambda t: t.name in self.parser.selected_types, l))
+        if self.parser.no_builtin:
+            l = list(filter(lambda t: not isinstance(t, BuiltinType), l))
+        l = list(filter(lambda t: t.name not in self.parser.types_imported, l))
+        return l
+
+    def print_header(self):
+        self.printh('''/* Generated from ''' + self.inname + ''' with script ''' + sys.argv[0] + '''
+ * on host ''' + platform.uname()[1] + ''' by user ''' + getpass.getuser() + ''' at ''' + time.strftime(
+            "%Y-%m-%d %I:%M:%S") + ''' */
+
+#ifndef ''' + self.parser.outname.upper() + '''_GENERATED_H_
+#define ''' + self.parser.outname.upper() + '''_GENERATED_H_
+
+#ifdef UA_ENABLE_AMALGAMATION
+#include "open62541.h"
+#else
+#include <open62541/types.h>
+''' + ('#include <open62541/types_generated.h>\n' if self.parser.outname != "types" else '') + '''
+#endif
+
+_UA_BEGIN_DECLS
+
+''')
+
+        self.printh('''/**
+ * Every type is assigned an index in an array containing the type descriptions.
+ * These descriptions are used during type handling (copying, deletion,
+ * binary encoding, ...). */''')
+        self.printh("#define UA_" + self.parser.outname.upper() + "_COUNT %s" % (str(len(self.filtered_types))))
+        self.printh(
+            "extern UA_EXPORT const UA_DataType UA_" + self.parser.outname.upper() + "[UA_" + self.parser.outname.upper() + "_COUNT];")
+
+        for i, t in enumerate(self.filtered_types):
+            self.printh("\n/**\n * " + t.name)
+            self.printh(" * " + "^" * len(t.name))
+            if t.description == "":
+                self.printh(" */")
+            else:
+                self.printh(" * " + t.description + " */")
+            if not isinstance(t, BuiltinType):
+                self.printh(self.print_datatype_typedef(t) + "\n")
+            self.printh(
+                "#define UA_" + makeCIdentifier(self.parser.outname.upper() + "_" + t.name.upper()) + " " + str(i))
+
+        self.printh('''
+
+_UA_END_DECLS
+
+#endif /* %s_GENERATED_H_ */''' % self.parser.outname.upper())
+
+    def print_handling(self):
+        self.printf('''/* Generated from ''' + self.inname + ''' with script ''' + sys.argv[0] + '''
+ * on host ''' + platform.uname()[1] + ''' by user ''' + getpass.getuser() + ''' at ''' + time.strftime(
+            "%Y-%m-%d %I:%M:%S") + ''' */
+
+#ifndef ''' + self.parser.outname.upper() + '''_GENERATED_HANDLING_H_
+#define ''' + self.parser.outname.upper() + '''_GENERATED_HANDLING_H_
+
+#include "''' + self.parser.outname + '''_generated.h"
+
+_UA_BEGIN_DECLS
+
+#if defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 6
+# pragma GCC diagnostic push
+# pragma GCC diagnostic ignored "-Wmissing-field-initializers"
+# pragma GCC diagnostic ignored "-Wmissing-braces"
+#endif
+''')
+
+        for t in self.filtered_types:
+            self.printf("\n/* " + t.name + " */")
+            self.printf(self.print_functions(t))
+
+        self.printf('''
+#if defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 6
+# pragma GCC diagnostic pop
+#endif
+
+_UA_END_DECLS
+
+#endif /* %s_GENERATED_HANDLING_H_ */''' % self.parser.outname.upper())
+
+    def print_description_array(self):
+        self.printc('''/* Generated from ''' + self.inname + ''' with script ''' + sys.argv[0] + '''
+ * on host ''' + platform.uname()[1] + ''' by user ''' + getpass.getuser() + ''' at ''' + time.strftime(
+            "%Y-%m-%d %I:%M:%S") + ''' */
+
+#include "''' + self.parser.outname + '''_generated.h"''')
+
+        for t in self.filtered_types:
+            self.printc("")
+            self.printc("/* " + t.name + " */")
+            self.printc(CGenerator.print_members(t))
+
+        self.printc(
+            "const UA_DataType UA_%s[UA_%s_COUNT] = {" % (self.parser.outname.upper(), self.parser.outname.upper()))
+
+        for t in self.filtered_types:
+            self.printc("/* " + t.name + " */")
+            self.printc(self.print_datatype(t) + ",")
+        self.printc("};\n")
+
+    def print_encoding(self):
+        self.printe('''/* Generated from ''' + self.inname + ''' with script ''' + sys.argv[0] + '''
+ * on host ''' + platform.uname()[1] + ''' by user ''' + getpass.getuser() + ''' at ''' + time.strftime(
+            "%Y-%m-%d %I:%M:%S") + ''' */
+
+#ifdef UA_ENABLE_AMALGAMATION
+# include "open62541.h"
+#else
+# include "ua_types_encoding_binary.h"
+# include "''' + self.parser.outname + '''_generated.h"
+#endif
+
+''')
+
+        for t in self.filtered_types:
+            self.printe("\n/* " + t.name + " */")
+            self.printe(self.print_datatype_encoding(t))

tools/nodeset_compiler/type_parser.py

@@ -0,0 +1,319 @@
+import abc
+import csv
+import json
+import xml.etree.ElementTree as etree
+import re
+from collections import OrderedDict
+import sys
+
+if sys.version_info[0] >= 3:
+    from nodeset_compiler.opaque_type_mapping import get_base_type_for_opaque as get_base_type_for_opaque_ns0
+    print("hello")
+else:
+    from opaque_type_mapping import get_base_type_for_opaque as get_base_type_for_opaque_ns0
+    # import opaque_type_mapping
+
+builtin_types = ["Boolean", "SByte", "Byte", "Int16", "UInt16", "Int32", "UInt32",
+                 "Int64", "UInt64", "Float", "Double", "String", "DateTime", "Guid",
+                 "ByteString", "XmlElement", "NodeId", "ExpandedNodeId", "StatusCode",
+                 "QualifiedName", "LocalizedText", "ExtensionObject", "DataValue",
+                 "Variant", "DiagnosticInfo"]
+
+excluded_types = ["NodeIdType", "InstanceNode", "TypeNode", "Node", "ObjectNode",
+                  "ObjectTypeNode", "VariableNode", "VariableTypeNode", "ReferenceTypeNode",
+                  "MethodNode", "ViewNode", "DataTypeNode",
+                  "NumericRange", "NumericRangeDimensions",
+                  "UA_ServerDiagnosticsSummaryDataType", "UA_SamplingIntervalDiagnosticsDataType",
+                  "UA_SessionSecurityDiagnosticsDataType", "UA_SubscriptionDiagnosticsDataType",
+                  "UA_SessionDiagnosticsDataType"]
+
+builtin_overlayable = ["Boolean", "SByte", "Byte", "Int16", "UInt16", "Int32", "UInt32", "Int64", "UInt64", "Float",
+                       "Double", "DateTime", "StatusCode", "Guid"]
+
+# Type aliases
+type_aliases = {"CharArray": "String"}
+
+user_opaque_type_mapping = {}  # contains user defined opaque type mapping
+
+
+def get_base_type_for_opaque(name):
+    if name in user_opaque_type_mapping:
+        return user_opaque_type_mapping[name]
+    else:
+        return get_base_type_for_opaque_ns0(name)
+
+
+def get_type_name(xml_type_name):
+    type_name = xml_type_name[xml_type_name.find(":") + 1:]
+    return type_aliases.get(type_name, type_name)
+
+
+class Type(object):
+    def __init__(self, outname, xml, namespace):
+        self.name = None
+        if xml is not None:
+            self.name = xml.get("Name")
+        self.outname = outname
+        self.namespace = namespace
+        self.pointerfree = False
+        self.members = []
+        self.description = ""
+        self.ns0 = (namespace == 0)
+        if xml is not None:
+            for child in xml:
+                if child.tag == "{http://opcfoundation.org/BinarySchema/}Documentation":
+                    self.description = child.text
+                    break
+
+
+class BuiltinType(Type):
+    def __init__(self, name):
+        Type.__init__(self, "types", None, 0)
+        self.name = name
+        if self.name in builtin_overlayable:
+            self.pointerfree = True
+
+
+class EnumerationType(Type):
+    def __init__(self, outname, xml, namespace):
+        Type.__init__(self, outname, xml, namespace)
+        self.pointerfree = True
+        self.elements = OrderedDict()
+        for child in xml:
+            if child.tag == "{http://opcfoundation.org/BinarySchema/}EnumeratedValue":
+                self.elements[child.get("Name")] = child.get("Value")
+
+
+class OpaqueType(Type):
+    def __init__(self, outname, xml, namespace, base_type):
+        Type.__init__(self, outname, xml, namespace)
+        self.base_type = base_type
+
+
+class StructMember(object):
+    def __init__(self, name, member_type, is_array):
+        self.name = name
+        self.member_type = member_type
+        self.is_array = is_array
+
+
+class StructType(Type):
+    def __init__(self, outname, xml, namespace, types):
+        Type.__init__(self, outname, xml, namespace)
+        length_fields = []
+
+        for child in xml:
+            length_field = child.get("LengthField")
+            if length_field:
+                length_fields.append(length_field)
+
+        for child in xml:
+            if not child.tag == "{http://opcfoundation.org/BinarySchema/}Field":
+                continue
+            if child.get("Name") in length_fields:
+                continue
+            member_name = child.get("Name")
+            member_name = member_name[:1].lower() + member_name[1:]
+            member_type_name = get_type_name(child.get("TypeName"))
+            member_type = types[member_type_name]
+            is_array = True if child.get("LengthField") else False
+            self.members.append(StructMember(member_name, member_type, is_array))
+
+        self.pointerfree = True
+        for m in self.members:
+            if m.is_array or not m.member_type.pointerfree:
+                self.pointerfree = False
+
+
+class TypeDescription(object):
+    def __init__(self, name, nodeid, namespaceid):
+        self.name = name
+        self.nodeid = nodeid
+        self.namespaceid = namespaceid
+        self.xmlEncodingId = "0"
+        self.binaryEncodingId = "0"
+
+
+class TypeParser():
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, opaque_map, selected_types, no_builtin, outname, namespace):
+        self.selected_types = []
+        self.fh = None
+        self.ff = None
+        self.fc = None
+        self.fe = None
+        self.opaque_map = opaque_map
+        self.selected_types = selected_types
+        self.no_builtin = no_builtin
+        self.outname = outname
+        self.namespace = namespace
+        self.types = OrderedDict()
+
+    @staticmethod
+    def merge_dicts(*dict_args):
+        """
+        Given any number of dicts, shallow copy and merge into a new dict,
+        precedence goes to key value pairs in latter dicts.
+        """
+        result = {}
+        for dictionary in dict_args:
+            result.update(dictionary)
+        return result
+
+    def parseTypeDefinitions(self, outname, xmlDescription, namespace, addToTypes=None):
+        def typeReady(element):
+            "Are all member types defined?"
+            for child in element:
+                if child.tag == "{http://opcfoundation.org/BinarySchema/}Field":
+                    childname = get_type_name(child.get("TypeName"))
+                    if childname not in self.types:
+                        return False
+            return True
+
+        def unknownTypes(element):
+            "Return all unknown types"
+            unknowns = []
+            for child in element:
+                if child.tag == "{http://opcfoundation.org/BinarySchema/}Field":
+                    childname = get_type_name(child.get("TypeName"))
+                    if childname not in self.types:
+                        unknowns.append(childname)
+            return unknowns
+
+        def skipType(name):
+            if name in excluded_types:
+                return True
+            if re.search("NodeId$", name) != None:
+                return True
+            return False
+
+        snippets = {}
+        for typeXml in etree.parse(xmlDescription).getroot():
+            if not typeXml.get("Name"):
+                continue
+            name = typeXml.get("Name")
+            snippets[name] = typeXml
+
+        detectLoop = len(snippets) + 1
+        while len(snippets) > 0:
+            if detectLoop == len(snippets):
+                name, typeXml = (snippets.items())[0]
+                raise RuntimeError(
+                    "Infinite loop detected trying to processing types " + name + ": unknonwn subtype " + str(
+                        unknownTypes(typeXml)))
+            detectLoop = len(snippets)
+            for name, typeXml in list(snippets.items()):
+                if name in self.types or skipType(name):
+                    del snippets[name]
+                    continue
+                if not typeReady(typeXml):
+                    continue
+                if name in builtin_types:
+                    new_type = BuiltinType(name)
+                elif typeXml.tag == "{http://opcfoundation.org/BinarySchema/}EnumeratedType":
+                    new_type = EnumerationType(outname, typeXml, namespace)
+                elif typeXml.tag == "{http://opcfoundation.org/BinarySchema/}OpaqueType":
+                    new_type = OpaqueType(outname, typeXml, namespace, get_base_type_for_opaque(name)['name'])
+                elif typeXml.tag == "{http://opcfoundation.org/BinarySchema/}StructuredType":
+                    new_type = StructType(outname, typeXml, namespace, self.types)
+                else:
+                    raise Exception("Type not known")
+
+                self.types[name] = new_type
+                if addToTypes is not None:
+                    addToTypes[name] = new_type
+
+                del snippets[name]
+
+    @abc.abstractmethod
+    def parse_types(self):
+        pass
+
+    def create_types(self):
+        for builtin in builtin_types:
+            self.types[builtin] = BuiltinType(builtin)
+
+        for f in self.opaque_map:
+            user_opaque_type_mapping.update(json.load(f))
+
+        self.parse_types()
+
+        # Read the selected data types
+        arg_selected_types = self.selected_types
+        self.selected_types = []
+        for f in arg_selected_types:
+            self.selected_types += list(filter(len, [line.strip() for line in f]))
+        # Use all types if none are selected
+        print("Selected types: ", len(self.selected_types))
+        if len(self.selected_types) == 0:
+            self.selected_types = self.types.keys()
+
+
+class CSVBSDTypeParser(TypeParser):
+    def __init__(self, opaque_map, selected_types, no_builtin, outname, namespace, import_bsd,
+                 type_bsd, type_csv):
+        TypeParser.__init__(self, opaque_map, selected_types, no_builtin, outname, namespace)
+        self.typedescriptions = {}
+        self.import_bsd = import_bsd
+        self.type_bsd = type_bsd
+        self.type_csv = type_csv
+        self.types_imported = {}
+
+    def parse_types(self):
+        for i in self.import_bsd:
+            (outname_import, file_import) = i.split("#")
+            outname_import = outname_import.lower()
+            if outname_import.startswith("ua_"):
+                outname_import = outname_import[3:]
+            self.parseTypeDefinitions(outname_import, file_import, self.namespace, addToTypes=self.types_imported)
+
+        for f in self.type_bsd:
+            self.parseTypeDefinitions(self.outname, f, self.namespace)
+
+        for f in self.type_csv:
+            self.typedescriptions = self.merge_dicts(self.typedescriptions,
+                                                     self.parseTypeDescriptions(f, self.namespace))
+
+    def parseTypeDescriptions(self, f, namespaceid):
+        definitions = {}
+
+        csvreader = csv.reader(f, delimiter=',')
+        delay_init = []
+
+        for row in csvreader:
+            if len(row) < 3:
+                continue
+            if row[2] == "Object":
+                # Check if node name ends with _Encoding_(DefaultXml|DefaultBinary) and store the node id in the
+                # corresponding DataType
+                m = re.match('(.*?)_Encoding_Default(Xml|Binary)$', row[0])
+                if m:
+                    baseType = m.group(1)
+                    if baseType not in self.types:
+                        continue
+
+                    delay_init.append({
+                        "baseType": baseType,
+                        "encoding": m.group(2),
+                        "id": row[1]
+                    })
+                continue
+            if row[2] != "DataType":
+                continue
+            if row[0] == "BaseDataType":
+                definitions["Variant"] = TypeDescription(row[0], row[1], namespaceid)
+            elif row[0] == "Structure":
+                definitions["ExtensionObject"] = TypeDescription(row[0], row[1], namespaceid)
+            elif row[0] not in self.types:
+                continue
+            else:
+                definitions[row[0]] = TypeDescription(row[0], row[1], namespaceid)
+        for i in delay_init:
+            if i["baseType"] not in definitions:
+                raise Exception("Type {} not found in definitions file.".format(i["baseType"]))
+            if i["encoding"] == "Xml":
+                definitions[i["baseType"]].xmlEncodingId = i["id"]
+            else:
+                definitions[i["baseType"]].binaryEncodingId = i["id"]
+        return definitions