open62541_XMLPreprocessor.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404
  1. #!/usr/bin/env/python
  2. # -*- coding: utf-8 -*-
  3. # This Source Code Form is subject to the terms of the Mozilla Public
  4. # License, v. 2.0. If a copy of the MPL was not distributed with this
  5. # file, You can obtain one at http://mozilla.org/MPL/2.0/.
  6. ###
  7. ### Author: Chris Iatrou (ichrispa@core-vector.net)
  8. ###
  9. ### This program was created for educational purposes and has been
  10. ### contributed to the open62541 project by the author. All licensing
  11. ### terms for this source is inherited by the terms and conditions
  12. ### specified for by the open62541 project (see the projects readme
  13. ### file for more information on the MPL v2 terms and restrictions).
  14. ###
  15. ### This program is not meant to be used in a production environment. The
  16. ### author is not liable for any complications arising due to the use of
  17. ### this program.
  18. ###
  19. import logging
  20. from ua_constants import *
  21. import tempfile
  22. import xml.dom.minidom as dom
  23. import os
  24. import string
  25. from collections import Counter
  26. import re
  27. from ua_namespace import opcua_node_id_t
  28. logger = logging.getLogger(__name__)
  29. class preProcessDocument:
  30. originXML = '' # Original XML passed to the preprocessor
  31. targetXML = () # tuple of (fileHandle, fileName)
  32. nodeset = '' # Parsed DOM XML object
  33. parseOK = False;
  34. containedNodes = [] # contains tuples of (opcua_node_id_t, xmlelement)
  35. referencedNodes = [] # contains tuples of (opcua_node_id_t, xmlelement)
  36. namespaceOrder = [] # contains xmlns:sX attributed as tuples (int ns, string name)
  37. namespaceQualifiers = [] # contains all xmlns:XYZ qualifiers that might prefix value aliases (like "<uax:Int32>")
  38. referencedNamesSpaceUris = [] # contains <NamespaceUris> URI elements
  39. def __init__(self, originXML):
  40. self.originXML = originXML
  41. self.targetXML = tempfile.mkstemp(prefix=os.path.basename(originXML)+"_preProcessed-" ,suffix=".xml")
  42. self.parseOK = True
  43. self.containedNodes = []
  44. self.referencedNodes = []
  45. self.namespaceOrder = []
  46. self.referencedNamesSpaceUris = []
  47. self.namespaceQualifiers = []
  48. try:
  49. self.nodeset = dom.parse(originXML)
  50. if len(self.nodeset.getElementsByTagName("UANodeSet")) == 0 or len(self.nodeset.getElementsByTagName("UANodeSet")) > 1:
  51. logger.error(self, "Document " + self.targetXML[1] + " contains no or more then 1 nodeset", LOG_LEVEL_ERROR)
  52. self.parseOK = False
  53. except:
  54. self.parseOK = False
  55. logger.debug("Adding new document to be preprocessed " + os.path.basename(originXML) + " as " + self.targetXML[1])
  56. def clean(self):
  57. #os.close(self.targetXML[0]) Don't -> done to flush() after finalize()
  58. os.remove(self.targetXML[1])
  59. def getTargetXMLName(self):
  60. if (self.parseOK):
  61. return self.targetXML[1]
  62. return None
  63. def extractNamespaceURIs(self):
  64. """ extractNamespaceURIs
  65. minidom gobbles up <NamespaceUris></NamespaceUris> elements, without a decent
  66. way to reliably access this dom2 <uri></uri> elements (only attribute xmlns= are
  67. accessible using minidom). We need them for dereferencing though... This
  68. function attempts to do just that.
  69. returns: Nothing
  70. """
  71. infile = open(self.originXML)
  72. foundURIs = False
  73. nsline = ""
  74. line = infile.readline()
  75. for line in infile:
  76. if "<namespaceuris>" in line.lower():
  77. foundURIs = True
  78. elif "</namespaceuris>" in line.lower():
  79. foundURIs = False
  80. nsline = nsline + line
  81. break
  82. if foundURIs:
  83. nsline = nsline + line
  84. if len(nsline) > 0:
  85. ns = dom.parseString(nsline).getElementsByTagName("NamespaceUris")
  86. for uri in ns[0].childNodes:
  87. if uri.nodeType != uri.ELEMENT_NODE:
  88. continue
  89. self.referencedNamesSpaceUris.append(uri.firstChild.data)
  90. infile.close()
  91. def analyze(self):
  92. """ analyze()
  93. analyze will gather information about the nodes and references contained in a XML File
  94. to facilitate later preprocessing stages that adresss XML dependency issues
  95. returns: No return value
  96. """
  97. nodeIds = []
  98. ns = self.nodeset.getElementsByTagName("UANodeSet")
  99. # We need to find out what the namespace calls itself and other referenced, as numeric id's are pretty
  100. # useless sans linked nodes. There is two information sources...
  101. self.extractNamespaceURIs() # From <URI>...</URI> definitions
  102. for key in ns[0].attributes.keys(): # from xmlns:sX attributes
  103. if "xmlns:" in key: # Any key: we will be removing these qualifiers from Values later
  104. self.namespaceQualifiers.append(key.replace("xmlns:",""))
  105. if "xmlns:s" in key: # get a numeric nsId and modelname/uri
  106. self.namespaceOrder.append((int(key.replace("xmlns:s","")), re.sub("[A-Za-z0-9-_\.]+\.[xXsSdD]{3}$","",ns[0].getAttribute(key))))
  107. # Get all nodeIds contained in this XML
  108. for nd in ns[0].childNodes:
  109. if nd.nodeType != nd.ELEMENT_NODE:
  110. continue
  111. if nd.hasAttribute(u'NodeId'):
  112. self.containedNodes.append( (opcua_node_id_t(nd.getAttribute(u'NodeId')), nd) )
  113. refs = nd.getElementsByTagName(u'References')[0]
  114. for ref in refs.childNodes:
  115. if ref.nodeType == ref.ELEMENT_NODE:
  116. self.referencedNodes.append( (opcua_node_id_t(ref.firstChild.data), ref) )
  117. logger.debug("Nodes: " + str(len(self.containedNodes)) + " References: " + str(len(self.referencedNodes)))
  118. def getNamespaceId(self):
  119. """ namespaceId()
  120. Counts the namespace IDs in all nodes of this XML and picks the most used
  121. namespace as the numeric identifier of this data model.
  122. returns: Integer ID of the most propable/most used namespace in this XML
  123. """
  124. max = 0;
  125. namespaceIdGuessed = 0;
  126. idDict = {}
  127. for ndid in self.containedNodes:
  128. if not ndid[0].ns in idDict.keys():
  129. idDict[ndid[0].ns] = 1
  130. else:
  131. idDict[ndid[0].ns] = idDict[ndid[0].ns] + 1
  132. for entry in idDict:
  133. if idDict[entry] > max:
  134. max = idDict[entry]
  135. namespaceIdGuessed = entry
  136. #logger.debug("XML Contents are propably in namespace " + str(entry) + " (used by " + str(idDict[entry]) + " Nodes)")
  137. return namespaceIdGuessed
  138. def getReferencedNamespaceUri(self, nsId):
  139. """ getReferencedNamespaceUri
  140. returns an URL that hopefully corresponds to the nsId that was used to reference this model
  141. return: URI string corresponding to nsId
  142. """
  143. # Might be the more reliable method: Get the URI from the xmlns attributes (they have numers)
  144. if len(self.namespaceOrder) > 0:
  145. for el in self.namespaceOrder:
  146. if el[0] == nsId:
  147. return el[1]
  148. # Fallback:
  149. # Some models do not have xmlns:sX attributes, but still <URI>s (usually when they only reference NS0)
  150. if len(self.referencedNamesSpaceUris) > 0 and len(self.referencedNamesSpaceUris) >= nsId-1:
  151. return self.referencedNamesSpaceUris[nsId-1]
  152. #Nope, not found.
  153. return ""
  154. def getNamespaceDependencies(self):
  155. deps = []
  156. for ndid in self.referencedNodes:
  157. if not ndid[0].ns in deps:
  158. deps.append(ndid[0].ns)
  159. return deps
  160. def finalize(self):
  161. outfile = self.targetXML[0]
  162. outline = self.nodeset.toxml()
  163. for qualifier in self.namespaceQualifiers:
  164. rq = qualifier+":"
  165. outline = outline.replace(rq, "")
  166. os.write(outfile, outline.encode('UTF-8'))
  167. os.close(outfile)
  168. def reassignReferencedNamespaceId(self, currentNsId, newNsId):
  169. """ reassignReferencedNamespaceId
  170. Iterates over all references in this document, find references to currentNsId and changes them to newNsId.
  171. NodeIds themselves are not altered.
  172. returns: nothing
  173. """
  174. for refNd in self.referencedNodes:
  175. if refNd[0].ns == currentNsId:
  176. refNd[1].firstChild.data = refNd[1].firstChild.data.replace("ns="+str(currentNsId), "ns="+str(newNsId))
  177. refNd[0].ns = newNsId
  178. refNd[0].toString()
  179. def reassignNamespaceId(self, currentNsId, newNsId):
  180. """ reassignNamespaceId
  181. Iterates over all nodes in this document, find those in namespace currentNsId and changes them to newNsId.
  182. returns: nothing
  183. """
  184. #change ids in aliases
  185. ns = self.nodeset.getElementsByTagName("Alias")
  186. for al in ns:
  187. if al.nodeType == al.ELEMENT_NODE:
  188. if al.hasAttribute("Alias"):
  189. al.firstChild.data = al.firstChild.data.replace("ns=" + str(currentNsId), "ns=" + str(newNsId))
  190. logger.debug("Migrating nodes /w ns index " + str(currentNsId) + " to " + str(newNsId))
  191. for nd in self.containedNodes:
  192. if nd[0].ns == currentNsId:
  193. # In our own document, update any references to this node
  194. for refNd in self.referencedNodes:
  195. if refNd[0].ns == currentNsId and refNd[0] == nd[0]:
  196. refNd[1].firstChild.data = refNd[1].firstChild.data.replace("ns="+str(currentNsId), "ns="+str(newNsId))
  197. refNd[0].ns = newNsId
  198. refNd[0].toString()
  199. nd[1].setAttribute(u'NodeId', nd[1].getAttribute(u'NodeId').replace("ns="+str(currentNsId), "ns="+str(newNsId)))
  200. nd[0].ns = newNsId
  201. nd[0].toString()
  202. class open62541_XMLPreprocessor:
  203. preProcDocuments = []
  204. def __init__(self):
  205. self.preProcDocuments = []
  206. def addDocument(self, documentPath):
  207. self.preProcDocuments.append(preProcessDocument(documentPath))
  208. def removePreprocessedFiles(self):
  209. for doc in self.preProcDocuments:
  210. doc.clean()
  211. def getPreProcessedFiles(self):
  212. files = []
  213. for doc in self.preProcDocuments:
  214. if (doc.parseOK):
  215. files.append(doc.getTargetXMLName())
  216. return files
  217. def testModelCongruencyAgainstReferences(self, doc, refs):
  218. """ testModelCongruencyAgainstReferences
  219. Counts how many of the nodes referencef in refs can be found in the model
  220. doc.
  221. returns: double corresponding to the percentage of hits
  222. """
  223. sspace = len(refs)
  224. if sspace == 0:
  225. return float(0)
  226. found = 0
  227. for ref in refs:
  228. for n in doc.containedNodes:
  229. if str(ref) == str(n[0]):
  230. print(ref, n[0])
  231. found = found + 1
  232. break
  233. return float(found)/float(sspace)
  234. def preprocess_assignUniqueNsIds(self):
  235. nsdep = []
  236. docLst = []
  237. # Search for namespace 0('s) - plural possible if user is overwriting NS0 defaults
  238. # Remove them from the list of namespaces, zero does not get demangled
  239. for doc in self.preProcDocuments:
  240. if doc.getNamespaceId() == 0:
  241. docLst.append(doc)
  242. for doc in docLst:
  243. self.preProcDocuments.remove(doc)
  244. # Reassign namespace id's to be in ascending order
  245. nsidx = 1 # next namespace id to assign on collision (first one will be "2")
  246. for doc in self.preProcDocuments:
  247. nsidx = nsidx + 1
  248. nsid = doc.getNamespaceId()
  249. doc.reassignNamespaceId(nsid, nsidx)
  250. docLst.append(doc)
  251. logger.info("Document " + doc.originXML + " is now namespace " + str(nsidx))
  252. self.preProcDocuments = docLst
  253. def getUsedNamespaceArrayNames(self):
  254. """ getUsedNamespaceArrayNames
  255. Returns the XML xmlns:s1 or <URI>[0] of each XML document (if contained/possible)
  256. returns: dict of int:nsId -> string:url
  257. """
  258. nsName = {}
  259. for doc in self.preProcDocuments:
  260. uri = doc.getReferencedNamespaceUri(1)
  261. if uri == None:
  262. uri = "http://modeluri.not/retrievable/from/xml"
  263. nsName[doc.getNamespaceId()] = doc.getReferencedNamespaceUri(1)
  264. return nsName
  265. def preprocess_linkDependantModels(self):
  266. revertToStochastic = [] # (doc, int id), where id was not resolvable using model URIs
  267. # Attemp to identify the model relations by using model URIs in xmlns:sX or <URI> contents
  268. for doc in self.preProcDocuments:
  269. nsid = doc.getNamespaceId()
  270. dependencies = doc.getNamespaceDependencies()
  271. for d in dependencies:
  272. if d != nsid and d != 0:
  273. # Attempt to identify the namespace URI this d referes to...
  274. nsUri = doc.getReferencedNamespaceUri(d) # FIXME: This could actually fail and return ""!
  275. logger.info("Need a namespace referenced as " + str(d) + ". Which hopefully is " + nsUri)
  276. targetDoc = None
  277. for tgt in self.preProcDocuments:
  278. # That model, whose URI is known but its current id is not, will
  279. # refer have referred to itself as "1"
  280. if tgt.getReferencedNamespaceUri(1) == nsUri:
  281. targetDoc = tgt
  282. break
  283. if not targetDoc == None:
  284. # Found the model... relink the references
  285. doc.reassignReferencedNamespaceId(d, targetDoc.getNamespaceId())
  286. continue
  287. else:
  288. revertToStochastic.append((doc, d))
  289. logger.warn("Failed to reliably identify which XML/Model " + os.path.basename(doc.originXML) + " calls ns=" +str(d))
  290. for (doc, d) in revertToStochastic:
  291. logger.warn("Attempting to find stochastic match for target namespace ns=" + str(d) + " of " + os.path.basename(doc.originXML))
  292. # Copy all references to the given namespace
  293. refs = []
  294. matches = [] # list of (match%, targetDoc) to pick from later
  295. for ref in doc.referencedNodes:
  296. if ref[0].ns == d:
  297. refs.append(opcua_node_id_t(str(ref[0])))
  298. for tDoc in self.preProcDocuments:
  299. tDocId = tDoc.getNamespaceId()
  300. # Scenario: If these references did target this documents namespace...
  301. for r in refs:
  302. r.ns = tDocId
  303. r.toString()
  304. # ... how many of them would be found!?
  305. c = self.testModelCongruencyAgainstReferences(tDoc, refs)
  306. print(c)
  307. if c>0:
  308. matches.append((c, tDoc))
  309. best = (0, None)
  310. for m in matches:
  311. print(m[0])
  312. if m[0] > best[0]:
  313. best = m
  314. if best[1] != None:
  315. logger.warn("Best match (" + str(best[1]*100) + "%) for what " + os.path.basename(doc.originXML) + " refers to as ns="+str(d)+" was " + os.path.basename(best[1].originXML))
  316. doc.reassignReferencedNamespaceId(d, best[1].getNamespaceId())
  317. else:
  318. logger.error("Failed to find a match for what " + os.path.basename(doc.originXML) + " refers to as ns=" + str(d))
  319. def preprocessAll(self):
  320. ##
  321. ## First: Gather statistics about the namespaces:
  322. for doc in self.preProcDocuments:
  323. doc.analyze()
  324. # Preprocess step: Remove XML specific Naming scheme ("uax:")
  325. # FIXME: Not implemented
  326. ##
  327. ## Preprocess step: Check namespace ID multiplicity and reassign IDs if necessary
  328. ##
  329. self.preprocess_assignUniqueNsIds()
  330. self.preprocess_linkDependantModels()
  331. ##
  332. ## Prep step: prevent any XML from using namespace 1 (reserved for instances)
  333. ## FIXME: Not implemented
  334. ##
  335. ## Final: Write modified XML tmp files
  336. for doc in self.preProcDocuments:
  337. doc.finalize()
  338. return True