internals.py 38 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144
  1. # Natural Language Toolkit: Internal utility functions
  2. #
  3. # Copyright (C) 2001-2019 NLTK Project
  4. # Author: Steven Bird <stevenbird1@gmail.com>
  5. # Edward Loper <edloper@gmail.com>
  6. # Nitin Madnani <nmadnani@ets.org>
  7. # URL: <http://nltk.org/>
  8. # For license information, see LICENSE.TXT
  9. from __future__ import print_function
  10. import subprocess
  11. import os
  12. import fnmatch
  13. import re
  14. import warnings
  15. import textwrap
  16. import types
  17. import sys
  18. import stat
  19. import locale
  20. # Use the c version of ElementTree, which is faster, if possible:
  21. try:
  22. from xml.etree import cElementTree as ElementTree
  23. except ImportError:
  24. from xml.etree import ElementTree
  25. from six import string_types
  26. from nltk import compat
  27. ##########################################################################
  28. # Java Via Command-Line
  29. ##########################################################################
  30. _java_bin = None
  31. _java_options = []
  32. # [xx] add classpath option to config_java?
  33. def config_java(bin=None, options=None, verbose=False):
  34. """
  35. Configure nltk's java interface, by letting nltk know where it can
  36. find the Java binary, and what extra options (if any) should be
  37. passed to Java when it is run.
  38. :param bin: The full path to the Java binary. If not specified,
  39. then nltk will search the system for a Java binary; and if
  40. one is not found, it will raise a ``LookupError`` exception.
  41. :type bin: str
  42. :param options: A list of options that should be passed to the
  43. Java binary when it is called. A common value is
  44. ``'-Xmx512m'``, which tells Java binary to increase
  45. the maximum heap size to 512 megabytes. If no options are
  46. specified, then do not modify the options list.
  47. :type options: list(str)
  48. """
  49. global _java_bin, _java_options
  50. _java_bin = find_binary(
  51. 'java',
  52. bin,
  53. env_vars=['JAVAHOME', 'JAVA_HOME'],
  54. verbose=verbose,
  55. binary_names=['java.exe'],
  56. )
  57. if options is not None:
  58. if isinstance(options, string_types):
  59. options = options.split()
  60. _java_options = list(options)
  61. def java(cmd, classpath=None, stdin=None, stdout=None, stderr=None, blocking=True):
  62. """
  63. Execute the given java command, by opening a subprocess that calls
  64. Java. If java has not yet been configured, it will be configured
  65. by calling ``config_java()`` with no arguments.
  66. :param cmd: The java command that should be called, formatted as
  67. a list of strings. Typically, the first string will be the name
  68. of the java class; and the remaining strings will be arguments
  69. for that java class.
  70. :type cmd: list(str)
  71. :param classpath: A ``':'`` separated list of directories, JAR
  72. archives, and ZIP archives to search for class files.
  73. :type classpath: str
  74. :param stdin, stdout, stderr: Specify the executed programs'
  75. standard input, standard output and standard error file
  76. handles, respectively. Valid values are ``subprocess.PIPE``,
  77. an existing file descriptor (a positive integer), an existing
  78. file object, 'pipe', 'stdout', 'devnull' and None. ``subprocess.PIPE`` indicates that a
  79. new pipe to the child should be created. With None, no
  80. redirection will occur; the child's file handles will be
  81. inherited from the parent. Additionally, stderr can be
  82. ``subprocess.STDOUT``, which indicates that the stderr data
  83. from the applications should be captured into the same file
  84. handle as for stdout.
  85. :param blocking: If ``false``, then return immediately after
  86. spawning the subprocess. In this case, the return value is
  87. the ``Popen`` object, and not a ``(stdout, stderr)`` tuple.
  88. :return: If ``blocking=True``, then return a tuple ``(stdout,
  89. stderr)``, containing the stdout and stderr outputs generated
  90. by the java command if the ``stdout`` and ``stderr`` parameters
  91. were set to ``subprocess.PIPE``; or None otherwise. If
  92. ``blocking=False``, then return a ``subprocess.Popen`` object.
  93. :raise OSError: If the java command returns a nonzero return code.
  94. """
  95. subprocess_output_dict = {'pipe': subprocess.PIPE, 'stdout': subprocess.STDOUT, 'devnull': subprocess.DEVNULL}
  96. stdin = subprocess_output_dict.get(stdin, stdin)
  97. stdout = subprocess_output_dict.get(stdout, stdout)
  98. stderr = subprocess_output_dict.get(stderr, stderr)
  99. if isinstance(cmd, string_types):
  100. raise TypeError('cmd should be a list of strings')
  101. # Make sure we know where a java binary is.
  102. if _java_bin is None:
  103. config_java()
  104. # Set up the classpath.
  105. if isinstance(classpath, string_types):
  106. classpaths = [classpath]
  107. else:
  108. classpaths = list(classpath)
  109. classpath = os.path.pathsep.join(classpaths)
  110. # Construct the full command string.
  111. cmd = list(cmd)
  112. cmd = ['-cp', classpath] + cmd
  113. cmd = [_java_bin] + _java_options + cmd
  114. # Call java via a subprocess
  115. p = subprocess.Popen(cmd, stdin=stdin, stdout=stdout, stderr=stderr)
  116. if not blocking:
  117. return p
  118. (stdout, stderr) = p.communicate()
  119. # Check the return code.
  120. if p.returncode != 0:
  121. print(_decode_stdoutdata(stderr))
  122. raise OSError('Java command failed : ' + str(cmd))
  123. return (stdout, stderr)
  124. if 0:
  125. # config_java(options='-Xmx512m')
  126. # Write:
  127. # java('weka.classifiers.bayes.NaiveBayes',
  128. # ['-d', '/tmp/names.model', '-t', '/tmp/train.arff'],
  129. # classpath='/Users/edloper/Desktop/weka/weka.jar')
  130. # Read:
  131. (a, b) = java(
  132. [
  133. 'weka.classifiers.bayes.NaiveBayes',
  134. '-l',
  135. '/tmp/names.model',
  136. '-T',
  137. '/tmp/test.arff',
  138. '-p',
  139. '0',
  140. ], # , '-distribution'],
  141. classpath='/Users/edloper/Desktop/weka/weka.jar',
  142. )
  143. ######################################################################
  144. # Parsing
  145. ######################################################################
  146. class ReadError(ValueError):
  147. """
  148. Exception raised by read_* functions when they fail.
  149. :param position: The index in the input string where an error occurred.
  150. :param expected: What was expected when an error occurred.
  151. """
  152. def __init__(self, expected, position):
  153. ValueError.__init__(self, expected, position)
  154. self.expected = expected
  155. self.position = position
  156. def __str__(self):
  157. return 'Expected %s at %s' % (self.expected, self.position)
  158. _STRING_START_RE = re.compile(r"[uU]?[rR]?(\"\"\"|\'\'\'|\"|\')")
  159. def read_str(s, start_position):
  160. """
  161. If a Python string literal begins at the specified position in the
  162. given string, then return a tuple ``(val, end_position)``
  163. containing the value of the string literal and the position where
  164. it ends. Otherwise, raise a ``ReadError``.
  165. :param s: A string that will be checked to see if within which a
  166. Python string literal exists.
  167. :type s: str
  168. :param start_position: The specified beginning position of the string ``s``
  169. to begin regex matching.
  170. :type start_position: int
  171. :return: A tuple containing the matched string literal evaluated as a
  172. string and the end position of the string literal.
  173. :rtype: tuple(str, int)
  174. :raise ReadError: If the ``_STRING_START_RE`` regex doesn't return a
  175. match in ``s`` at ``start_position``, i.e., open quote. If the
  176. ``_STRING_END_RE`` regex doesn't return a match in ``s`` at the
  177. end of the first match, i.e., close quote.
  178. :raise ValueError: If an invalid string (i.e., contains an invalid
  179. escape sequence) is passed into the ``eval``.
  180. :Example:
  181. >>> from nltk.internals import read_str
  182. >>> read_str('"Hello", World!', 0)
  183. ('Hello', 7)
  184. """
  185. # Read the open quote, and any modifiers.
  186. m = _STRING_START_RE.match(s, start_position)
  187. if not m:
  188. raise ReadError('open quote', start_position)
  189. quotemark = m.group(1)
  190. # Find the close quote.
  191. _STRING_END_RE = re.compile(r'\\|%s' % quotemark)
  192. position = m.end()
  193. while True:
  194. match = _STRING_END_RE.search(s, position)
  195. if not match:
  196. raise ReadError('close quote', position)
  197. if match.group(0) == '\\':
  198. position = match.end() + 1
  199. else:
  200. break
  201. # Process it, using eval. Strings with invalid escape sequences
  202. # might raise ValueEerror.
  203. try:
  204. return eval(s[start_position : match.end()]), match.end()
  205. except ValueError as e:
  206. raise ReadError('invalid string (%s)' % e)
  207. _READ_INT_RE = re.compile(r'-?\d+')
  208. def read_int(s, start_position):
  209. """
  210. If an integer begins at the specified position in the given
  211. string, then return a tuple ``(val, end_position)`` containing the
  212. value of the integer and the position where it ends. Otherwise,
  213. raise a ``ReadError``.
  214. :param s: A string that will be checked to see if within which a
  215. Python integer exists.
  216. :type s: str
  217. :param start_position: The specified beginning position of the string ``s``
  218. to begin regex matching.
  219. :type start_position: int
  220. :return: A tuple containing the matched integer casted to an int,
  221. and the end position of the int in ``s``.
  222. :rtype: tuple(int, int)
  223. :raise ReadError: If the ``_READ_INT_RE`` regex doesn't return a
  224. match in ``s`` at ``start_position``.
  225. :Example:
  226. >>> from nltk.internals import read_int
  227. >>> read_int('42 is the answer', 0)
  228. (42, 2)
  229. """
  230. m = _READ_INT_RE.match(s, start_position)
  231. if not m:
  232. raise ReadError('integer', start_position)
  233. return int(m.group()), m.end()
  234. _READ_NUMBER_VALUE = re.compile(r'-?(\d*)([.]?\d*)?')
  235. def read_number(s, start_position):
  236. """
  237. If an integer or float begins at the specified position in the
  238. given string, then return a tuple ``(val, end_position)``
  239. containing the value of the number and the position where it ends.
  240. Otherwise, raise a ``ReadError``.
  241. :param s: A string that will be checked to see if within which a
  242. Python number exists.
  243. :type s: str
  244. :param start_position: The specified beginning position of the string ``s``
  245. to begin regex matching.
  246. :type start_position: int
  247. :return: A tuple containing the matched number casted to a ``float``,
  248. and the end position of the number in ``s``.
  249. :rtype: tuple(float, int)
  250. :raise ReadError: If the ``_READ_NUMBER_VALUE`` regex doesn't return a
  251. match in ``s`` at ``start_position``.
  252. :Example:
  253. >>> from nltk.internals import read_number
  254. >>> read_number('Pi is 3.14159', 6)
  255. (3.14159, 13)
  256. """
  257. m = _READ_NUMBER_VALUE.match(s, start_position)
  258. if not m or not (m.group(1) or m.group(2)):
  259. raise ReadError('number', start_position)
  260. if m.group(2):
  261. return float(m.group()), m.end()
  262. else:
  263. return int(m.group()), m.end()
  264. ######################################################################
  265. # Check if a method has been overridden
  266. ######################################################################
  267. def overridden(method):
  268. """
  269. :return: True if ``method`` overrides some method with the same
  270. name in a base class. This is typically used when defining
  271. abstract base classes or interfaces, to allow subclasses to define
  272. either of two related methods:
  273. >>> class EaterI:
  274. ... '''Subclass must define eat() or batch_eat().'''
  275. ... def eat(self, food):
  276. ... if overridden(self.batch_eat):
  277. ... return self.batch_eat([food])[0]
  278. ... else:
  279. ... raise NotImplementedError()
  280. ... def batch_eat(self, foods):
  281. ... return [self.eat(food) for food in foods]
  282. :type method: instance method
  283. """
  284. # [xx] breaks on classic classes!
  285. if isinstance(method, types.MethodType) and compat.get_im_class(method) is not None:
  286. name = method.__name__
  287. funcs = [
  288. cls.__dict__[name]
  289. for cls in _mro(compat.get_im_class(method))
  290. if name in cls.__dict__
  291. ]
  292. return len(funcs) > 1
  293. else:
  294. raise TypeError('Expected an instance method.')
  295. def _mro(cls):
  296. """
  297. Return the method resolution order for ``cls`` -- i.e., a list
  298. containing ``cls`` and all its base classes, in the order in which
  299. they would be checked by ``getattr``. For new-style classes, this
  300. is just cls.__mro__. For classic classes, this can be obtained by
  301. a depth-first left-to-right traversal of ``__bases__``.
  302. """
  303. if isinstance(cls, type):
  304. return cls.__mro__
  305. else:
  306. mro = [cls]
  307. for base in cls.__bases__:
  308. mro.extend(_mro(base))
  309. return mro
  310. ######################################################################
  311. # Deprecation decorator & base class
  312. ######################################################################
  313. # [xx] dedent msg first if it comes from a docstring.
  314. def _add_epytext_field(obj, field, message):
  315. """Add an epytext @field to a given object's docstring."""
  316. indent = ''
  317. # If we already have a docstring, then add a blank line to separate
  318. # it from the new field, and check its indentation.
  319. if obj.__doc__:
  320. obj.__doc__ = obj.__doc__.rstrip() + '\n\n'
  321. indents = re.findall(r'(?<=\n)[ ]+(?!\s)', obj.__doc__.expandtabs())
  322. if indents:
  323. indent = min(indents)
  324. # If we don't have a docstring, add an empty one.
  325. else:
  326. obj.__doc__ = ''
  327. obj.__doc__ += textwrap.fill(
  328. '@%s: %s' % (field, message),
  329. initial_indent=indent,
  330. subsequent_indent=indent + ' ',
  331. )
  332. def deprecated(message):
  333. """
  334. A decorator used to mark functions as deprecated. This will cause
  335. a warning to be printed the when the function is used. Usage:
  336. >>> from nltk.internals import deprecated
  337. >>> @deprecated('Use foo() instead')
  338. ... def bar(x):
  339. ... print(x/10)
  340. """
  341. def decorator(func):
  342. msg = "Function %s() has been deprecated. %s" % (func.__name__, message)
  343. msg = '\n' + textwrap.fill(msg, initial_indent=' ', subsequent_indent=' ')
  344. def newFunc(*args, **kwargs):
  345. warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
  346. return func(*args, **kwargs)
  347. # Copy the old function's name, docstring, & dict
  348. newFunc.__dict__.update(func.__dict__)
  349. newFunc.__name__ = func.__name__
  350. newFunc.__doc__ = func.__doc__
  351. newFunc.__deprecated__ = True
  352. # Add a @deprecated field to the docstring.
  353. _add_epytext_field(newFunc, 'deprecated', message)
  354. return newFunc
  355. return decorator
  356. class Deprecated(object):
  357. """
  358. A base class used to mark deprecated classes. A typical usage is to
  359. alert users that the name of a class has changed:
  360. >>> from nltk.internals import Deprecated
  361. >>> class NewClassName(object):
  362. ... pass # All logic goes here.
  363. ...
  364. >>> class OldClassName(Deprecated, NewClassName):
  365. ... "Use NewClassName instead."
  366. The docstring of the deprecated class will be used in the
  367. deprecation warning message.
  368. """
  369. def __new__(cls, *args, **kwargs):
  370. # Figure out which class is the deprecated one.
  371. dep_cls = None
  372. for base in _mro(cls):
  373. if Deprecated in base.__bases__:
  374. dep_cls = base
  375. break
  376. assert dep_cls, 'Unable to determine which base is deprecated.'
  377. # Construct an appropriate warning.
  378. doc = dep_cls.__doc__ or ''.strip()
  379. # If there's a @deprecated field, strip off the field marker.
  380. doc = re.sub(r'\A\s*@deprecated:', r'', doc)
  381. # Strip off any indentation.
  382. doc = re.sub(r'(?m)^\s*', '', doc)
  383. # Construct a 'name' string.
  384. name = 'Class %s' % dep_cls.__name__
  385. if cls != dep_cls:
  386. name += ' (base class for %s)' % cls.__name__
  387. # Put it all together.
  388. msg = '%s has been deprecated. %s' % (name, doc)
  389. # Wrap it.
  390. msg = '\n' + textwrap.fill(msg, initial_indent=' ', subsequent_indent=' ')
  391. warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
  392. # Do the actual work of __new__.
  393. return object.__new__(cls)
  394. ##########################################################################
  395. # COUNTER, FOR UNIQUE NAMING
  396. ##########################################################################
  397. class Counter:
  398. """
  399. A counter that auto-increments each time its value is read.
  400. """
  401. def __init__(self, initial_value=0):
  402. self._value = initial_value
  403. def get(self):
  404. self._value += 1
  405. return self._value
  406. ##########################################################################
  407. # Search for files/binaries
  408. ##########################################################################
  409. def find_file_iter(
  410. filename,
  411. env_vars=(),
  412. searchpath=(),
  413. file_names=None,
  414. url=None,
  415. verbose=False,
  416. finding_dir=False,
  417. ):
  418. """
  419. Search for a file to be used by nltk.
  420. :param filename: The name or path of the file.
  421. :param env_vars: A list of environment variable names to check.
  422. :param file_names: A list of alternative file names to check.
  423. :param searchpath: List of directories to search.
  424. :param url: URL presented to user for download help.
  425. :param verbose: Whether or not to print path when a file is found.
  426. """
  427. file_names = [filename] + (file_names or [])
  428. assert isinstance(filename, string_types)
  429. assert not isinstance(file_names, string_types)
  430. assert not isinstance(searchpath, string_types)
  431. if isinstance(env_vars, string_types):
  432. env_vars = env_vars.split()
  433. yielded = False
  434. # File exists, no magic
  435. for alternative in file_names:
  436. path_to_file = os.path.join(filename, alternative)
  437. if os.path.isfile(path_to_file):
  438. if verbose:
  439. print('[Found %s: %s]' % (filename, path_to_file))
  440. yielded = True
  441. yield path_to_file
  442. # Check the bare alternatives
  443. if os.path.isfile(alternative):
  444. if verbose:
  445. print('[Found %s: %s]' % (filename, alternative))
  446. yielded = True
  447. yield alternative
  448. # Check if the alternative is inside a 'file' directory
  449. path_to_file = os.path.join(filename, 'file', alternative)
  450. if os.path.isfile(path_to_file):
  451. if verbose:
  452. print('[Found %s: %s]' % (filename, path_to_file))
  453. yielded = True
  454. yield path_to_file
  455. # Check environment variables
  456. for env_var in env_vars:
  457. if env_var in os.environ:
  458. if finding_dir: # This is to file a directory instead of file
  459. yielded = True
  460. yield os.environ[env_var]
  461. for env_dir in os.environ[env_var].split(os.pathsep):
  462. # Check if the environment variable contains a direct path to the bin
  463. if os.path.isfile(env_dir):
  464. if verbose:
  465. print('[Found %s: %s]' % (filename, env_dir))
  466. yielded = True
  467. yield env_dir
  468. # Check if the possible bin names exist inside the environment variable directories
  469. for alternative in file_names:
  470. path_to_file = os.path.join(env_dir, alternative)
  471. if os.path.isfile(path_to_file):
  472. if verbose:
  473. print('[Found %s: %s]' % (filename, path_to_file))
  474. yielded = True
  475. yield path_to_file
  476. # Check if the alternative is inside a 'file' directory
  477. # path_to_file = os.path.join(env_dir, 'file', alternative)
  478. # Check if the alternative is inside a 'bin' directory
  479. path_to_file = os.path.join(env_dir, 'bin', alternative)
  480. if os.path.isfile(path_to_file):
  481. if verbose:
  482. print('[Found %s: %s]' % (filename, path_to_file))
  483. yielded = True
  484. yield path_to_file
  485. # Check the path list.
  486. for directory in searchpath:
  487. for alternative in file_names:
  488. path_to_file = os.path.join(directory, alternative)
  489. if os.path.isfile(path_to_file):
  490. yielded = True
  491. yield path_to_file
  492. # If we're on a POSIX system, then try using the 'which' command
  493. # to find the file.
  494. if os.name == 'posix':
  495. for alternative in file_names:
  496. try:
  497. p = subprocess.Popen(
  498. ['which', alternative],
  499. stdout=subprocess.PIPE,
  500. stderr=subprocess.PIPE,
  501. )
  502. stdout, stderr = p.communicate()
  503. path = _decode_stdoutdata(stdout).strip()
  504. if path.endswith(alternative) and os.path.exists(path):
  505. if verbose:
  506. print('[Found %s: %s]' % (filename, path))
  507. yielded = True
  508. yield path
  509. except (KeyboardInterrupt, SystemExit, OSError):
  510. raise
  511. finally:
  512. pass
  513. if not yielded:
  514. msg = (
  515. "NLTK was unable to find the %s file!"
  516. "\nUse software specific "
  517. "configuration paramaters" % filename
  518. )
  519. if env_vars:
  520. msg += ' or set the %s environment variable' % env_vars[0]
  521. msg += '.'
  522. if searchpath:
  523. msg += '\n\n Searched in:'
  524. msg += ''.join('\n - %s' % d for d in searchpath)
  525. if url:
  526. msg += '\n\n For more information on %s, see:\n <%s>' % (filename, url)
  527. div = '=' * 75
  528. raise LookupError('\n\n%s\n%s\n%s' % (div, msg, div))
  529. def find_file(
  530. filename, env_vars=(), searchpath=(), file_names=None, url=None, verbose=False
  531. ):
  532. return next(
  533. find_file_iter(filename, env_vars, searchpath, file_names, url, verbose)
  534. )
  535. def find_dir(
  536. filename, env_vars=(), searchpath=(), file_names=None, url=None, verbose=False
  537. ):
  538. return next(
  539. find_file_iter(
  540. filename, env_vars, searchpath, file_names, url, verbose, finding_dir=True
  541. )
  542. )
  543. def find_binary_iter(
  544. name,
  545. path_to_bin=None,
  546. env_vars=(),
  547. searchpath=(),
  548. binary_names=None,
  549. url=None,
  550. verbose=False,
  551. ):
  552. """
  553. Search for a file to be used by nltk.
  554. :param name: The name or path of the file.
  555. :param path_to_bin: The user-supplied binary location (deprecated)
  556. :param env_vars: A list of environment variable names to check.
  557. :param file_names: A list of alternative file names to check.
  558. :param searchpath: List of directories to search.
  559. :param url: URL presented to user for download help.
  560. :param verbose: Whether or not to print path when a file is found.
  561. """
  562. for file in find_file_iter(
  563. path_to_bin or name, env_vars, searchpath, binary_names, url, verbose
  564. ):
  565. yield file
  566. def find_binary(
  567. name,
  568. path_to_bin=None,
  569. env_vars=(),
  570. searchpath=(),
  571. binary_names=None,
  572. url=None,
  573. verbose=False,
  574. ):
  575. return next(
  576. find_binary_iter(
  577. name, path_to_bin, env_vars, searchpath, binary_names, url, verbose
  578. )
  579. )
  580. def find_jar_iter(
  581. name_pattern,
  582. path_to_jar=None,
  583. env_vars=(),
  584. searchpath=(),
  585. url=None,
  586. verbose=False,
  587. is_regex=False,
  588. ):
  589. """
  590. Search for a jar that is used by nltk.
  591. :param name_pattern: The name of the jar file
  592. :param path_to_jar: The user-supplied jar location, or None.
  593. :param env_vars: A list of environment variable names to check
  594. in addition to the CLASSPATH variable which is
  595. checked by default.
  596. :param searchpath: List of directories to search.
  597. :param is_regex: Whether name is a regular expression.
  598. """
  599. assert isinstance(name_pattern, string_types)
  600. assert not isinstance(searchpath, string_types)
  601. if isinstance(env_vars, string_types):
  602. env_vars = env_vars.split()
  603. yielded = False
  604. # Make sure we check the CLASSPATH first
  605. env_vars = ['CLASSPATH'] + list(env_vars)
  606. # If an explicit location was given, then check it, and yield it if
  607. # it's present; otherwise, complain.
  608. if path_to_jar is not None:
  609. if os.path.isfile(path_to_jar):
  610. yielded = True
  611. yield path_to_jar
  612. else:
  613. raise LookupError(
  614. 'Could not find %s jar file at %s' % (name_pattern, path_to_jar)
  615. )
  616. # Check environment variables
  617. for env_var in env_vars:
  618. if env_var in os.environ:
  619. if env_var == 'CLASSPATH':
  620. classpath = os.environ['CLASSPATH']
  621. for cp in classpath.split(os.path.pathsep):
  622. if os.path.isfile(cp):
  623. filename = os.path.basename(cp)
  624. if (
  625. is_regex
  626. and re.match(name_pattern, filename)
  627. or (not is_regex and filename == name_pattern)
  628. ):
  629. if verbose:
  630. print('[Found %s: %s]' % (name_pattern, cp))
  631. yielded = True
  632. yield cp
  633. # The case where user put directory containing the jar file in the classpath
  634. if os.path.isdir(cp):
  635. if not is_regex:
  636. if os.path.isfile(os.path.join(cp, name_pattern)):
  637. if verbose:
  638. print('[Found %s: %s]' % (name_pattern, cp))
  639. yielded = True
  640. yield os.path.join(cp, name_pattern)
  641. else:
  642. # Look for file using regular expression
  643. for file_name in os.listdir(cp):
  644. if re.match(name_pattern, file_name):
  645. if verbose:
  646. print(
  647. '[Found %s: %s]'
  648. % (
  649. name_pattern,
  650. os.path.join(cp, file_name),
  651. )
  652. )
  653. yielded = True
  654. yield os.path.join(cp, file_name)
  655. else:
  656. jar_env = os.environ[env_var]
  657. jar_iter = (
  658. (
  659. os.path.join(jar_env, path_to_jar)
  660. for path_to_jar in os.listdir(jar_env)
  661. )
  662. if os.path.isdir(jar_env)
  663. else (jar_env,)
  664. )
  665. for path_to_jar in jar_iter:
  666. if os.path.isfile(path_to_jar):
  667. filename = os.path.basename(path_to_jar)
  668. if (
  669. is_regex
  670. and re.match(name_pattern, filename)
  671. or (not is_regex and filename == name_pattern)
  672. ):
  673. if verbose:
  674. print('[Found %s: %s]' % (name_pattern, path_to_jar))
  675. yielded = True
  676. yield path_to_jar
  677. # Check the path list.
  678. for directory in searchpath:
  679. if is_regex:
  680. for filename in os.listdir(directory):
  681. path_to_jar = os.path.join(directory, filename)
  682. if os.path.isfile(path_to_jar):
  683. if re.match(name_pattern, filename):
  684. if verbose:
  685. print('[Found %s: %s]' % (filename, path_to_jar))
  686. yielded = True
  687. yield path_to_jar
  688. else:
  689. path_to_jar = os.path.join(directory, name_pattern)
  690. if os.path.isfile(path_to_jar):
  691. if verbose:
  692. print('[Found %s: %s]' % (name_pattern, path_to_jar))
  693. yielded = True
  694. yield path_to_jar
  695. if not yielded:
  696. # If nothing was found, raise an error
  697. msg = "NLTK was unable to find %s!" % name_pattern
  698. if env_vars:
  699. msg += ' Set the %s environment variable' % env_vars[0]
  700. msg = textwrap.fill(msg + '.', initial_indent=' ', subsequent_indent=' ')
  701. if searchpath:
  702. msg += '\n\n Searched in:'
  703. msg += ''.join('\n - %s' % d for d in searchpath)
  704. if url:
  705. msg += '\n\n For more information, on %s, see:\n <%s>' % (
  706. name_pattern,
  707. url,
  708. )
  709. div = '=' * 75
  710. raise LookupError('\n\n%s\n%s\n%s' % (div, msg, div))
  711. def find_jar(
  712. name_pattern,
  713. path_to_jar=None,
  714. env_vars=(),
  715. searchpath=(),
  716. url=None,
  717. verbose=False,
  718. is_regex=False,
  719. ):
  720. return next(
  721. find_jar_iter(
  722. name_pattern, path_to_jar, env_vars, searchpath, url, verbose, is_regex
  723. )
  724. )
  725. def find_jars_within_path(path_to_jars):
  726. return [
  727. os.path.join(root, filename)
  728. for root, dirnames, filenames in os.walk(path_to_jars)
  729. for filename in fnmatch.filter(filenames, '*.jar')
  730. ]
  731. def _decode_stdoutdata(stdoutdata):
  732. """ Convert data read from stdout/stderr to unicode """
  733. if not isinstance(stdoutdata, bytes):
  734. return stdoutdata
  735. encoding = getattr(sys.__stdout__, "encoding", locale.getpreferredencoding())
  736. if encoding is None:
  737. return stdoutdata.decode()
  738. return stdoutdata.decode(encoding)
  739. ##########################################################################
  740. # Import Stdlib Module
  741. ##########################################################################
  742. def import_from_stdlib(module):
  743. """
  744. When python is run from within the nltk/ directory tree, the
  745. current directory is included at the beginning of the search path.
  746. Unfortunately, that means that modules within nltk can sometimes
  747. shadow standard library modules. As an example, the stdlib
  748. 'inspect' module will attempt to import the stdlib 'tokenize'
  749. module, but will instead end up importing NLTK's 'tokenize' module
  750. instead (causing the import to fail).
  751. """
  752. old_path = sys.path
  753. sys.path = [d for d in sys.path if d not in ('', '.')]
  754. m = __import__(module)
  755. sys.path = old_path
  756. return m
  757. ##########################################################################
  758. # Wrapper for ElementTree Elements
  759. ##########################################################################
  760. @compat.python_2_unicode_compatible
  761. class ElementWrapper(object):
  762. """
  763. A wrapper around ElementTree Element objects whose main purpose is
  764. to provide nicer __repr__ and __str__ methods. In addition, any
  765. of the wrapped Element's methods that return other Element objects
  766. are overridden to wrap those values before returning them.
  767. This makes Elements more convenient to work with in
  768. interactive sessions and doctests, at the expense of some
  769. efficiency.
  770. """
  771. # Prevent double-wrapping:
  772. def __new__(cls, etree):
  773. """
  774. Create and return a wrapper around a given Element object.
  775. If ``etree`` is an ``ElementWrapper``, then ``etree`` is
  776. returned as-is.
  777. """
  778. if isinstance(etree, ElementWrapper):
  779. return etree
  780. else:
  781. return object.__new__(ElementWrapper)
  782. def __init__(self, etree):
  783. r"""
  784. Initialize a new Element wrapper for ``etree``.
  785. If ``etree`` is a string, then it will be converted to an
  786. Element object using ``ElementTree.fromstring()`` first:
  787. >>> ElementWrapper("<test></test>")
  788. <Element "<?xml version='1.0' encoding='utf8'?>\n<test />">
  789. """
  790. if isinstance(etree, string_types):
  791. etree = ElementTree.fromstring(etree)
  792. self.__dict__['_etree'] = etree
  793. def unwrap(self):
  794. """
  795. Return the Element object wrapped by this wrapper.
  796. """
  797. return self._etree
  798. ##////////////////////////////////////////////////////////////
  799. # { String Representation
  800. ##////////////////////////////////////////////////////////////
  801. def __repr__(self):
  802. s = ElementTree.tostring(self._etree, encoding='utf8').decode('utf8')
  803. if len(s) > 60:
  804. e = s.rfind('<')
  805. if (len(s) - e) > 30:
  806. e = -20
  807. s = '%s...%s' % (s[:30], s[e:])
  808. return '<Element %r>' % s
  809. def __str__(self):
  810. """
  811. :return: the result of applying ``ElementTree.tostring()`` to
  812. the wrapped Element object.
  813. """
  814. return (
  815. ElementTree.tostring(self._etree, encoding='utf8').decode('utf8').rstrip()
  816. )
  817. ##////////////////////////////////////////////////////////////
  818. # { Element interface Delegation (pass-through)
  819. ##////////////////////////////////////////////////////////////
  820. def __getattr__(self, attrib):
  821. return getattr(self._etree, attrib)
  822. def __setattr__(self, attr, value):
  823. return setattr(self._etree, attr, value)
  824. def __delattr__(self, attr):
  825. return delattr(self._etree, attr)
  826. def __setitem__(self, index, element):
  827. self._etree[index] = element
  828. def __delitem__(self, index):
  829. del self._etree[index]
  830. def __setslice__(self, start, stop, elements):
  831. self._etree[start:stop] = elements
  832. def __delslice__(self, start, stop):
  833. del self._etree[start:stop]
  834. def __len__(self):
  835. return len(self._etree)
  836. ##////////////////////////////////////////////////////////////
  837. # { Element interface Delegation (wrap result)
  838. ##////////////////////////////////////////////////////////////
  839. def __getitem__(self, index):
  840. return ElementWrapper(self._etree[index])
  841. def __getslice__(self, start, stop):
  842. return [ElementWrapper(elt) for elt in self._etree[start:stop]]
  843. def getchildren(self):
  844. return [ElementWrapper(elt) for elt in self._etree]
  845. def getiterator(self, tag=None):
  846. return (ElementWrapper(elt) for elt in self._etree.getiterator(tag))
  847. def makeelement(self, tag, attrib):
  848. return ElementWrapper(self._etree.makeelement(tag, attrib))
  849. def find(self, path):
  850. elt = self._etree.find(path)
  851. if elt is None:
  852. return elt
  853. else:
  854. return ElementWrapper(elt)
  855. def findall(self, path):
  856. return [ElementWrapper(elt) for elt in self._etree.findall(path)]
  857. ######################################################################
  858. # Helper for Handling Slicing
  859. ######################################################################
  860. def slice_bounds(sequence, slice_obj, allow_step=False):
  861. """
  862. Given a slice, return the corresponding (start, stop) bounds,
  863. taking into account None indices and negative indices. The
  864. following guarantees are made for the returned start and stop values:
  865. - 0 <= start <= len(sequence)
  866. - 0 <= stop <= len(sequence)
  867. - start <= stop
  868. :raise ValueError: If ``slice_obj.step`` is not None.
  869. :param allow_step: If true, then the slice object may have a
  870. non-None step. If it does, then return a tuple
  871. (start, stop, step).
  872. """
  873. start, stop = (slice_obj.start, slice_obj.stop)
  874. # If allow_step is true, then include the step in our return
  875. # value tuple.
  876. if allow_step:
  877. step = slice_obj.step
  878. if step is None:
  879. step = 1
  880. # Use a recursive call without allow_step to find the slice
  881. # bounds. If step is negative, then the roles of start and
  882. # stop (in terms of default values, etc), are swapped.
  883. if step < 0:
  884. start, stop = slice_bounds(sequence, slice(stop, start))
  885. else:
  886. start, stop = slice_bounds(sequence, slice(start, stop))
  887. return start, stop, step
  888. # Otherwise, make sure that no non-default step value is used.
  889. elif slice_obj.step not in (None, 1):
  890. raise ValueError(
  891. 'slices with steps are not supported by %s' % sequence.__class__.__name__
  892. )
  893. # Supply default offsets.
  894. if start is None:
  895. start = 0
  896. if stop is None:
  897. stop = len(sequence)
  898. # Handle negative indices.
  899. if start < 0:
  900. start = max(0, len(sequence) + start)
  901. if stop < 0:
  902. stop = max(0, len(sequence) + stop)
  903. # Make sure stop doesn't go past the end of the list. Note that
  904. # we avoid calculating len(sequence) if possible, because for lazy
  905. # sequences, calculating the length of a sequence can be expensive.
  906. if stop > 0:
  907. try:
  908. sequence[stop - 1]
  909. except IndexError:
  910. stop = len(sequence)
  911. # Make sure start isn't past stop.
  912. start = min(start, stop)
  913. # That's all folks!
  914. return start, stop
  915. ######################################################################
  916. # Permission Checking
  917. ######################################################################
  918. def is_writable(path):
  919. # Ensure that it exists.
  920. if not os.path.exists(path):
  921. return False
  922. # If we're on a posix system, check its permissions.
  923. if hasattr(os, 'getuid'):
  924. statdata = os.stat(path)
  925. perm = stat.S_IMODE(statdata.st_mode)
  926. # is it world-writable?
  927. if perm & 0o002:
  928. return True
  929. # do we own it?
  930. elif statdata.st_uid == os.getuid() and (perm & 0o200):
  931. return True
  932. # are we in a group that can write to it?
  933. elif (statdata.st_gid in [os.getgid()] + os.getgroups()) and (perm & 0o020):
  934. return True
  935. # otherwise, we can't write to it.
  936. else:
  937. return False
  938. # Otherwise, we'll assume it's writable.
  939. # [xx] should we do other checks on other platforms?
  940. return True
  941. ######################################################################
  942. # NLTK Error reporting
  943. ######################################################################
  944. def raise_unorderable_types(ordering, a, b):
  945. raise TypeError(
  946. "unorderable types: %s() %s %s()"
  947. % (type(a).__name__, ordering, type(b).__name__)
  948. )