sbcharsetprober.py 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133
  1. ######################## BEGIN LICENSE BLOCK ########################
  2. # The Original Code is Mozilla Universal charset detector code.
  3. #
  4. # The Initial Developer of the Original Code is
  5. # Netscape Communications Corporation.
  6. # Portions created by the Initial Developer are Copyright (C) 2001
  7. # the Initial Developer. All Rights Reserved.
  8. #
  9. # Contributor(s):
  10. # Mark Pilgrim - port to Python
  11. # Shy Shalom - original C code
  12. #
  13. # This library is free software; you can redistribute it and/or
  14. # modify it under the terms of the GNU Lesser General Public
  15. # License as published by the Free Software Foundation; either
  16. # version 2.1 of the License, or (at your option) any later version.
  17. #
  18. # This library is distributed in the hope that it will be useful,
  19. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  21. # Lesser General Public License for more details.
  22. #
  23. # You should have received a copy of the GNU Lesser General Public
  24. # License along with this library; if not, write to the Free Software
  25. # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
  26. # 02110-1301 USA
  27. ######################### END LICENSE BLOCK #########################
  28. from .charsetprober import CharSetProber
  29. from .enums import CharacterCategory, ProbingState, SequenceLikelihood
  30. class SingleByteCharSetProber(CharSetProber):
  31. SAMPLE_SIZE = 64
  32. SB_ENOUGH_REL_THRESHOLD = 1024 # 0.25 * SAMPLE_SIZE^2
  33. POSITIVE_SHORTCUT_THRESHOLD = 0.95
  34. NEGATIVE_SHORTCUT_THRESHOLD = 0.05
  35. def __init__(self, model, reversed=False, name_prober=None):
  36. super(SingleByteCharSetProber, self).__init__()
  37. self._model = model
  38. # TRUE if we need to reverse every pair in the model lookup
  39. self._reversed = reversed
  40. # Optional auxiliary prober for name decision
  41. self._name_prober = name_prober
  42. self._last_order = None
  43. self._seq_counters = None
  44. self._total_seqs = None
  45. self._total_char = None
  46. self._freq_char = None
  47. self.reset()
  48. def reset(self):
  49. super(SingleByteCharSetProber, self).reset()
  50. # char order of last character
  51. self._last_order = 255
  52. self._seq_counters = [0] * SequenceLikelihood.get_num_categories()
  53. self._total_seqs = 0
  54. self._total_char = 0
  55. # characters that fall in our sampling range
  56. self._freq_char = 0
  57. @property
  58. def charset_name(self):
  59. if self._name_prober:
  60. return self._name_prober.charset_name
  61. else:
  62. return self._model['charset_name']
  63. @property
  64. def language(self):
  65. if self._name_prober:
  66. return self._name_prober.language
  67. else:
  68. return self._model.get('language')
  69. def feed(self, byte_str):
  70. if not self._model['keep_english_letter']:
  71. byte_str = self.filter_international_words(byte_str)
  72. if not byte_str:
  73. return self.state
  74. char_to_order_map = self._model['char_to_order_map']
  75. for i, c in enumerate(byte_str):
  76. # XXX: Order is in range 1-64, so one would think we want 0-63 here,
  77. # but that leads to 27 more test failures than before.
  78. order = char_to_order_map[c]
  79. # XXX: This was SYMBOL_CAT_ORDER before, with a value of 250, but
  80. # CharacterCategory.SYMBOL is actually 253, so we use CONTROL
  81. # to make it closer to the original intent. The only difference
  82. # is whether or not we count digits and control characters for
  83. # _total_char purposes.
  84. if order < CharacterCategory.CONTROL:
  85. self._total_char += 1
  86. if order < self.SAMPLE_SIZE:
  87. self._freq_char += 1
  88. if self._last_order < self.SAMPLE_SIZE:
  89. self._total_seqs += 1
  90. if not self._reversed:
  91. i = (self._last_order * self.SAMPLE_SIZE) + order
  92. model = self._model['precedence_matrix'][i]
  93. else: # reverse the order of the letters in the lookup
  94. i = (order * self.SAMPLE_SIZE) + self._last_order
  95. model = self._model['precedence_matrix'][i]
  96. self._seq_counters[model] += 1
  97. self._last_order = order
  98. charset_name = self._model['charset_name']
  99. if self.state == ProbingState.DETECTING:
  100. if self._total_seqs > self.SB_ENOUGH_REL_THRESHOLD:
  101. confidence = self.get_confidence()
  102. if confidence > self.POSITIVE_SHORTCUT_THRESHOLD:
  103. self.logger.debug('%s confidence = %s, we have a winner',
  104. charset_name, confidence)
  105. self._state = ProbingState.FOUND_IT
  106. elif confidence < self.NEGATIVE_SHORTCUT_THRESHOLD:
  107. self.logger.debug('%s confidence = %s, below negative '
  108. 'shortcut threshhold %s', charset_name,
  109. confidence,
  110. self.NEGATIVE_SHORTCUT_THRESHOLD)
  111. self._state = ProbingState.NOT_ME
  112. return self.state
  113. def get_confidence(self):
  114. r = 0.01
  115. if self._total_seqs > 0:
  116. r = ((1.0 * self._seq_counters[SequenceLikelihood.POSITIVE]) /
  117. self._total_seqs / self._model['typical_positive_ratio'])
  118. r = r * self._freq_char / self._total_char
  119. if r >= 1.0:
  120. r = 0.99
  121. return r