isri.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398
  1. # -*- coding: utf-8 -*-
  2. #
  3. # Natural Language Toolkit: The ISRI Arabic Stemmer
  4. #
  5. # Copyright (C) 2001-2019 NLTK Proejct
  6. # Algorithm: Kazem Taghva, Rania Elkhoury, and Jeffrey Coombs (2005)
  7. # Author: Hosam Algasaier <hosam_hme@yahoo.com>
  8. # URL: <http://nltk.org/>
  9. # For license information, see LICENSE.TXT
  10. """
  11. ISRI Arabic Stemmer
  12. The algorithm for this stemmer is described in:
  13. Taghva, K., Elkoury, R., and Coombs, J. 2005. Arabic Stemming without a root dictionary.
  14. Information Science Research Institute. University of Nevada, Las Vegas, USA.
  15. The Information Science Research Institute’s (ISRI) Arabic stemmer shares many features
  16. with the Khoja stemmer. However, the main difference is that ISRI stemmer does not use root
  17. dictionary. Also, if a root is not found, ISRI stemmer returned normalized form, rather than
  18. returning the original unmodified word.
  19. Additional adjustments were made to improve the algorithm:
  20. 1- Adding 60 stop words.
  21. 2- Adding the pattern (تفاعيل) to ISRI pattern set.
  22. 3- The step 2 in the original algorithm was normalizing all hamza. This step is discarded because it
  23. increases the word ambiguities and changes the original root.
  24. """
  25. from __future__ import unicode_literals
  26. import re
  27. from nltk.stem.api import StemmerI
  28. class ISRIStemmer(StemmerI):
  29. '''
  30. ISRI Arabic stemmer based on algorithm: Arabic Stemming without a root dictionary.
  31. Information Science Research Institute. University of Nevada, Las Vegas, USA.
  32. A few minor modifications have been made to ISRI basic algorithm.
  33. See the source code of this module for more information.
  34. isri.stem(token) returns Arabic root for the given token.
  35. The ISRI Stemmer requires that all tokens have Unicode string types.
  36. If you use Python IDLE on Arabic Windows you have to decode text first
  37. using Arabic '1256' coding.
  38. '''
  39. def __init__(self):
  40. # length three prefixes
  41. self.p3 = [
  42. '\u0643\u0627\u0644',
  43. '\u0628\u0627\u0644',
  44. '\u0648\u0644\u0644',
  45. '\u0648\u0627\u0644',
  46. ]
  47. # length two prefixes
  48. self.p2 = ['\u0627\u0644', '\u0644\u0644']
  49. # length one prefixes
  50. self.p1 = [
  51. '\u0644',
  52. '\u0628',
  53. '\u0641',
  54. '\u0633',
  55. '\u0648',
  56. '\u064a',
  57. '\u062a',
  58. '\u0646',
  59. '\u0627',
  60. ]
  61. # length three suffixes
  62. self.s3 = [
  63. '\u062a\u0645\u0644',
  64. '\u0647\u0645\u0644',
  65. '\u062a\u0627\u0646',
  66. '\u062a\u064a\u0646',
  67. '\u0643\u0645\u0644',
  68. ]
  69. # length two suffixes
  70. self.s2 = [
  71. '\u0648\u0646',
  72. '\u0627\u062a',
  73. '\u0627\u0646',
  74. '\u064a\u0646',
  75. '\u062a\u0646',
  76. '\u0643\u0645',
  77. '\u0647\u0646',
  78. '\u0646\u0627',
  79. '\u064a\u0627',
  80. '\u0647\u0627',
  81. '\u062a\u0645',
  82. '\u0643\u0646',
  83. '\u0646\u064a',
  84. '\u0648\u0627',
  85. '\u0645\u0627',
  86. '\u0647\u0645',
  87. ]
  88. # length one suffixes
  89. self.s1 = ['\u0629', '\u0647', '\u064a', '\u0643', '\u062a', '\u0627', '\u0646']
  90. # groups of length four patterns
  91. self.pr4 = {
  92. 0: ['\u0645'],
  93. 1: ['\u0627'],
  94. 2: ['\u0627', '\u0648', '\u064A'],
  95. 3: ['\u0629'],
  96. }
  97. # Groups of length five patterns and length three roots
  98. self.pr53 = {
  99. 0: ['\u0627', '\u062a'],
  100. 1: ['\u0627', '\u064a', '\u0648'],
  101. 2: ['\u0627', '\u062a', '\u0645'],
  102. 3: ['\u0645', '\u064a', '\u062a'],
  103. 4: ['\u0645', '\u062a'],
  104. 5: ['\u0627', '\u0648'],
  105. 6: ['\u0627', '\u0645'],
  106. }
  107. self.re_short_vowels = re.compile(r'[\u064B-\u0652]')
  108. self.re_hamza = re.compile(r'[\u0621\u0624\u0626]')
  109. self.re_initial_hamza = re.compile(r'^[\u0622\u0623\u0625]')
  110. self.stop_words = [
  111. '\u064a\u0643\u0648\u0646',
  112. '\u0648\u0644\u064a\u0633',
  113. '\u0648\u0643\u0627\u0646',
  114. '\u0643\u0630\u0644\u0643',
  115. '\u0627\u0644\u062a\u064a',
  116. '\u0648\u0628\u064a\u0646',
  117. '\u0639\u0644\u064a\u0647\u0627',
  118. '\u0645\u0633\u0627\u0621',
  119. '\u0627\u0644\u0630\u064a',
  120. '\u0648\u0643\u0627\u0646\u062a',
  121. '\u0648\u0644\u0643\u0646',
  122. '\u0648\u0627\u0644\u062a\u064a',
  123. '\u062a\u0643\u0648\u0646',
  124. '\u0627\u0644\u064a\u0648\u0645',
  125. '\u0627\u0644\u0644\u0630\u064a\u0646',
  126. '\u0639\u0644\u064a\u0647',
  127. '\u0643\u0627\u0646\u062a',
  128. '\u0644\u0630\u0644\u0643',
  129. '\u0623\u0645\u0627\u0645',
  130. '\u0647\u0646\u0627\u0643',
  131. '\u0645\u0646\u0647\u0627',
  132. '\u0645\u0627\u0632\u0627\u0644',
  133. '\u0644\u0627\u0632\u0627\u0644',
  134. '\u0644\u0627\u064a\u0632\u0627\u0644',
  135. '\u0645\u0627\u064a\u0632\u0627\u0644',
  136. '\u0627\u0635\u0628\u062d',
  137. '\u0623\u0635\u0628\u062d',
  138. '\u0623\u0645\u0633\u0649',
  139. '\u0627\u0645\u0633\u0649',
  140. '\u0623\u0636\u062d\u0649',
  141. '\u0627\u0636\u062d\u0649',
  142. '\u0645\u0627\u0628\u0631\u062d',
  143. '\u0645\u0627\u0641\u062a\u0626',
  144. '\u0645\u0627\u0627\u0646\u0641\u0643',
  145. '\u0644\u0627\u0633\u064a\u0645\u0627',
  146. '\u0648\u0644\u0627\u064a\u0632\u0627\u0644',
  147. '\u0627\u0644\u062d\u0627\u0644\u064a',
  148. '\u0627\u0644\u064a\u0647\u0627',
  149. '\u0627\u0644\u0630\u064a\u0646',
  150. '\u0641\u0627\u0646\u0647',
  151. '\u0648\u0627\u0644\u0630\u064a',
  152. '\u0648\u0647\u0630\u0627',
  153. '\u0644\u0647\u0630\u0627',
  154. '\u0641\u0643\u0627\u0646',
  155. '\u0633\u062a\u0643\u0648\u0646',
  156. '\u0627\u0644\u064a\u0647',
  157. '\u064a\u0645\u0643\u0646',
  158. '\u0628\u0647\u0630\u0627',
  159. '\u0627\u0644\u0630\u0649',
  160. ]
  161. def stem(self, token):
  162. """
  163. Stemming a word token using the ISRI stemmer.
  164. """
  165. token = self.norm(
  166. token, 1
  167. ) # remove diacritics which representing Arabic short vowels
  168. if token in self.stop_words:
  169. return token # exclude stop words from being processed
  170. token = self.pre32(
  171. token
  172. ) # remove length three and length two prefixes in this order
  173. token = self.suf32(
  174. token
  175. ) # remove length three and length two suffixes in this order
  176. token = self.waw(
  177. token
  178. ) # remove connective ‘و’ if it precedes a word beginning with ‘و’
  179. token = self.norm(token, 2) # normalize initial hamza to bare alif
  180. # if 4 <= word length <= 7, then stem; otherwise, no stemming
  181. if len(token) == 4: # length 4 word
  182. token = self.pro_w4(token)
  183. elif len(token) == 5: # length 5 word
  184. token = self.pro_w53(token)
  185. token = self.end_w5(token)
  186. elif len(token) == 6: # length 6 word
  187. token = self.pro_w6(token)
  188. token = self.end_w6(token)
  189. elif len(token) == 7: # length 7 word
  190. token = self.suf1(token)
  191. if len(token) == 7:
  192. token = self.pre1(token)
  193. if len(token) == 6:
  194. token = self.pro_w6(token)
  195. token = self.end_w6(token)
  196. return token
  197. def norm(self, word, num=3):
  198. """
  199. normalization:
  200. num=1 normalize diacritics
  201. num=2 normalize initial hamza
  202. num=3 both 1&2
  203. """
  204. if num == 1:
  205. word = self.re_short_vowels.sub('', word)
  206. elif num == 2:
  207. word = self.re_initial_hamza.sub('\u0627', word)
  208. elif num == 3:
  209. word = self.re_short_vowels.sub('', word)
  210. word = self.re_initial_hamza.sub('\u0627', word)
  211. return word
  212. def pre32(self, word):
  213. """remove length three and length two prefixes in this order"""
  214. if len(word) >= 6:
  215. for pre3 in self.p3:
  216. if word.startswith(pre3):
  217. return word[3:]
  218. if len(word) >= 5:
  219. for pre2 in self.p2:
  220. if word.startswith(pre2):
  221. return word[2:]
  222. return word
  223. def suf32(self, word):
  224. """remove length three and length two suffixes in this order"""
  225. if len(word) >= 6:
  226. for suf3 in self.s3:
  227. if word.endswith(suf3):
  228. return word[:-3]
  229. if len(word) >= 5:
  230. for suf2 in self.s2:
  231. if word.endswith(suf2):
  232. return word[:-2]
  233. return word
  234. def waw(self, word):
  235. """remove connective ‘و’ if it precedes a word beginning with ‘و’ """
  236. if len(word) >= 4 and word[:2] == '\u0648\u0648':
  237. word = word[1:]
  238. return word
  239. def pro_w4(self, word):
  240. """process length four patterns and extract length three roots"""
  241. if word[0] in self.pr4[0]: # مفعل
  242. word = word[1:]
  243. elif word[1] in self.pr4[1]: # فاعل
  244. word = word[:1] + word[2:]
  245. elif word[2] in self.pr4[2]: # فعال - فعول - فعيل
  246. word = word[:2] + word[3]
  247. elif word[3] in self.pr4[3]: # فعلة
  248. word = word[:-1]
  249. else:
  250. word = self.suf1(word) # do - normalize short sufix
  251. if len(word) == 4:
  252. word = self.pre1(word) # do - normalize short prefix
  253. return word
  254. def pro_w53(self, word):
  255. """process length five patterns and extract length three roots"""
  256. if word[2] in self.pr53[0] and word[0] == '\u0627': # افتعل - افاعل
  257. word = word[1] + word[3:]
  258. elif word[3] in self.pr53[1] and word[0] == '\u0645': # مفعول - مفعال - مفعيل
  259. word = word[1:3] + word[4]
  260. elif word[0] in self.pr53[2] and word[4] == '\u0629': # مفعلة - تفعلة - افعلة
  261. word = word[1:4]
  262. elif word[0] in self.pr53[3] and word[2] == '\u062a': # مفتعل - يفتعل - تفتعل
  263. word = word[1] + word[3:]
  264. elif word[0] in self.pr53[4] and word[2] == '\u0627': # مفاعل - تفاعل
  265. word = word[1] + word[3:]
  266. elif word[2] in self.pr53[5] and word[4] == '\u0629': # فعولة - فعالة
  267. word = word[:2] + word[3]
  268. elif word[0] in self.pr53[6] and word[1] == '\u0646': # انفعل - منفعل
  269. word = word[2:]
  270. elif word[3] == '\u0627' and word[0] == '\u0627': # افعال
  271. word = word[1:3] + word[4]
  272. elif word[4] == '\u0646' and word[3] == '\u0627': # فعلان
  273. word = word[:3]
  274. elif word[3] == '\u064a' and word[0] == '\u062a': # تفعيل
  275. word = word[1:3] + word[4]
  276. elif word[3] == '\u0648' and word[1] == '\u0627': # فاعول
  277. word = word[0] + word[2] + word[4]
  278. elif word[2] == '\u0627' and word[1] == '\u0648': # فواعل
  279. word = word[0] + word[3:]
  280. elif word[3] == '\u0626' and word[2] == '\u0627': # فعائل
  281. word = word[:2] + word[4]
  282. elif word[4] == '\u0629' and word[1] == '\u0627': # فاعلة
  283. word = word[0] + word[2:4]
  284. elif word[4] == '\u064a' and word[2] == '\u0627': # فعالي
  285. word = word[:2] + word[3]
  286. else:
  287. word = self.suf1(word) # do - normalize short sufix
  288. if len(word) == 5:
  289. word = self.pre1(word) # do - normalize short prefix
  290. return word
  291. def pro_w54(self, word):
  292. """process length five patterns and extract length four roots"""
  293. if word[0] in self.pr53[2]: # تفعلل - افعلل - مفعلل
  294. word = word[1:]
  295. elif word[4] == '\u0629': # فعللة
  296. word = word[:4]
  297. elif word[2] == '\u0627': # فعالل
  298. word = word[:2] + word[3:]
  299. return word
  300. def end_w5(self, word):
  301. """ending step (word of length five)"""
  302. if len(word) == 4:
  303. word = self.pro_w4(word)
  304. elif len(word) == 5:
  305. word = self.pro_w54(word)
  306. return word
  307. def pro_w6(self, word):
  308. """process length six patterns and extract length three roots"""
  309. if word.startswith('\u0627\u0633\u062a') or word.startswith(
  310. '\u0645\u0633\u062a'
  311. ): # مستفعل - استفعل
  312. word = word[3:]
  313. elif (
  314. word[0] == '\u0645' and word[3] == '\u0627' and word[5] == '\u0629'
  315. ): # مفعالة
  316. word = word[1:3] + word[4]
  317. elif (
  318. word[0] == '\u0627' and word[2] == '\u062a' and word[4] == '\u0627'
  319. ): # افتعال
  320. word = word[1] + word[3] + word[5]
  321. elif (
  322. word[0] == '\u0627' and word[3] == '\u0648' and word[2] == word[4]
  323. ): # افعوعل
  324. word = word[1] + word[4:]
  325. elif (
  326. word[0] == '\u062a' and word[2] == '\u0627' and word[4] == '\u064a'
  327. ): # تفاعيل new pattern
  328. word = word[1] + word[3] + word[5]
  329. else:
  330. word = self.suf1(word) # do - normalize short sufix
  331. if len(word) == 6:
  332. word = self.pre1(word) # do - normalize short prefix
  333. return word
  334. def pro_w64(self, word):
  335. """process length six patterns and extract length four roots"""
  336. if word[0] == '\u0627' and word[4] == '\u0627': # افعلال
  337. word = word[1:4] + word[5]
  338. elif word.startswith('\u0645\u062a'): # متفعلل
  339. word = word[2:]
  340. return word
  341. def end_w6(self, word):
  342. """ending step (word of length six)"""
  343. if len(word) == 5:
  344. word = self.pro_w53(word)
  345. word = self.end_w5(word)
  346. elif len(word) == 6:
  347. word = self.pro_w64(word)
  348. return word
  349. def suf1(self, word):
  350. """normalize short sufix"""
  351. for sf1 in self.s1:
  352. if word.endswith(sf1):
  353. return word[:-1]
  354. return word
  355. def pre1(self, word):
  356. """normalize short prefix"""
  357. for sp1 in self.p1:
  358. if word.startswith(sp1):
  359. return word[1:]
  360. return word